diff --git a/.changeset/fix-persisted-lifecycle-durability.md b/.changeset/fix-persisted-lifecycle-durability.md new file mode 100644 index 000000000..6b98d6164 --- /dev/null +++ b/.changeset/fix-persisted-lifecycle-durability.md @@ -0,0 +1,6 @@ +--- +'@tanstack/db-sqlite-persistence-core': patch +'@tanstack/electric-db-collection': patch +--- + +Fail-stop persisted collections when hydration or durability fails, while preserving lifecycle ownership across buffered and recovered sync work. Electric collections now surface these failures through the collection error state instead of continuing from an incomplete durable baseline. diff --git a/packages/db-sqlite-persistence-core/README.md b/packages/db-sqlite-persistence-core/README.md index 1ed225957..239f86343 100644 --- a/packages/db-sqlite-persistence-core/README.md +++ b/packages/db-sqlite-persistence-core/README.md @@ -81,6 +81,12 @@ while still handling per-collection schema versions correctly. - `InvalidPersistedStorageKeyError` - `InvalidPersistedStorageKeyEncodingError` - `PersistenceUnavailableError` +- `PersistenceDurabilityError` + +Adapter commit failures reject the applied receipt and become the collection's +terminal sync error as `PersistenceDurabilityError`. The error preserves its +original `cause` and any available string `code` and `path`. Hydration and +source errors remain distinct and are not wrapped as durability errors. ## Typical usage (via runtime wrappers) diff --git a/packages/db-sqlite-persistence-core/src/errors.ts b/packages/db-sqlite-persistence-core/src/errors.ts index f0ffc330e..8af8c525d 100644 --- a/packages/db-sqlite-persistence-core/src/errors.ts +++ b/packages/db-sqlite-persistence-core/src/errors.ts @@ -71,3 +71,27 @@ export class PersistenceUnavailableError extends PersistedCollectionCoreError { this.name = `PersistenceUnavailableError` } } + +export class PersistenceDurabilityError extends PersistedCollectionCoreError { + readonly code: string | undefined + readonly path: string + + constructor(cause: unknown, fallbackPath: string) { + const causeMessage = + cause instanceof Error ? cause.message : `Unknown persistence failure` + const causeRecord = + typeof cause === `object` && cause !== null + ? (cause as Record) + : undefined + const code = + typeof causeRecord?.code === `string` ? causeRecord.code : undefined + const path = + typeof causeRecord?.path === `string` ? causeRecord.path : fallbackPath + + super(`Persistence durability failed at ${path}: ${causeMessage}`) + this.name = `PersistenceDurabilityError` + this.cause = cause + this.code = code + this.path = path + } +} diff --git a/packages/db-sqlite-persistence-core/src/persisted.ts b/packages/db-sqlite-persistence-core/src/persisted.ts index 9ee0c810f..e164c32aa 100644 --- a/packages/db-sqlite-persistence-core/src/persisted.ts +++ b/packages/db-sqlite-persistence-core/src/persisted.ts @@ -1,4 +1,6 @@ import { + NoPendingSyncTransactionCommitError, + NoPendingSyncTransactionWriteError, SyncTransactionAbortedError, compileSingleRowExpression, safeRandomUUID, @@ -12,6 +14,7 @@ import { InvalidPersistedStorageKeyError, InvalidPersistenceAdapterError, InvalidSyncConfigError, + PersistenceDurabilityError, } from './errors' import type { StandardSchemaV1 } from '@standard-schema/spec' import type { @@ -29,6 +32,7 @@ import type { SyncConfig, SyncConfigRes, SyncMetadataApi, + SyncTransactionHandle, UpdateMutationFnParams, UtilsRecord, } from '@tanstack/db' @@ -441,6 +445,7 @@ type SyncControlFns = { commit: ((signal?: AbortSignal) => SyncAppliedReceipt) | null truncate: (() => void) | null metadata: SyncMetadataApi | null + markError: ((error: unknown) => void) | null } /** @@ -591,16 +596,25 @@ type BufferedSyncTransaction = { > truncate: boolean internal: boolean + lifecycleGeneration: number + beginOptions?: { immediate?: boolean } signal?: AbortSignal + applyToCollection: () => SyncAppliedReceipt + shouldFailStopOnAbort?: () => boolean resolveApplied?: () => void rejectApplied?: (error: unknown) => void } -type OpenSyncTransaction< - T extends object, - TKey extends string | number, -> = BufferedSyncTransaction & { +type OpenSyncTransaction = Omit< + BufferedSyncTransaction, + `applyToCollection` | `shouldFailStopOnAbort` +> & { + reservation?: SyncTransactionHandle + signal?: AbortSignal + applicationReceipt?: SyncAppliedReceipt queuedBecauseHydrating: boolean + hasDependentSuccessor: boolean + terminalFailure?: { error: unknown } } type SyncWriteNormalization = { @@ -619,13 +633,40 @@ type SyncWriteNormalization = { class ApplyMutex { private queue: Promise = Promise.resolve() + private pending = 0 + + run(task: () => Promise | T): Promise { + const runImmediately = this.pending === 0 + this.pending++ + + if (runImmediately) { + let releaseReservation!: () => void + this.queue = new Promise((resolve) => { + releaseReservation = resolve + }) + let taskPromise: Promise + try { + taskPromise = Promise.resolve(task()) + } catch (error) { + taskPromise = Promise.reject(error) + } + const markComplete = () => { + this.pending-- + releaseReservation() + } + void taskPromise.then(markComplete, markComplete) + return taskPromise + } - async run(task: () => Promise): Promise { const taskPromise = this.queue.then(() => task()) this.queue = taskPromise.then( () => undefined, () => undefined, ) + const markComplete = () => { + this.pending-- + } + void taskPromise.then(markComplete, markComplete) return taskPromise } } @@ -791,7 +832,10 @@ class PersistedCollectionRuntime< private readonly queuedHydrationTransactions: Array< BufferedSyncTransaction > = [] - private readonly queuedTxCommitted: Array = [] + private readonly queuedTxCommitted: Array<{ + txCommitted: TxCommitted + lifecycleGeneration: number + }> = [] private readonly requestIds = new WeakMap() private collection: Collection | null = @@ -802,6 +846,7 @@ class PersistedCollectionRuntime< commit: null, truncate: null, metadata: null, + markError: null, } private started = false private startupMetadataPromise: Promise | null = null @@ -812,6 +857,9 @@ class PersistedCollectionRuntime< private appliedReceiptSequence = 0 private readonly pendingAppliedReceipts = new Map>() private hydratingGeneration: number | null = null + private terminalFailure: + | { lifecycleGeneration: number; error: unknown } + | undefined private coordinatorUnsubscribe: (() => void) | null = null private indexAddedUnsubscribe: (() => void) | null = null private indexRemovedUnsubscribe: (() => void) | null = null @@ -864,13 +912,14 @@ class PersistedCollectionRuntime< ) } - clearSyncControls(): void { + private clearSyncControls(): void { this.syncControls = { begin: null, write: null, commit: null, truncate: null, metadata: null, + markError: null, } } @@ -878,6 +927,53 @@ class PersistedCollectionRuntime< return this.hydratingGeneration === this.lifecycleGeneration } + getCurrentTerminalFailure(): { error: unknown } | undefined { + const failure = this.terminalFailure + return failure?.lifecycleGeneration === this.lifecycleGeneration + ? failure + : undefined + } + + getLifecycleGeneration(): number { + return this.lifecycleGeneration + } + + private throwIfLifecycleReplaced(lifecycleGeneration: number): void { + if (lifecycleGeneration !== this.lifecycleGeneration) { + throw new SyncTransactionAbortedError() + } + } + + private throwIfTerminal(): void { + const failure = this.getCurrentTerminalFailure() + if (failure) throw failure.error + } + + private markTerminalFailure( + error: unknown, + lifecycleGeneration = this.lifecycleGeneration, + ): unknown { + if (lifecycleGeneration !== this.lifecycleGeneration) { + return error + } + + const existing = this.getCurrentTerminalFailure() + if (existing) { + return existing.error + } + + this.terminalFailure = { lifecycleGeneration, error } + this.pendingRemoteSubsetEnsures.clear() + this.queuedTxCommitted.length = 0 + if (this.remoteEnsureRetryTimer !== null) { + clearTimeout(this.remoteEnsureRetryTimer) + this.remoteEnsureRetryTimer = null + } + this.rejectQueuedHydrationTransactions(error) + this.syncControls.markError?.(error) + return error + } + isApplyingInternally(): boolean { return this.internalApplyDepth > 0 } @@ -901,16 +997,22 @@ class PersistedCollectionRuntime< } async ensureStarted(): Promise { + this.throwIfTerminal() if (this.startPromise) { return this.startPromise } const lifecycleGeneration = this.lifecycleGeneration - this.startPromise = this.startInternal(lifecycleGeneration) + this.startPromise = this.startInternal(lifecycleGeneration).catch( + (error) => { + throw this.markTerminalFailure(error, lifecycleGeneration) + }, + ) return this.startPromise } ensureResumeBaselineHydrated(): Promise { + this.throwIfTerminal() if (this.resumeBaselinePromise) { return this.resumeBaselinePromise } @@ -922,7 +1024,9 @@ class PersistedCollectionRuntime< if (this.syncMode !== `on-demand`) return await this.hydrateBaseline(lifecycleGeneration) - })() + })().catch((error) => { + throw this.markTerminalFailure(error, lifecycleGeneration) + }) return this.resumeBaselinePromise } @@ -944,13 +1048,17 @@ class PersistedCollectionRuntime< } async ensureStartupMetadataLoaded(): Promise { + this.throwIfTerminal() if (this.startupMetadataPromise) { return this.startupMetadataPromise } const lifecycleGeneration = this.lifecycleGeneration - this.startupMetadataPromise = - this.loadStartupMetadataInternal(lifecycleGeneration) + this.startupMetadataPromise = this.loadStartupMetadataInternal( + lifecycleGeneration, + ).catch((error) => { + throw this.markTerminalFailure(error, lifecycleGeneration) + }) return this.startupMetadataPromise } @@ -993,7 +1101,8 @@ class PersistedCollectionRuntime< const collectionMetadata = await this.loadCollectionMetadataSnapshot() if (lifecycleGeneration !== this.lifecycleGeneration) return - this.replaceCollectionMetadataSnapshot(collectionMetadata) + const applied = this.replaceCollectionMetadataSnapshot(collectionMetadata) + if (applied !== true) await applied } private async loadCollectionMetadataSnapshot(): Promise< @@ -1008,13 +1117,13 @@ class PersistedCollectionRuntime< private replaceCollectionMetadataSnapshot( collectionMetadata: Array<{ key: string; value: unknown }>, - ): void { + ): SyncAppliedReceipt { if ( !this.syncControls.begin || !this.syncControls.commit || !this.syncControls.metadata ) { - return + return true } const nextMetadata = new Map( @@ -1024,7 +1133,7 @@ class PersistedCollectionRuntime< .list() .map(({ key }) => key) - this.withInternalApply(() => { + return this.withInternalApply(() => { this.syncControls.begin?.({ immediate: true }) currentKeys.forEach((key) => { @@ -1037,7 +1146,7 @@ class PersistedCollectionRuntime< this.syncControls.metadata?.collection.set(key, value) }) - this.syncControls.commit?.() + return this.syncControls.commit?.() ?? true }) } @@ -1045,18 +1154,23 @@ class PersistedCollectionRuntime< options: LoadSubsetOptions, upstreamLoadSubset?: LoadSubsetFn, ): Promise { + this.throwIfTerminal() const lifecycleGeneration = this.lifecycleGeneration this.activeSubsets.set(this.getSubsetKey(options), options) const appliedCursor = this.appliedReceiptSequence - await this.applyMutex.run(() => - this.hydrateSubsetUnsafe(options, { - requestRemoteEnsure: this.mode === `sync-present`, - lifecycleGeneration, - }), - ) - if (lifecycleGeneration !== this.lifecycleGeneration) return - await this.waitForAppliedReceiptsAfter(appliedCursor) + try { + await this.applyMutex.run(() => + this.hydrateSubsetUnsafe(options, { + requestRemoteEnsure: this.mode === `sync-present`, + lifecycleGeneration, + }), + ) + if (lifecycleGeneration !== this.lifecycleGeneration) return + await this.waitForAppliedReceiptsAfter(appliedCursor) + } catch (error) { + throw this.markTerminalFailure(error, lifecycleGeneration) + } if (upstreamLoadSubset) { try { @@ -1084,12 +1198,14 @@ class PersistedCollectionRuntime< options: LoadSubsetOptions, upstreamUnloadSubset?: (options: LoadSubsetOptions) => void, ): void { - this.activeSubsets.delete(this.getSubsetKey(options)) - this.pendingRemoteSubsetEnsures.delete(this.getSubsetKey(options)) + const subsetKey = this.getSubsetKey(options) + this.activeSubsets.delete(subsetKey) + this.pendingRemoteSubsetEnsures.delete(subsetKey) upstreamUnloadSubset?.(options) } async forceReloadSubset(options: LoadSubsetOptions): Promise { + this.throwIfTerminal() const lifecycleGeneration = this.lifecycleGeneration // A one-shot refresh does not acquire an enduring subscription lease. await this.applyMutex.run(() => @@ -1103,17 +1219,61 @@ class PersistedCollectionRuntime< queueHydrationBufferedTransaction( transaction: BufferedSyncTransaction, ): void { + const failure = this.getCurrentTerminalFailure() + if (failure) { + transaction.rejectApplied?.(failure.error) + return + } this.queuedHydrationTransactions.push(transaction) } - async persistAndBroadcastExternalSyncTransaction( + applyHydrationBufferedTransaction( transaction: BufferedSyncTransaction, ): Promise { - await this.applyMutex.run(() => - this.persistAndBroadcastExternalSyncTransactionUnsafe(transaction), + const failure = this.getCurrentTerminalFailure() + if (failure) return Promise.reject(failure.error) + if (transaction.beginOptions?.immediate) { + return this.applyImmediateBufferedSyncTransaction(transaction) + } + return this.applyMutex.run(() => + this.applyBufferedSyncTransactionUnsafe(transaction), ) } + private async applyImmediateBufferedSyncTransaction( + transaction: BufferedSyncTransaction, + ): Promise { + try { + this.throwIfTerminal() + this.throwIfLifecycleReplaced(transaction.lifecycleGeneration) + if (transaction.signal?.aborted) { + throw new SyncTransactionAbortedError() + } + // Core immediate transactions are the escape hatch for a user mutation + // that depends on sync work. Apply that owned reservation without waiting + // behind a normal receipt, then serialize its durability suffix. + const applied = transaction.internal + ? this.withInternalApply(transaction.applyToCollection) + : transaction.applyToCollection() + if (applied !== true) await applied + this.throwIfLifecycleReplaced(transaction.lifecycleGeneration) + + if (!transaction.internal) { + await this.applyMutex.run(() => + this.persistAndBroadcastExternalSyncTransactionUnsafe(transaction), + ) + } + transaction.resolveApplied?.() + } catch (error) { + const terminalError = this.classifyBufferedTransactionFailure( + transaction, + error, + ) + transaction.rejectApplied?.(terminalError) + throw terminalError + } + } + normalizeSyncWriteMessage( message: ChangeMessageOrDeleteKeyMessage, ): SyncWriteNormalization { @@ -1177,14 +1337,21 @@ class PersistedCollectionRuntime< async persistAndConfirmCollectionMutations( mutations: Array>, + lifecycleGeneration = this.lifecycleGeneration, ): Promise { + this.throwIfTerminal() + this.throwIfLifecycleReplaced(lifecycleGeneration) if (mutations.length === 0) { return } await this.applyMutex.run(async () => { - const acceptedMutationIds = - await this.persistCollectionMutationsUnsafe(mutations) + this.throwIfLifecycleReplaced(lifecycleGeneration) + const acceptedMutationIds = await this.persistCollectionMutationsUnsafe( + mutations, + lifecycleGeneration, + ) + this.throwIfLifecycleReplaced(lifecycleGeneration) const acceptedMutationIdSet = new Set(acceptedMutationIds) const acceptedMutations = mutations.filter((mutation) => acceptedMutationIdSet.has(mutation.mutationId), @@ -1196,13 +1363,18 @@ class PersistedCollectionRuntime< ) } - this.confirmMutationsSyncUnsafe(acceptedMutations) + try { + await this.confirmMutationsSyncUnsafe(acceptedMutations) + } catch (error) { + throw this.markTerminalFailure(error, lifecycleGeneration) + } }) } async acceptTransactionMutations(transaction: { mutations: Array>> }): Promise { + this.throwIfTerminal() const collectionMutations = this.filterMutationsForCollection( transaction.mutations, ) @@ -1248,6 +1420,7 @@ class PersistedCollectionRuntime< this.startupMetadataPromise = null this.startPromise = null this.resumeBaselinePromise = null + this.terminalFailure = undefined } private withInternalApply(task: () => TResult): TResult { @@ -1293,6 +1466,7 @@ class PersistedCollectionRuntime< async scanPersistedRows( options?: PersistedRowScanOptions, ): Promise>> { + this.throwIfTerminal() return this.applyMutex.run(() => this.scanPersistedRowsUnsafe(options)) } @@ -1303,38 +1477,51 @@ class PersistedCollectionRuntime< lifecycleGeneration: number }, ): Promise { - this.hydratingGeneration = config.lifecycleGeneration try { - const rows = await this.loadSubsetRowsUnsafe(options) - if (config.lifecycleGeneration !== this.lifecycleGeneration) return + this.throwIfTerminal() + this.hydratingGeneration = config.lifecycleGeneration + try { + const rows = await this.loadSubsetRowsUnsafe(options) + if (config.lifecycleGeneration !== this.lifecycleGeneration) return + + const applied = this.applyRowsToCollection(rows) + if (applied !== true) await applied + } finally { + if (this.hydratingGeneration === config.lifecycleGeneration) { + this.hydratingGeneration = null + } + } - this.applyRowsToCollection(rows) - } finally { - if (this.hydratingGeneration === config.lifecycleGeneration) { - this.hydratingGeneration = null + await this.flushQueuedHydrationTransactionsUnsafe() + await this.flushQueuedTxCommittedUnsafe() + + if (config.requestRemoteEnsure) { + this.queueRemoteSubsetEnsure(options) } + } catch (error) { + throw this.markTerminalFailure(error, config.lifecycleGeneration) } + } - await this.flushQueuedHydrationTransactionsUnsafe() - await this.flushQueuedTxCommittedUnsafe() - - if (config.requestRemoteEnsure) { - this.queueRemoteSubsetEnsure(options) + private rejectQueuedHydrationTransactions(error: unknown): void { + for (const transaction of this.queuedHydrationTransactions) { + transaction.rejectApplied?.(error) } + this.queuedHydrationTransactions.length = 0 } private applyRowsToCollection( rows: Array<{ key: TKey; value: T; metadata?: unknown }>, - ): void { + ): SyncAppliedReceipt { if ( !this.syncControls.begin || !this.syncControls.write || !this.syncControls.commit ) { - return + return true } - this.withInternalApply(() => { + return this.withInternalApply(() => { this.syncControls.begin?.({ immediate: true }) for (const row of rows) { @@ -1348,21 +1535,21 @@ class PersistedCollectionRuntime< }) } - this.syncControls.commit?.() + return this.syncControls.commit?.() ?? true }) } private replaceCollectionSnapshot( rows: Array<{ key: TKey; value: T; metadata?: unknown }>, collectionMetadata: Array<{ key: string; value: unknown }>, - ): void { + ): SyncAppliedReceipt { if ( !this.syncControls.begin || !this.syncControls.write || !this.syncControls.commit || !this.syncControls.metadata ) { - return + return true } const nextMetadata = new Map( @@ -1372,7 +1559,7 @@ class PersistedCollectionRuntime< .list() .map(({ key }) => key) - this.withInternalApply(() => { + return this.withInternalApply(() => { this.syncControls.begin?.({ immediate: true }) this.syncControls.truncate?.() @@ -1394,7 +1581,7 @@ class PersistedCollectionRuntime< this.syncControls.metadata?.collection.set(key, value) }) - this.syncControls.commit?.() + return this.syncControls.commit?.() ?? true }) } @@ -1420,74 +1607,51 @@ class PersistedCollectionRuntime< private async applyBufferedSyncTransactionUnsafe( transaction: BufferedSyncTransaction, ): Promise { - if (transaction.signal?.aborted) { - transaction.rejectApplied?.(new SyncTransactionAbortedError()) - return - } - - const { begin, write, commit, truncate, metadata } = this.syncControls - if (!begin || !write || !commit) { - transaction.rejectApplied?.(new SyncTransactionAbortedError()) - return - } - - const applyToCollection = (): SyncAppliedReceipt => { - begin() - - if (transaction.truncate) { - truncate?.() - } - - for (const operation of transaction.operations) { - if (operation.type === `delete`) { - write({ - type: `delete`, - key: operation.key, - }) - } else { - write({ - type: `update`, - value: operation.value, - metadata: operation.metadata, - }) - } - } - - for (const [key, metadataWrite] of transaction.rowMetadataWrites) { - if (metadataWrite.type === `delete`) { - metadata?.row.delete(key) - } else { - metadata?.row.set(key, metadataWrite.value) - } - } - - for (const [key, metadataWrite] of transaction.collectionMetadataWrites) { - if (metadataWrite.type === `delete`) { - metadata?.collection.delete(key) - } else { - metadata?.collection.set(key, metadataWrite.value) - } - } - - return commit(transaction.signal) - } + this.throwIfTerminal() + this.throwIfLifecycleReplaced(transaction.lifecycleGeneration) try { const applied = transaction.internal - ? this.withInternalApply(applyToCollection) - : applyToCollection() + ? this.withInternalApply(transaction.applyToCollection) + : transaction.applyToCollection() if (applied !== true) { await applied } + this.throwIfLifecycleReplaced(transaction.lifecycleGeneration) if (!transaction.internal) { await this.persistAndBroadcastExternalSyncTransactionUnsafe(transaction) } transaction.resolveApplied?.() } catch (error) { - transaction.rejectApplied?.(error) - throw error + const aborted = + transaction.signal?.aborted || + error instanceof SyncTransactionAbortedError + const shouldFailStop = + aborted && transaction.shouldFailStopOnAbort?.() === true + const terminalError = this.classifyBufferedTransactionFailure( + transaction, + error, + ) + transaction.rejectApplied?.(terminalError) + if (aborted && !shouldFailStop && transaction.rejectApplied) { + return + } + throw terminalError + } + } + + private classifyBufferedTransactionFailure( + transaction: BufferedSyncTransaction, + error: unknown, + ): unknown { + const aborted = + transaction.signal?.aborted || + error instanceof SyncTransactionAbortedError + if (aborted && transaction.shouldFailStopOnAbort?.() !== true) { + return error } + return this.markTerminalFailure(error, transaction.lifecycleGeneration) } private async persistAndBroadcastExternalSyncTransactionUnsafe( @@ -1497,6 +1661,8 @@ class PersistedCollectionRuntime< return } + this.throwIfLifecycleReplaced(transaction.lifecycleGeneration) + const streamPosition = this.nextLocalStreamPosition() if ( @@ -1521,7 +1687,8 @@ class PersistedCollectionRuntime< const tx = this.createPersistedTxFromOperations(transaction, streamPosition) - await this.persistence.adapter.applyCommittedTx(this.collectionId, tx) + await this.applyCommittedTx(tx, transaction.lifecycleGeneration) + if (transaction.lifecycleGeneration !== this.lifecycleGeneration) return this.publishTxCommittedEvent( this.createTxCommittedPayload({ term: tx.term, @@ -1544,6 +1711,25 @@ class PersistedCollectionRuntime< ) } + private async applyCommittedTx( + tx: PersistedTx, + lifecycleGeneration: number, + ): Promise { + try { + this.throwIfTerminal() + await this.persistence.adapter.applyCommittedTx(this.collectionId, tx) + } catch (error) { + const durabilityError = + error instanceof PersistenceDurabilityError + ? error + : new PersistenceDurabilityError( + error, + `persistence.adapter.applyCommittedTx`, + ) + throw this.markTerminalFailure(durabilityError, lifecycleGeneration) + } + } + private createPersistedTxFromOperations( transaction: BufferedSyncTransaction, streamPosition: { term: number; seq: number; rowVersion: number }, @@ -1623,9 +1809,9 @@ class PersistedCollectionRuntime< } } - private confirmMutationsSyncUnsafe( + private async confirmMutationsSyncUnsafe( mutations: Array>, - ): void { + ): Promise { if ( !this.syncControls.begin || !this.syncControls.write || @@ -1634,7 +1820,7 @@ class PersistedCollectionRuntime< return } - this.withInternalApply(() => { + const applied = this.withInternalApply(() => { this.syncControls.begin?.({ immediate: true }) for (const mutation of mutations) { @@ -1651,8 +1837,12 @@ class PersistedCollectionRuntime< } } - this.syncControls.commit?.() + return this.syncControls.commit?.() ?? true }) + + if (applied !== true) { + await applied + } } private filterMutationsForCollection( @@ -1669,7 +1859,9 @@ class PersistedCollectionRuntime< private async persistCollectionMutationsUnsafe( mutations: Array>, + lifecycleGeneration: number, ): Promise> { + this.throwIfLifecycleReplaced(lifecycleGeneration) // When a coordinator with requestApplyLocalMutations is available, always // route through it — even on the leader tab. This ensures the coordinator's // seq/rowVersion counters stay in sync with actual writes. Without this, @@ -1689,6 +1881,8 @@ class PersistedCollectionRuntime< envelopeMutations, ) + this.throwIfLifecycleReplaced(lifecycleGeneration) + if (!response.ok) { throw new Error( `failed to apply local mutations through coordinator: ${response.error}`, @@ -1724,7 +1918,9 @@ class PersistedCollectionRuntime< // SingleProcessCoordinator). Apply directly and broadcast. const streamPosition = this.nextLocalStreamPosition() const tx = this.createPersistedTxFromMutations(mutations, streamPosition) - await this.persistence.adapter.applyCommittedTx(this.collectionId, tx) + await this.applyCommittedTx(tx, lifecycleGeneration) + + this.throwIfLifecycleReplaced(lifecycleGeneration) this.publishTxCommittedEvent( this.createTxCommittedPayload({ @@ -1871,19 +2067,23 @@ class PersistedCollectionRuntime< private queueRemoteSubsetEnsure(options: LoadSubsetOptions): void { if ( + this.getCurrentTerminalFailure() || this.mode !== `sync-present` || - !this.persistence.coordinator.requestEnsureRemoteSubset || - this.activeSubsets.get(this.getSubsetKey(options)) !== options + !this.persistence.coordinator.requestEnsureRemoteSubset ) { return } - this.pendingRemoteSubsetEnsures.set(this.getSubsetKey(options), options) + const subsetKey = this.getSubsetKey(options) + if (this.activeSubsets.get(subsetKey) !== options) return + + this.pendingRemoteSubsetEnsures.set(subsetKey, options) void this.flushPendingRemoteSubsetEnsures() } private scheduleRemoteEnsureRetry(): void { if ( + this.getCurrentTerminalFailure() || this.mode !== `sync-present` || !this.persistence.coordinator.requestEnsureRemoteSubset ) { @@ -1905,6 +2105,7 @@ class PersistedCollectionRuntime< private async flushPendingRemoteSubsetEnsures(): Promise { if ( + this.getCurrentTerminalFailure() || this.mode !== `sync-present` || !this.persistence.coordinator.requestEnsureRemoteSubset ) { @@ -1948,8 +2149,12 @@ class PersistedCollectionRuntime< if (message.collectionId !== this.collectionId) { return } + if (this.getCurrentTerminalFailure()) { + return + } const { payload } = message + const lifecycleGeneration = this.lifecycleGeneration const isSelf = message.senderId === this.persistence.coordinator.getNodeId() // Allow tx:committed from self — the coordinator produces these on behalf @@ -1957,14 +2162,17 @@ class PersistedCollectionRuntime< // processCommittedTxUnsafe prevents double-processing of our own writes. if (isTxCommittedPayload(payload)) { if (this.isHydratingNow()) { - this.queuedTxCommitted.push(payload) + this.queuedTxCommitted.push({ + txCommitted: payload, + lifecycleGeneration, + }) return } void this.applyMutex - .run(() => this.processCommittedTxUnsafe(payload)) + .run(() => this.processCommittedTxUnsafe(payload, lifecycleGeneration)) .catch((error) => { - console.warn(`Failed to process tx:committed message:`, error) + this.markTerminalFailure(error, lifecycleGeneration) }) return } @@ -1976,9 +2184,9 @@ class PersistedCollectionRuntime< if (isCollectionResetPayload(payload)) { void this.applyMutex - .run(() => this.truncateAndReloadUnsafe()) + .run(() => this.truncateAndReloadUnsafe(lifecycleGeneration)) .catch((error) => { - console.warn(`Failed to process collection reset message:`, error) + this.markTerminalFailure(error, lifecycleGeneration) }) } } @@ -1989,13 +2197,18 @@ class PersistedCollectionRuntime< if (!queued) { continue } - await this.processCommittedTxUnsafe(queued) + await this.processCommittedTxUnsafe( + queued.txCommitted, + queued.lifecycleGeneration, + ) } } private async processCommittedTxUnsafe( txCommitted: TxCommitted, + lifecycleGeneration = this.lifecycleGeneration, ): Promise { + if (lifecycleGeneration !== this.lifecycleGeneration) return if (txCommitted.term < this.latestTerm) { return } @@ -2015,7 +2228,8 @@ class PersistedCollectionRuntime< const hasGap = hasGapInCurrentTerm || hasGapAcrossTerms if (hasGap) { - await this.recoverFromSeqGapUnsafe() + await this.recoverFromSeqGapUnsafe(lifecycleGeneration) + if (lifecycleGeneration !== this.lifecycleGeneration) return if ( txCommitted.term < this.latestTerm || (txCommitted.term === this.latestTerm && @@ -2034,13 +2248,23 @@ class PersistedCollectionRuntime< await this.invalidateFromCommittedTxUnsafe(txCommitted) } - private async recoverFromSeqGapUnsafe(): Promise { + private async recoverFromSeqGapUnsafe( + lifecycleGeneration: number, + ): Promise { + if (lifecycleGeneration !== this.lifecycleGeneration) return if (this.persistence.coordinator.pullSince && this.latestRowVersion >= 0) { + let pullResponse: PullSinceResponse | undefined try { - const pullResponse = await this.persistence.coordinator.pullSince( + pullResponse = await this.persistence.coordinator.pullSince( this.collectionId, this.latestRowVersion, ) + } catch (error) { + console.warn(`Failed pullSince recovery attempt:`, error) + } + + if (pullResponse) { + if (lifecycleGeneration !== this.lifecycleGeneration) return if (pullResponse.ok) { this.observeStreamPosition( @@ -2054,6 +2278,7 @@ class PersistedCollectionRuntime< } for (const delta of pullResponse.deltas) { + if (lifecycleGeneration !== this.lifecycleGeneration) return await this.invalidateFromCommittedTxUnsafe({ type: `tx:committed`, term: pullResponse.latestTerm, @@ -2066,15 +2291,15 @@ class PersistedCollectionRuntime< rowMetadataMutations: delta.rowMetadataMutations, collectionMetadataMutations: delta.collectionMetadataMutations, }) + if (lifecycleGeneration !== this.lifecycleGeneration) return } return } - } catch (error) { - console.warn(`Failed pullSince recovery attempt:`, error) } } - await this.truncateAndReloadUnsafe() + if (lifecycleGeneration !== this.lifecycleGeneration) return + await this.truncateAndReloadUnsafe(lifecycleGeneration) if (this.mode === `sync-present`) { for (const options of this.activeSubsets.values()) { @@ -2083,15 +2308,20 @@ class PersistedCollectionRuntime< } } - private async truncateAndReloadUnsafe(): Promise { + private async truncateAndReloadUnsafe( + lifecycleGeneration = this.lifecycleGeneration, + ): Promise { + if (lifecycleGeneration !== this.lifecycleGeneration) return if (this.syncControls.begin && this.syncControls.commit) { - this.withInternalApply(() => { + const applied = this.withInternalApply(() => { this.syncControls.begin?.({ immediate: true }) this.syncControls.truncate?.() - this.syncControls.commit?.() + return this.syncControls.commit?.() ?? true }) + if (applied !== true) await applied } + if (lifecycleGeneration !== this.lifecycleGeneration) return await this.reloadActiveSubsetsUnsafe() } @@ -2131,7 +2361,7 @@ class PersistedCollectionRuntime< (opt) => (opt.where ? compileSingleRowExpression(opt.where) : null), ) - this.withInternalApply(() => { + const applied = this.withInternalApply(() => { this.syncControls.begin?.({ immediate: true }) for (const { @@ -2176,8 +2406,9 @@ class PersistedCollectionRuntime< } }) - this.syncControls.commit?.() + return this.syncControls.commit?.() ?? true }) + if (applied !== true) await applied } private async reloadActiveSubsetsUnsafe(): Promise { @@ -2203,7 +2434,7 @@ class PersistedCollectionRuntime< } } - this.replaceCollectionSnapshot( + const applied = this.replaceCollectionSnapshot( Array.from(mergedRows.entries()).map(([key, row]) => ({ key, value: row.value, @@ -2211,6 +2442,7 @@ class PersistedCollectionRuntime< })), collectionMetadata, ) + if (applied !== true) await applied } finally { if (this.hydratingGeneration === lifecycleGeneration) { this.hydratingGeneration = null @@ -2327,17 +2559,150 @@ function createWrappedSyncConfig< ...sourceSyncConfig, sync: (params) => { const transactionStack: Array> = [] + const pendingPublicationTransactions: Array< + OpenSyncTransaction + > = [] const getOpenTransaction = () => transactionStack[transactionStack.length - 1] + const markPendingMetadataDependency = ( + transaction: OpenSyncTransaction, + ) => { + const openTransaction = getOpenTransaction() + if (openTransaction && !openTransaction.internal) { + transaction.hasDependentSuccessor = true + } + } + const removePendingPublicationTransaction = ( + transaction: OpenSyncTransaction, + ) => { + const index = pendingPublicationTransactions.indexOf(transaction) + if (index !== -1) pendingPublicationTransactions.splice(index, 1) + } + const reservePublicationTransaction = ( + transaction: OpenSyncTransaction, + ) => { + const reservation = params.begin(transaction.beginOptions) + if (!reservation) { + throw new InvalidPersistedCollectionConfigError( + `wrapped sync begin did not reserve a publication transaction`, + ) + } + transaction.reservation = reservation + } + const settlePendingTransaction = ( + transaction: OpenSyncTransaction, + ) => { + removePendingPublicationTransaction(transaction) + } + const settleRuntimeTransaction = ( + transaction: OpenSyncTransaction, + applied: Promise, + ) => { + void applied.then( + () => settlePendingTransaction(transaction), + () => { + transaction.reservation?.abort() + settlePendingTransaction(transaction) + }, + ) + } + const getPendingRowMetadataWrite = (key: TKey) => { + for ( + let index = pendingPublicationTransactions.length - 1; + index >= 0; + index-- + ) { + const transaction = pendingPublicationTransactions[index]! + const write = transaction.rowMetadataWrites.get(key) + if (write) { + markPendingMetadataDependency(transaction) + return { found: true, write } + } + if (transaction.truncate) { + markPendingMetadataDependency(transaction) + return { + found: true, + write: { type: `delete` as const }, + } + } + } + return { found: false as const } + } + const getPendingCollectionMetadataWrite = (key: string) => { + for ( + let index = pendingPublicationTransactions.length - 1; + index >= 0; + index-- + ) { + const transaction = pendingPublicationTransactions[index]! + const write = transaction.collectionMetadataWrites.get(key) + if (write) { + markPendingMetadataDependency(transaction) + return { found: true, write } + } + } + return { found: false as const } + } let fullStartPromise: Promise | null = null const startupState = { cleanedUp: false } const acquisitions = new Map() + const getTerminalFailure = () => runtime.getCurrentTerminalFailure() + const createHandledRejection = (error: unknown): Promise => { + const rejected = Promise.reject(error) + void rejected.catch(() => undefined) + return rejected + } + const applyTransactionToCollection = ( + transaction: OpenSyncTransaction, + signal?: AbortSignal, + ): SyncAppliedReceipt => { + if (transaction.applicationReceipt !== undefined) { + return transaction.applicationReceipt + } + const reservation = transaction.reservation + if (!reservation) { + throw new InvalidPersistedCollectionConfigError( + `wrapped sync transaction lost its publication reservation`, + ) + } + try { + if (transaction.beginOptions?.immediate) { + const index = pendingPublicationTransactions.indexOf(transaction) + const predecessors = + index === -1 ? [] : pendingPublicationTransactions.slice(0, index) + for (const predecessor of predecessors) { + applyTransactionToCollection(predecessor, predecessor.signal) + } + } + const applied = reservation.commit(signal) + transaction.applicationReceipt = applied + if (applied === true) { + removePendingPublicationTransaction(transaction) + } else { + void applied.then( + () => removePendingPublicationTransaction(transaction), + () => removePendingPublicationTransaction(transaction), + ) + } + return applied + } catch (error) { + // Queued source transactions must remain publicly atomic, but later + // source work still needs their accepted metadata decisions while + // publication is pending. Once publication runs, the collection's + // metadata API becomes authoritative again. A prepublication failure + // removes the layer without exposing it. + reservation.abort() + removePendingPublicationTransaction(transaction) + throw error + } + } runtime.setSyncControls({ begin: params.begin, write: params.write as SyncControlFns[`write`], commit: params.commit, truncate: params.truncate, metadata: params.metadata ?? null, + markError: params.markError, }) runtime.setCollection( params.collection as Collection, @@ -2346,47 +2711,72 @@ function createWrappedSyncConfig< const wrappedParams = { ...params, markReady: () => { - if (startupState.cleanedUp) return + if (startupState.cleanedUp || getTerminalFailure()) return void (fullStartPromise ?? runtime.ensureStarted()) .then(() => { - if (startupState.cleanedUp) return - params.markReady() - }) - .catch((error) => { - if (startupState.cleanedUp) return - console.warn( - `Failed persisted sync startup before markReady:`, - error, - ) + if (startupState.cleanedUp || getTerminalFailure()) return params.markReady() }) + .catch(() => undefined) }, begin: (options?: { immediate?: boolean }) => { - if (startupState.cleanedUp) return + if (startupState.cleanedUp) return undefined + const terminalFailure = getTerminalFailure() + const internal = runtime.isApplyingInternally() const transaction: OpenSyncTransaction = { operations: [], rowMetadataWrites: new Map(), collectionMetadataWrites: new Map(), truncate: false, - internal: runtime.isApplyingInternally(), + internal, + lifecycleGeneration: runtime.getLifecycleGeneration(), + beginOptions: options, + hasDependentSuccessor: false, queuedBecauseHydrating: - !runtime.isApplyingInternally() && runtime.isHydratingNow(), + terminalFailure === undefined && + !internal && + runtime.isHydratingNow(), + ...(terminalFailure === undefined ? {} : { terminalFailure }), } - transactionStack.push(transaction) - - if (!transaction.queuedBecauseHydrating) { - params.begin(options) + if (!terminalFailure) { + reservePublicationTransaction(transaction) } + transactionStack.push(transaction) + // The persisted wrapper owns the core reservation. Upstream source + // callbacks continue through the wrapped write/commit surface so + // publication and durability remain one transaction. + return undefined }, write: (message: ChangeMessageOrDeleteKeyMessage) => { if (startupState.cleanedUp) return - const normalization = runtime.normalizeSyncWriteMessage(message) const openTransaction = getOpenTransaction() - - if (!openTransaction) { - params.write(normalization.forwardMessage) + const terminalFailure = + openTransaction?.terminalFailure ?? getTerminalFailure() + if (terminalFailure) { + if (openTransaction) + openTransaction.terminalFailure = terminalFailure return } + if (!openTransaction) { + throw new NoPendingSyncTransactionWriteError() + } + const normalization = runtime.normalizeSyncWriteMessage(message) + + if ( + message.type === `update` && + sourceSyncConfig.rowUpdateMode !== `full` + ) { + for (const pending of pendingPublicationTransactions) { + if ( + pending.truncate || + pending.operations.some( + (operation) => operation.key === normalization.operation.key, + ) + ) { + markPendingMetadataDependency(pending) + } + } + } openTransaction.operations.push(normalization.operation) if (normalization.operation.type === `delete`) { @@ -2418,8 +2808,22 @@ function createWrappedSyncConfig< value: normalization.operation.metadata, }) } - if (!openTransaction.queuedBecauseHydrating) { - params.write(normalization.forwardMessage) + openTransaction.reservation?.write( + normalization.forwardMessage, + normalization.operation.key, + ) + const stagedMetadata = openTransaction.rowMetadataWrites.get( + normalization.operation.key, + ) + if (stagedMetadata?.type === `delete`) { + openTransaction.reservation?.metadata.row.delete( + normalization.operation.key, + ) + } else if (stagedMetadata) { + openTransaction.reservation?.metadata.row.set( + normalization.operation.key, + stagedMetadata.value, + ) } }, metadata: params.metadata @@ -2442,6 +2846,12 @@ function createWrappedSyncConfig< if (openTransaction?.truncate) { return undefined } + const pending = getPendingRowMetadataWrite(key) + if (pending.found) { + return pending.write.type === `delete` + ? undefined + : pending.write.value + } return params.metadata!.row.get(key) }, scanPersisted: (options?: PersistedRowScanOptions) => @@ -2451,6 +2861,8 @@ function createWrappedSyncConfig< set: (key: TKey, value: unknown) => { if (startupState.cleanedUp) return const openTransaction = getOpenTransaction() + if (openTransaction?.terminalFailure ?? getTerminalFailure()) + return if (!openTransaction) { throw new InvalidPersistedCollectionConfigError( `metadata.row.set must be called within an open sync transaction`, @@ -2460,13 +2872,13 @@ function createWrappedSyncConfig< type: `set`, value, }) - if (!openTransaction.queuedBecauseHydrating) { - params.metadata!.row.set(key, value) - } + openTransaction.reservation?.metadata.row.set(key, value) }, delete: (key: TKey) => { if (startupState.cleanedUp) return const openTransaction = getOpenTransaction() + if (openTransaction?.terminalFailure ?? getTerminalFailure()) + return if (!openTransaction) { throw new InvalidPersistedCollectionConfigError( `metadata.row.delete must be called within an open sync transaction`, @@ -2475,9 +2887,7 @@ function createWrappedSyncConfig< openTransaction.rowMetadataWrites.set(key, { type: `delete`, }) - if (!openTransaction.queuedBecauseHydrating) { - params.metadata!.row.delete(key) - } + openTransaction.reservation?.metadata.row.delete(key) }, }, collection: { @@ -2491,11 +2901,19 @@ function createWrappedSyncConfig< ? undefined : pendingWrite.value } + const pending = getPendingCollectionMetadataWrite(key) + if (pending.found) { + return pending.write.type === `delete` + ? undefined + : pending.write.value + } return params.metadata!.collection.get(key) }, set: (key: string, value: unknown) => { if (startupState.cleanedUp) return const openTransaction = getOpenTransaction() + if (openTransaction?.terminalFailure ?? getTerminalFailure()) + return if (!openTransaction) { throw new InvalidPersistedCollectionConfigError( `metadata.collection.set must be called within an open sync transaction`, @@ -2505,13 +2923,16 @@ function createWrappedSyncConfig< type: `set`, value, }) - if (!openTransaction.queuedBecauseHydrating) { - params.metadata!.collection.set(key, value) - } + openTransaction.reservation?.metadata.collection.set( + key, + value, + ) }, delete: (key: string) => { if (startupState.cleanedUp) return const openTransaction = getOpenTransaction() + if (openTransaction?.terminalFailure ?? getTerminalFailure()) + return if (!openTransaction) { throw new InvalidPersistedCollectionConfigError( `metadata.collection.delete must be called within an open sync transaction`, @@ -2520,9 +2941,7 @@ function createWrappedSyncConfig< openTransaction.collectionMetadataWrites.set(key, { type: `delete`, }) - if (!openTransaction.queuedBecauseHydrating) { - params.metadata!.collection.delete(key) - } + openTransaction.reservation?.metadata.collection.delete(key) }, list: (prefix?: string) => { if (startupState.cleanedUp) return [] @@ -2531,6 +2950,28 @@ function createWrappedSyncConfig< .metadata!.collection.list() .map(({ key, value }) => [key, value]), ) + const pendingOwners = new Map< + string, + OpenSyncTransaction + >() + for (const transaction of pendingPublicationTransactions) { + for (const [ + key, + metadataWrite, + ] of transaction.collectionMetadataWrites) { + if (!prefix || key.startsWith(prefix)) { + pendingOwners.set(key, transaction) + } + if (metadataWrite.type === `delete`) { + merged.delete(key) + } else { + merged.set(key, metadataWrite.value) + } + } + } + for (const owner of new Set(pendingOwners.values())) { + markPendingMetadataDependency(owner) + } const openTransaction = getOpenTransaction() if (openTransaction) { for (const [ @@ -2558,9 +2999,9 @@ function createWrappedSyncConfig< truncate: () => { if (startupState.cleanedUp) return const openTransaction = getOpenTransaction() + if (openTransaction?.terminalFailure ?? getTerminalFailure()) return if (!openTransaction) { - params.truncate() - return + throw new NoPendingSyncTransactionWriteError() } openTransaction.operations = [] @@ -2570,23 +3011,52 @@ function createWrappedSyncConfig< // collection-scoped metadata before truncating row data, and those // writes must commit atomically with the truncate transaction. openTransaction.truncate = true - if (!openTransaction.queuedBecauseHydrating) { - params.truncate() - } + openTransaction.reservation?.truncate() }, commit: (signal?: AbortSignal) => { if (startupState.cleanedUp) return true const openTransaction = transactionStack.pop() + const terminalFailure = + openTransaction?.terminalFailure ?? getTerminalFailure() + if (terminalFailure) { + if (openTransaction) { + openTransaction.reservation?.abort() + settlePendingTransaction(openTransaction) + } + return createHandledRejection(terminalFailure.error) + } if (!openTransaction) { - return params.commit(signal) + throw new NoPendingSyncTransactionCommitError() } - if (openTransaction.queuedBecauseHydrating) { - if (signal?.aborted) { - const aborted = Promise.reject(new SyncTransactionAbortedError()) - void aborted.catch(() => undefined) - return aborted - } + if (openTransaction.internal) { + return openTransaction.reservation?.commit(signal) ?? true + } + + if (signal?.aborted) { + openTransaction.reservation?.abort() + settlePendingTransaction(openTransaction) + return createHandledRejection(new SyncTransactionAbortedError()) + } + const transaction = { + operations: openTransaction.operations, + rowMetadataWrites: openTransaction.rowMetadataWrites, + collectionMetadataWrites: openTransaction.collectionMetadataWrites, + truncate: openTransaction.truncate, + internal: false, + lifecycleGeneration: openTransaction.lifecycleGeneration, + beginOptions: openTransaction.beginOptions, + signal, + applyToCollection: () => + applyTransactionToCollection(openTransaction, signal), + shouldFailStopOnAbort: () => openTransaction.hasDependentSuccessor, + } + openTransaction.signal = signal + pendingPublicationTransactions.push(openTransaction) + if ( + openTransaction.queuedBecauseHydrating && + runtime.isHydratingNow() + ) { let resolveApplied!: () => void let rejectApplied!: (error: unknown) => void const applied = new Promise((resolve, reject) => { @@ -2595,36 +3065,16 @@ function createWrappedSyncConfig< }) void applied.catch(() => undefined) runtime.queueHydrationBufferedTransaction({ - operations: openTransaction.operations, - rowMetadataWrites: openTransaction.rowMetadataWrites, - collectionMetadataWrites: - openTransaction.collectionMetadataWrites, - truncate: openTransaction.truncate, - internal: openTransaction.internal, - signal, + ...transaction, resolveApplied, rejectApplied, }) + settleRuntimeTransaction(openTransaction, applied) return applied } - const applied = params.commit(signal) - if (!openTransaction.internal) { - const persistAfterApplication = async () => { - if (applied !== true) await applied - await runtime.persistAndBroadcastExternalSyncTransaction({ - operations: openTransaction.operations, - rowMetadataWrites: openTransaction.rowMetadataWrites, - collectionMetadataWrites: - openTransaction.collectionMetadataWrites, - truncate: openTransaction.truncate, - internal: false, - }) - } - const persisted = persistAfterApplication() - void persisted.catch(() => undefined) - return persisted - } + const applied = runtime.applyHydrationBufferedTransaction(transaction) + settleRuntimeTransaction(openTransaction, applied) return applied }, } @@ -2648,9 +3098,16 @@ function createWrappedSyncConfig< cleanup: () => { startupState.cleanedUp = true acquisitions.clear() + for (const transaction of new Set([ + ...transactionStack, + ...pendingPublicationTransactions, + ])) { + transaction.reservation?.abort() + } + pendingPublicationTransactions.length = 0 + transactionStack.length = 0 sourceResult.cleanup?.() runtime.cleanup() - runtime.clearSyncControls() }, loadSubset: async (options: LoadSubsetOptions) => { const acquisition = { forwarded: false } @@ -2709,6 +3166,7 @@ function createLoopbackSyncConfig< commit: params.commit, truncate: params.truncate, metadata: params.metadata ?? null, + markError: params.markError, }) runtime.setCollection( params.collection as Collection, @@ -2717,17 +3175,14 @@ function createLoopbackSyncConfig< void runtime .ensureStarted() .then(() => { + if (runtime.getCurrentTerminalFailure()) return params.markReady() }) - .catch((error) => { - console.warn(`Failed persisted loopback startup:`, error) - params.markReady() - }) + .catch(() => undefined) return { cleanup: () => { runtime.cleanup() - runtime.clearSyncControls() }, loadSubset: (options: LoadSubsetOptions) => runtime.loadSubset(options), unloadSubset: (options: LoadSubsetOptions) => @@ -2890,6 +3345,7 @@ export function persistedCollectionOptions< const wrappedOnInsert = async ( params: InsertMutationFnParams, ) => { + const lifecycleGeneration = runtime.getLifecycleGeneration() const handlerResult = localOnlyOptions.onInsert ? await localOnlyOptions.onInsert( params as unknown as InsertMutationFnParams, @@ -2898,6 +3354,7 @@ export function persistedCollectionOptions< await runtime.persistAndConfirmCollectionMutations( params.transaction.mutations as Array>, + lifecycleGeneration, ) return handlerResult ?? {} @@ -2906,6 +3363,7 @@ export function persistedCollectionOptions< const wrappedOnUpdate = async ( params: UpdateMutationFnParams, ) => { + const lifecycleGeneration = runtime.getLifecycleGeneration() const handlerResult = localOnlyOptions.onUpdate ? await localOnlyOptions.onUpdate( params as unknown as UpdateMutationFnParams, @@ -2914,6 +3372,7 @@ export function persistedCollectionOptions< await runtime.persistAndConfirmCollectionMutations( params.transaction.mutations as Array>, + lifecycleGeneration, ) return handlerResult ?? {} @@ -2922,6 +3381,7 @@ export function persistedCollectionOptions< const wrappedOnDelete = async ( params: DeleteMutationFnParams, ) => { + const lifecycleGeneration = runtime.getLifecycleGeneration() const handlerResult = localOnlyOptions.onDelete ? await localOnlyOptions.onDelete( params as unknown as DeleteMutationFnParams, @@ -2930,6 +3390,7 @@ export function persistedCollectionOptions< await runtime.persistAndConfirmCollectionMutations( params.transaction.mutations as Array>, + lifecycleGeneration, ) return handlerResult ?? {} diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index 313cdad86..9aa39e21b 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -1,17 +1,26 @@ import { describe, expect, it, vi } from 'vitest' +import { fc, test as fcTest } from '@fast-check/vitest' import { BasicIndex, DbClient, IR, + NoPendingSyncTransactionCommitError, + NoPendingSyncTransactionWriteError, collectionOptions, createCollection, createTransaction, } from '@tanstack/db' +import { + oraclePropertyOptions, + oracleRuns, + readOracleRunConfig, +} from '../../db/tests/oracle-config.js' import { InvalidPersistedCollectionCoordinatorError, InvalidPersistedStorageKeyEncodingError, InvalidPersistedStorageKeyError, InvalidSyncConfigError, + PersistenceDurabilityError, SingleProcessCoordinator, createPersistedTableName, decodePersistedStorageKey, @@ -27,7 +36,36 @@ import type { PullSinceResponse, TxCommitted, } from '../src' -import type { LoadSubsetOptions, SyncConfig } from '@tanstack/db' +import type { + LoadSubsetOptions, + PendingMutation, + SyncConfig, +} from '@tanstack/db' + +/** + * # Does persisted wrapping preserve one Collection history? + * + * RFC #1659 invariant 7 requires each accepted sync transaction to become + * durable, remain replayable, or fail through an observable channel. Startup + * hydrates rows and metadata, buffers concurrent source work, then publishes a + * coherent public snapshot. Publication precedes durability, but a rejected + * durability boundary must reject its applied receipt and fail-stop that sync + * run without admitting a suffix. + * + * `foldDurabilityLedger` is the independent model for append-only source + * obligations. The history grammar crosses hydration, a held adapter, + * immediate/normal ordering, independent/dependent aborts, reservation failure + * boundaries, ambient owner operations, receipt rejection, coordinator replay, + * cleanup, and restart. Tests drive the real persisted wrapper, Collection, + * coordinator, adapter, and local mutation path. Named checkpoints compare + * public and durable rows, metadata, receipt settlement, exact errors, call + * order, and lifecycle ownership. + * + * Fixed witnesses and bounded schedule tables preserve the known failure + * paths. The model's omission and reordering controls challenge its judgment. + * Native SQLite hosts, live Electric service behavior, adapter cancellation, + * and the open B2-failure/E4 schedule remain separate evidence. + */ /** * # Does persisted wrapping preserve one Collection history? @@ -53,8 +91,15 @@ import type { LoadSubsetOptions, SyncConfig } from '@tanstack/db' type Todo = { id: string title: string + detail?: string } +type TodoSyncParams = Parameters[`sync`]>[0] + +const requestedOracleReplayProperty = readOracleRunConfig().replayProperty +const describeUnlessOracleReplay = + requestedOracleReplayProperty === undefined ? describe : describe.skip + type RecordingAdapter = PersistenceAdapter & { applyCommittedTxCalls: Array<{ collectionId: string @@ -261,6 +306,52 @@ function createCoordinatorHarness(): CoordinatorHarness { return harness } +type FailStopCoordinatorHarness = PersistedCollectionCoordinator & { + emit: (payload: unknown, senderId?: string) => void + publishCalls: Array> + remoteEnsureCalls: Array + unsubscribeCalls: number +} + +function createFailStopCoordinatorHarness( + collectionId: string, +): FailStopCoordinatorHarness { + let subscriber: ((message: ProtocolEnvelope) => void) | undefined + const harness: FailStopCoordinatorHarness = { + publishCalls: [], + remoteEnsureCalls: [], + unsubscribeCalls: 0, + getNodeId: () => `fail-stop-coordinator`, + subscribe: (_collectionId, onMessage) => { + subscriber = onMessage + return () => { + harness.unsubscribeCalls++ + subscriber = undefined + } + }, + publish: (_collectionId, message) => { + harness.publishCalls.push(message) + }, + isLeader: () => true, + ensureLeadership: async () => {}, + requestEnsurePersistedIndex: async () => {}, + requestEnsureRemoteSubset: async (_collectionId, options) => { + harness.remoteEnsureCalls.push(options) + }, + emit: (payload, senderId = `remote-fail-stop-peer`) => { + subscriber?.({ + v: 1, + dbName: `test-db`, + collectionId, + senderId, + ts: Date.now(), + payload, + }) + }, + } + return harness +} + const stripVirtualProps = | undefined>( value: T, ): T => { @@ -279,207 +370,5536 @@ async function flushAsyncWork(delayMs: number = 0): Promise { await new Promise((resolve) => setTimeout(resolve, delayMs)) } -describe(`persistedCollectionOptions`, () => { - it(`provides a sync-absent loopback configuration with persisted utils`, async () => { - const adapter = createRecordingAdapter() - const collection = createCollection( - persistedCollectionOptions({ - id: `persisted-loopback`, - getKey: (item) => item.id, - persistence: { - adapter, - }, +function createEventGate(): { + promise: Promise + resolve: () => void + reject: (error: unknown) => void +} { + let resolve!: () => void + let reject!: (error: unknown) => void + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise + reject = rejectPromise + }) + return { promise, resolve, reject } +} + +async function atPersistedOracleCheckpoint( + promise: Promise, + label: string, + timeout = 1000, +): Promise { + let timer: ReturnType | undefined + try { + return await Promise.race([ + promise, + new Promise((_resolve, reject) => { + timer = setTimeout( + () => + reject( + new Error(`Persisted oracle checkpoint timed out: ${label}`), + ), + timeout, + ) }), + ]) + } finally { + clearTimeout(timer) + } +} + +async function cleanupPersistedOracle( + actions: Array<() => void | Promise>, + hasPrimaryFailure: boolean, +): Promise { + const failures: Array = [] + for (const [index, action] of actions.entries()) { + try { + await atPersistedOracleCheckpoint( + Promise.resolve().then(action), + `cleanup stage ${index}`, + 250, + ) + } catch (error) { + failures.push(error) + } + } + if (failures.length > 0) { + if (hasPrimaryFailure) { + console.warn( + `Persisted oracle cleanup failed after the primary failure:`, + failures, + ) + } else { + throw new AggregateError(failures, `Persisted oracle cleanup failed`) + } + } +} + +type DurabilityLedgerEvent = + | { type: `begin`; transactionId: string } + | { type: `write`; transactionId: string; row: Todo } + | { type: `commit`; transactionId: string } + | { type: `abort`; transactionId: string } + +// RFC #1659 durability law: a source commit remains one append-only obligation +// across a persistence hydration boundary. The independent fold deliberately +// has no hydration phase, queue, generation, or production transaction helper. +// These fixtures exercise the core adapter contract with in-memory storage; +// they do not claim native SQLite host or device execution. +function foldDurabilityLedger(events: ReadonlyArray): { + committedRows: Map + commitOrder: Array +} { + const open = new Map>() + const committedRows = new Map() + const commitOrder: Array = [] + + for (const event of events) { + if (event.type === `begin`) { + open.set(event.transactionId, []) + } else if (event.type === `write`) { + open.get(event.transactionId)?.push(structuredClone(event.row)) + } else if (event.type === `abort`) { + open.delete(event.transactionId) + } else { + const rows = open.get(event.transactionId) + if (!rows) continue + for (const row of rows) committedRows.set(row.id, row) + commitOrder.push(event.transactionId) + open.delete(event.transactionId) + } + } + + return { committedRows, commitOrder } +} + +function observeSettlement(promise: Promise): { + read: () => + | { status: `pending` } + | { status: `fulfilled` } + | { status: `rejected`; reason: unknown } +} { + let outcome: + | { status: `pending` } + | { status: `fulfilled` } + | { status: `rejected`; reason: unknown } = { status: `pending` } + void promise.then( + () => { + outcome = { status: `fulfilled` } + }, + (reason: unknown) => { + outcome = { status: `rejected`, reason } + }, + ) + return { read: () => outcome } +} + +async function runRejectedHydrationBufferWitness( + id: string, + rows: ReadonlyArray, + options: { commitAfterHydrationFailure?: boolean } = {}, +): Promise { + const hydrationEntered = createEventGate() + const releaseHydration = createEventGate() + const hydrationRejected = createEventGate() + const hydrationError = new Error(`persisted hydration rejected exactly`) + const adapter = createRecordingAdapter() + const successfulLoadSubset = adapter.loadSubset.bind(adapter) + adapter.loadSubset = async () => { + hydrationEntered.resolve() + await releaseHydration.promise + hydrationRejected.resolve() + throw hydrationError + } + + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: (() => true | Promise) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit + markReady() + }, + }, + persistence: { adapter }, + }), + ) + let hasPrimaryFailure = false + let initialCollectionCleaned = false + const cleanupActions: Array<() => void | Promise> = [ + () => (initialCollectionCleaned ? undefined : collection.cleanup()), + ] + const warning = vi.spyOn(console, `warn`).mockImplementation(() => {}) + + try { + void collection.stateWhenReady().catch(() => undefined) + await atPersistedOracleCheckpoint( + hydrationEntered.promise, + `${id} hydration entered`, ) + expect(remoteBegin).toBeTypeOf(`function`) + expect(remoteWrite).toBeTypeOf(`function`) + expect(remoteCommit).toBeTypeOf(`function`) - const insertTx = collection.insert({ - id: `1`, - title: `Phase 0`, + const committers = rows.map((row) => { + remoteBegin!() + remoteWrite!({ type: `insert`, value: row }) + return () => { + const receipt = remoteCommit!() + expect(receipt).toBeInstanceOf(Promise) + return observeSettlement(Promise.resolve(receipt).then(() => undefined)) + } }) + let receipts = options.commitAfterHydrationFailure + ? [] + : committers.map((commit) => commit()) + + releaseHydration.resolve() + await atPersistedOracleCheckpoint( + hydrationRejected.promise, + `${id} hydration rejected`, + ) + await flushAsyncWork() - await insertTx.isPersisted.promise + if (options.commitAfterHydrationFailure) { + receipts = committers.map((commit) => commit()) + await flushAsyncWork() + } - expect(stripVirtualProps(collection.get(`1`))).toEqual({ - id: `1`, - title: `Phase 0`, + const receiptOutcomes = receipts.map((receipt) => { + const outcome = receipt.read() + return outcome.status === `rejected` + ? { + status: outcome.status, + exactReason: outcome.reason === hydrationError, + } + : outcome }) - expect(adapter.applyCommittedTxCalls).toHaveLength(1) - expect(adapter.applyCommittedTxCalls[0]?.tx.mutations[0]?.type).toBe( - `insert`, + expect({ + status: collection.status, + exactPublicError: collection._lifecycle.getSyncError() === hydrationError, + receiptOutcomes, + visibleRows: rows.filter((row) => collection.has(row.id)), + durableRows: rows.filter((row) => adapter.rows.has(row.id)), + durabilityCalls: adapter.applyCommittedTxCalls.length, + }).toEqual({ + status: `error`, + exactPublicError: true, + receiptOutcomes: rows.map(() => ({ + status: `rejected`, + exactReason: true, + })), + visibleRows: [], + durableRows: [], + durabilityCalls: 0, + }) + + await atPersistedOracleCheckpoint( + collection.cleanup(), + `${id} failed collection cleanup`, ) - expect(typeof collection.utils.acceptMutations).toBe(`function`) - expect(collection.utils.getLeadershipState?.().isLeader).toBe(true) - }) + initialCollectionCleaned = true + adapter.loadSubset = successfulLoadSubset - it(`supports acceptMutations for manual transactions`, async () => { - const adapter = createRecordingAdapter() - const collection = createCollection( + const reopened = createCollection( persistedCollectionOptions({ - id: `persisted-manual`, + id, getKey: (item) => item.id, - persistence: { - adapter, - }, + persistence: { adapter }, }), ) + cleanupActions.push(() => reopened.cleanup()) + await atPersistedOracleCheckpoint( + reopened.stateWhenReady(), + `${id} reopened hydration`, + ) + await flushAsyncWork() - const tx = createTransaction({ - autoCommit: false, - mutationFn: async ({ transaction }) => { - await collection.utils.acceptMutations(transaction) - }, + expect({ + visibleRows: rows.filter((row) => reopened.has(row.id)), + durableRows: rows.filter((row) => adapter.rows.has(row.id)), + rowMetadata: Array.from(adapter.rowMetadata.entries()), + collectionMetadata: Array.from(adapter.collectionMetadata.entries()), + durabilityCalls: adapter.applyCommittedTxCalls.length, + }).toEqual({ + visibleRows: [], + durableRows: [], + rowMetadata: [], + collectionMetadata: [], + durabilityCalls: 0, }) - tx.mutate(() => { - collection.insert({ - id: `manual-1`, - title: `Manual`, - }) + const freshRow = { id: `${id}:fresh`, title: `post-restart control` } + const freshTransaction = reopened.insert(freshRow) + await atPersistedOracleCheckpoint( + (async () => { + await freshTransaction.isPersisted.promise + })(), + `${id} fresh transaction persisted`, + ) + await flushAsyncWork() + + expect({ + visibleFreshRow: stripVirtualProps(reopened.get(freshRow.id)), + durableFreshRow: adapter.rows.get(freshRow.id), + staleVisibleRows: rows.filter((row) => reopened.has(row.id)), + staleDurableRows: rows.filter((row) => adapter.rows.has(row.id)), + rowMetadata: Array.from(adapter.rowMetadata.entries()), + collectionMetadata: Array.from(adapter.collectionMetadata.entries()), + appliedKeys: adapter.applyCommittedTxCalls.flatMap(({ tx }) => + tx.mutations.map((mutation) => mutation.key), + ), + }).toEqual({ + visibleFreshRow: freshRow, + durableFreshRow: freshRow, + staleVisibleRows: [], + staleDurableRows: [], + rowMetadata: [], + collectionMetadata: [], + appliedKeys: [freshRow.id], }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseHydration.resolve() + warning.mockRestore() + await cleanupPersistedOracle(cleanupActions, hasPrimaryFailure) + } +} - await tx.commit() +let generatedOwnershipHistoryId = 0 - expect(stripVirtualProps(collection.get(`manual-1`))).toEqual({ - id: `manual-1`, - title: `Manual`, - }) - expect(adapter.applyCommittedTxCalls).toHaveLength(1) +function sortedTodoRows(rows: Iterable): Array { + return Array.from(rows).sort((left, right) => left.id.localeCompare(right.id)) +} + +type ImmediateOrderingObservation = { + publicRows: Array + durableRows: Array + status: string + pendingMarkers: number +} + +function expectImmediateOrderingObservation( + actual: ImmediateOrderingObservation, + expectedRows: ReadonlyArray, +): void { + expect(actual).toEqual({ + publicRows: expectedRows, + durableRows: expectedRows, + status: `ready`, + pendingMarkers: 0, }) +} - it(`loads collection metadata into collection state during startup`, async () => { - const adapter = createRecordingAdapter() - adapter.collectionMetadata.set(`electric:resume`, { - kind: `resume`, - offset: `10_0`, - handle: `handle-1`, - shapeId: `shape-1`, - updatedAt: 1, - }) +type AbortGraphObservation = { + predecessorStatus: `fulfilled` | `rejected` + predecessorName: string | undefined + successorStatus: `fulfilled` | `rejected` + sameFailure: boolean + status: string + publicSuccessor: Todo | undefined + durableSuccessor: Todo | undefined + pendingMarkers: number +} - const collection = createCollection( - persistedCollectionOptions({ - id: `persisted-startup-metadata`, - getKey: (item) => item.id, - persistence: { - adapter, +function expectAbortGraphObservation( + actual: AbortGraphObservation, + relationship: `independent` | `same-key`, + successorKey: string, + successorTitle: string, +): void { + expect(actual).toEqual( + relationship === `same-key` + ? { + predecessorStatus: `rejected`, + predecessorName: `AbortError`, + successorStatus: `rejected`, + sameFailure: true, + status: `error`, + publicSuccessor: undefined, + durableSuccessor: undefined, + pendingMarkers: 0, + } + : { + predecessorStatus: `rejected`, + predecessorName: `AbortError`, + successorStatus: `fulfilled`, + sameFailure: false, + status: `ready`, + publicSuccessor: { id: successorKey, title: successorTitle }, + durableSuccessor: { id: successorKey, title: successorTitle }, + pendingMarkers: 0, }, - }), - ) + ) +} - await collection.stateWhenReady() +type OwnerIsolationAdmissionObservation = { + exactError: boolean + strayVisible: boolean +} - expect(adapter.loadCollectionMetadataCalls).toEqual([ - `persisted-startup-metadata`, - ]) - expect( - collection._state.syncedCollectionMetadata.get(`electric:resume`), - ).toEqual({ - kind: `resume`, - offset: `10_0`, - handle: `handle-1`, - shapeId: `shape-1`, - updatedAt: 1, - }) +function expectOwnerIsolationAdmission( + actual: OwnerIsolationAdmissionObservation, +): void { + expect(actual).toEqual({ exactError: true, strayVisible: false }) +} + +type ReservationBoundaryObservation = { + status: string + pendingMarkers: number +} + +function expectReservationBoundaryObservation( + actual: ReservationBoundaryObservation, + boundary: `abort` | `cleanup` | `terminal-failure`, +): void { + expect(actual).toEqual({ + status: + boundary === `cleanup` + ? `cleaned-up` + : boundary === `abort` + ? `ready` + : `error`, + pendingMarkers: 0, }) +} - it(`restores row and collection metadata after metadata-bearing full reload`, async () => { - const adapter = createRecordingAdapter([ - { - id: `1`, - title: `Tracked`, +async function runImmediateOrderingLaw( + relation: `same-key` | `disjoint`, + olderTitle: string, + newerTitle: string, +): Promise { + const historyId = ++generatedOwnershipHistoryId + const adapter = createRecordingAdapter() + const firstPersistenceEntered = createEventGate() + const releaseFirstPersistence = createEventGate() + const applyCommittedTx = adapter.applyCommittedTx.bind(adapter) + let applyCalls = 0 + adapter.applyCommittedTx = async (...args) => { + applyCalls++ + if (applyCalls === 1) { + firstPersistenceEntered.resolve() + await releaseFirstPersistence.promise + } + await applyCommittedTx(...args) + } + let sourceParams!: TodoSyncParams + const collection = createCollection( + persistedCollectionOptions({ + id: `generated-immediate-order-${historyId}`, + getKey: (row) => row.id, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() + }, }, - ]) - adapter.rowMetadata.set(`1`, { - source: `initial`, - }) - adapter.collectionMetadata.set(`electric:resume`, { - kind: `resume`, - offset: `10_0`, - handle: `handle-1`, - shapeId: `shape-1`, - updatedAt: 1, - }) - const coordinator = createCoordinatorHarness() + persistence: { adapter }, + }), + ) + const receipts: Array> = [] + let hasPrimaryFailure = false - const collection = createCollection( - persistedCollectionOptions({ - id: `sync-present`, - getKey: (item) => item.id, - sync: { - sync: ({ markReady }) => { - markReady() - }, - }, - persistence: { - adapter, - coordinator, - }, - }), + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `generated immediate ordering ready`, ) - - await collection.preload() - await flushAsyncWork() - - expect(collection._state.syncedMetadata.get(`1`)).toEqual({ - source: `initial`, - }) - expect( - collection._state.syncedCollectionMetadata.get(`electric:resume`), - ).toEqual({ - kind: `resume`, - offset: `10_0`, - handle: `handle-1`, - shapeId: `shape-1`, - updatedAt: 1, + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `gate`, title: `holds adapter` }, }) + receipts.push(Promise.resolve(sourceParams.commit()).then(() => undefined)) + receipts.forEach((receipt) => void receipt.catch(() => undefined)) + await atPersistedOracleCheckpoint( + firstPersistenceEntered.promise, + `generated immediate ordering adapter hold`, + ) - adapter.rowMetadata.set(`1`, { - source: `reloaded`, - }) - adapter.collectionMetadata.delete(`electric:resume`) - adapter.collectionMetadata.set(`queryCollection:gc:q1`, { - queryHash: `q1`, - mode: `until-revalidated`, - }) + const olderKey = relation === `same-key` ? `shared` : `older` + const newerKey = relation === `same-key` ? `shared` : `newer` + const expected = new Map([ + [`gate`, { id: `gate`, title: `holds adapter` }], + ]) + expected.set(olderKey, { id: olderKey, title: olderTitle }) - coordinator.emit({ - type: `tx:committed`, - term: 1, - seq: 1, - txId: `tx-reload`, - latestRowVersion: 1, - requiresFullReload: true, + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: olderKey, title: olderTitle }, }) + const olderReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + receipts.push(olderReceipt) + void olderReceipt.catch(() => undefined) - await flushAsyncWork() - await flushAsyncWork() + sourceParams.begin({ immediate: true }) + sourceParams.write({ + type: relation === `same-key` ? `update` : `insert`, + value: { id: newerKey, title: newerTitle }, + }) + expected.set(newerKey, { id: newerKey, title: newerTitle }) + const newerReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + receipts.push(newerReceipt) + void newerReceipt.catch(() => undefined) - expect(collection._state.syncedMetadata.get(`1`)).toEqual({ - source: `reloaded`, - }) expect( - collection._state.syncedCollectionMetadata.has(`electric:resume`), - ).toBe(false) + sortedTodoRows(Array.from(collection.values()).map(stripVirtualProps)), + ).toEqual(sortedTodoRows(expected.values())) + + releaseFirstPersistence.resolve() + for (const [index, receipt] of receipts.entries()) { + await atPersistedOracleCheckpoint( + receipt, + `generated immediate receipt ${index}`, + ) + } + expectImmediateOrderingObservation( + { + publicRows: sortedTodoRows( + Array.from(collection.values()).map(stripVirtualProps), + ), + durableRows: sortedTodoRows(adapter.rows.values()), + status: collection.status, + pendingMarkers: collection._state.pendingSyncedTransactions.length, + }, + sortedTodoRows(expected.values()), + ) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseFirstPersistence.resolve() + await cleanupPersistedOracle( + [ + ...receipts.map((receipt) => () => receipt.catch(() => undefined)), + () => collection.cleanup(), + ], + hasPrimaryFailure, + ) + } +} + +async function runAbortRelationshipLaw( + relationship: `independent` | `same-key`, + predecessorTitle: string, + successorTitle: string, +): Promise { + const historyId = ++generatedOwnershipHistoryId + const adapter = createRecordingAdapter() + const firstPersistenceEntered = createEventGate() + const releaseFirstPersistence = createEventGate() + const applyCommittedTx = adapter.applyCommittedTx.bind(adapter) + let applyCalls = 0 + adapter.applyCommittedTx = async (...args) => { + applyCalls++ + if (applyCalls === 1) { + firstPersistenceEntered.resolve() + await releaseFirstPersistence.promise + } + await applyCommittedTx(...args) + } + let sourceParams!: TodoSyncParams + const collection = createCollection( + persistedCollectionOptions({ + id: `generated-abort-graph-${historyId}`, + getKey: (row) => row.id, + sync: { + rowUpdateMode: `partial`, + sync: (params) => { + sourceParams = params + params.markReady() + }, + }, + persistence: { adapter }, + }), + ) + const abortController = new AbortController() + const receipts: Array> = [] + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `generated abort graph ready`, + ) + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `gate`, title: `holds adapter` }, + }) + const gateReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + receipts.push(gateReceipt) + void gateReceipt.catch(() => undefined) + await atPersistedOracleCheckpoint( + firstPersistenceEntered.promise, + `generated abort graph adapter hold`, + ) + + const predecessorKey = relationship === `same-key` ? `shared` : `aborted` + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { + id: predecessorKey, + title: predecessorTitle, + detail: `predecessor-only`, + }, + }) + const predecessorReceipt = Promise.resolve( + sourceParams.commit(abortController.signal), + ).then(() => undefined) + receipts.push(predecessorReceipt) + void predecessorReceipt.catch(() => undefined) + + const successorKey = relationship === `same-key` ? `shared` : `survivor` + sourceParams.begin() + sourceParams.write({ + type: relationship === `same-key` ? `update` : `insert`, + value: { id: successorKey, title: successorTitle }, + }) + const successorReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + receipts.push(successorReceipt) + void successorReceipt.catch(() => undefined) + + abortController.abort() + releaseFirstPersistence.resolve() + await atPersistedOracleCheckpoint(gateReceipt, `generated abort gate`) + const predecessor = await atPersistedOracleCheckpoint( + predecessorReceipt.then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `generated predecessor abort`, + ) + const successor = await atPersistedOracleCheckpoint( + successorReceipt.then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `generated successor settlement`, + ) + const abortError = + predecessor.status === `rejected` ? predecessor.reason : undefined + + expectAbortGraphObservation( + { + predecessorStatus: predecessor.status, + predecessorName: + abortError instanceof Error ? abortError.name : undefined, + successorStatus: successor.status, + sameFailure: + successor.status === `rejected` && successor.reason === abortError, + status: collection.status, + publicSuccessor: stripVirtualProps(collection.get(successorKey)), + durableSuccessor: adapter.rows.get(successorKey), + pendingMarkers: collection._state.pendingSyncedTransactions.length, + }, + relationship, + successorKey, + successorTitle, + ) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseFirstPersistence.resolve() + await cleanupPersistedOracle( + [ + ...receipts.map((receipt) => () => receipt.catch(() => undefined)), + () => collection.cleanup(), + ], + hasPrimaryFailure, + ) + } +} + +async function runBareOwnerOperationLaw( + operation: `write` | `commit` | `truncate`, + queuedTitle: string, +): Promise { + const historyId = ++generatedOwnershipHistoryId + const adapter = createRecordingAdapter() + const firstPersistenceEntered = createEventGate() + const releaseFirstPersistence = createEventGate() + const applyCommittedTx = adapter.applyCommittedTx.bind(adapter) + let applyCalls = 0 + adapter.applyCommittedTx = async (...args) => { + applyCalls++ + if (applyCalls === 1) { + firstPersistenceEntered.resolve() + await releaseFirstPersistence.promise + } + await applyCommittedTx(...args) + } + let sourceParams!: TodoSyncParams + const collection = createCollection( + persistedCollectionOptions({ + id: `generated-owner-isolation-${historyId}`, + getKey: (row) => row.id, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() + }, + }, + persistence: { adapter }, + }), + ) + const receipts: Array> = [] + let unexpectedReceipt: Promise | undefined + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `generated owner isolation ready`, + ) + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `first`, title: `must survive` }, + }) + const firstReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + receipts.push(firstReceipt) + void firstReceipt.catch(() => undefined) + await atPersistedOracleCheckpoint( + firstPersistenceEntered.promise, + `generated owner isolation adapter hold`, + ) + + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `queued`, title: queuedTitle }, + }) + const queuedReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + receipts.push(queuedReceipt) + void queuedReceipt.catch(() => undefined) + + let observedError: unknown + try { + if (operation === `write`) { + sourceParams.write({ + type: `insert`, + value: { id: `stray`, title: `must not publish` }, + }) + } else if (operation === `truncate`) { + sourceParams.truncate() + } else { + const result = sourceParams.commit() + if (result !== true) { + unexpectedReceipt = Promise.resolve(result).then(() => undefined) + void unexpectedReceipt.catch(() => undefined) + } + } + } catch (error) { + observedError = error + } + expectOwnerIsolationAdmission({ + exactError: + operation === `commit` + ? observedError instanceof NoPendingSyncTransactionCommitError + : observedError instanceof NoPendingSyncTransactionWriteError, + strayVisible: collection.has(`stray`), + }) + + releaseFirstPersistence.resolve() + await atPersistedOracleCheckpoint(firstReceipt, `generated first owner`) + await atPersistedOracleCheckpoint(queuedReceipt, `generated queued owner`) + expect({ + publicRows: sortedTodoRows( + Array.from(collection.values()).map(stripVirtualProps), + ), + durableRows: sortedTodoRows(adapter.rows.values()), + pendingMarkers: collection._state.pendingSyncedTransactions.length, + }).toEqual({ + publicRows: sortedTodoRows([ + { id: `first`, title: `must survive` }, + { id: `queued`, title: queuedTitle }, + ]), + durableRows: sortedTodoRows([ + { id: `first`, title: `must survive` }, + { id: `queued`, title: queuedTitle }, + ]), + pendingMarkers: 0, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseFirstPersistence.resolve() + await cleanupPersistedOracle( + [ + ...receipts.map((receipt) => () => receipt.catch(() => undefined)), + () => unexpectedReceipt?.catch(() => undefined), + () => collection.cleanup(), + ], + hasPrimaryFailure, + ) + } +} + +async function runInternalReservationBoundaryLaw( + boundary: `abort` | `cleanup` | `terminal-failure`, + hydratedTitle: string, +): Promise { + const historyId = ++generatedOwnershipHistoryId + const hydrated = { id: `hydrated`, title: hydratedTitle } + const adapter = createRecordingAdapter([hydrated]) + const hydrationEntered = createEventGate() + const releaseHydration = createEventGate() + adapter.loadSubset = async () => { + hydrationEntered.resolve() + await releaseHydration.promise + return [{ key: hydrated.id, value: hydrated }] + } + let sourceParams!: TodoSyncParams + const collection = createCollection( + persistedCollectionOptions({ + id: `generated-internal-reservation-${historyId}`, + getKey: (row) => row.id, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() + }, + }, + persistence: { adapter }, + }), + ) + let internalReservationStarted = false + const subscription = collection.subscribeChanges((changes) => { + if ( + !internalReservationStarted && + changes.some((change) => change.key === hydrated.id) + ) { + internalReservationStarted = true + sourceParams.begin() + } + }) + const receipts: Array> = [] + let cleanedUp = false + let hasPrimaryFailure = false + + try { + const ready = collection.stateWhenReady() + await atPersistedOracleCheckpoint( + hydrationEntered.promise, + `generated internal hydration entered`, + ) + releaseHydration.resolve() + await atPersistedOracleCheckpoint(ready, `generated internal ready`) + expect({ + internalReservationStarted, + pendingMarkers: collection._state.pendingSyncedTransactions.length, + }).toEqual({ internalReservationStarted: true, pendingMarkers: 1 }) + + if (boundary === `cleanup`) { + await atPersistedOracleCheckpoint( + collection.cleanup(), + `generated internal cleanup`, + ) + cleanedUp = true + expectReservationBoundaryObservation( + { + status: collection.status, + pendingMarkers: collection._state.pendingSyncedTransactions.length, + }, + boundary, + ) + return + } + + if (boundary === `abort`) { + const abortController = new AbortController() + abortController.abort() + const receipt = Promise.resolve( + sourceParams.commit(abortController.signal), + ).then(() => undefined) + receipts.push(receipt) + void receipt.catch(() => undefined) + await expect( + atPersistedOracleCheckpoint(receipt, `generated internal abort`), + ).rejects.toMatchObject({ name: `AbortError` }) + expectReservationBoundaryObservation( + { + status: collection.status, + pendingMarkers: collection._state.pendingSyncedTransactions.length, + }, + boundary, + ) + return + } + + const terminalError = new Error(`generated terminal reservation failure`) + adapter.applyCommittedTx = () => Promise.reject(terminalError) + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `terminal`, title: `published before failure` }, + }) + const externalReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + receipts.push(externalReceipt) + void externalReceipt.catch(() => undefined) + await expect( + atPersistedOracleCheckpoint( + externalReceipt, + `generated external terminal failure`, + ), + ).rejects.toMatchObject({ + name: `PersistenceDurabilityError`, + cause: terminalError, + }) + + const internalReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + receipts.push(internalReceipt) + void internalReceipt.catch(() => undefined) + await expect( + atPersistedOracleCheckpoint( + internalReceipt, + `generated terminal internal settlement`, + ), + ).rejects.toMatchObject({ name: `PersistenceDurabilityError` }) + expectReservationBoundaryObservation( + { + status: collection.status, + pendingMarkers: collection._state.pendingSyncedTransactions.length, + }, + boundary, + ) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseHydration.resolve() + subscription.unsubscribe() + await cleanupPersistedOracle( + [ + ...receipts.map((receipt) => () => receipt.catch(() => undefined)), + () => (cleanedUp ? undefined : collection.cleanup()), + ], + hasPrimaryFailure, + ) + } +} + +type GeneratedPersistenceHistory = { + axes: Array + titles: Array +} + +type GeneratedPersistenceGrammar = { + property: string + axes: ReadonlyArray + axisContribution: Readonly> + titleCount: number + witness: GeneratedPersistenceHistory +} + +const immediateOrderingAxes = [`same-key`, `disjoint`] as const +const abortGraphAxes = [`independent`, `same-key`] as const +const ownerIsolationAxes = [`write`, `commit`, `truncate`] as const +const reservationBoundaryAxes = [ + `abort`, + `cleanup`, + `terminal-failure`, +] as const +const generatedTitleMaxLength = 12 +const generatedPersistenceFixedSeeds = { + immediateOrdering: 18_530_101, + abortGraph: 18_530_102, + ownerIsolation: 18_530_103, + reservationBoundary: 18_530_104, +} as const + +const generatedPersistenceGrammars: ReadonlyArray = + [ + { + property: `sqlite-persistence.immediate-order`, + axes: immediateOrderingAxes, + axisContribution: { + [`same-key`]: `the later source value wins in source order`, + disjoint: `both unrelated values and receipts survive`, + }, + titleCount: 2, + witness: { + axes: [`same-key`, `disjoint`], + titles: [`older`, `newer`], + }, + }, + { + property: `sqlite-persistence.abort-graph`, + axes: abortGraphAxes, + axisContribution: { + independent: `an unrelated sibling survives the abort`, + [`same-key`]: `a dependent suffix shares the predecessor failure`, + }, + titleCount: 2, + witness: { + axes: [`independent`, `same-key`], + titles: [`predecessor`, `successor`], + }, + }, + { + property: `sqlite-persistence.owner-isolation`, + axes: ownerIsolationAxes, + axisContribution: { + write: `bare write rejects without publishing a stray row`, + commit: `bare commit rejects without settling another owner`, + truncate: `bare truncate rejects without clearing another owner`, + }, + titleCount: 1, + witness: { + axes: [`write`, `commit`, `truncate`], + titles: [`queued`], + }, + }, + { + property: `sqlite-persistence.reservation-boundary`, + axes: reservationBoundaryAxes, + axisContribution: { + abort: `explicit abort releases its reservation`, + cleanup: `cleanup releases its reservation`, + [`terminal-failure`]: `fail-stop releases its reservation`, + }, + titleCount: 1, + witness: { + axes: [`abort`, `cleanup`, `terminal-failure`], + titles: [`hydrated`], + }, + }, + ] + +function reconstructGeneratedPersistenceHistory( + grammar: GeneratedPersistenceGrammar, + history: GeneratedPersistenceHistory, +): GeneratedPersistenceHistory { + const requiredAxes = new Set(grammar.axes) + const axesAreExact = + history.axes.length === grammar.axes.length && + new Set(history.axes).size === grammar.axes.length && + history.axes.every((axis) => requiredAxes.has(axis)) + const titlesAreInRange = + history.titles.length === grammar.titleCount && + history.titles.every( + (title) => [...title].length <= generatedTitleMaxLength, + ) + if (!axesAreExact || !titlesAreInRange) { + throw new Error( + `invalid generated persistence history: ${grammar.property}`, + ) + } + return { + axes: [...history.axes], + titles: [...history.titles], + } +} + +function fullAxisOrder( + axes: ReadonlyArray, +): fc.Arbitrary> { + return fc.shuffledSubarray([...axes], { + minLength: axes.length, + maxLength: axes.length, + }) +} + +const generatedTitle = fc.string({ maxLength: generatedTitleMaxLength }) + +type ImmediateOrderingHistory = { + relations: Array<`same-key` | `disjoint`> + titles: [string, string] +} + +type AbortGraphHistory = { + relationships: Array<`independent` | `same-key`> + titles: [string, string] +} + +type OwnerIsolationHistory = { + operations: Array<`write` | `commit` | `truncate`> + queuedTitle: string +} + +type ReservationBoundaryHistory = { + boundaries: Array<`abort` | `cleanup` | `terminal-failure`> + hydratedTitle: string +} + +const immediateOrderingHistory: fc.Arbitrary = + fc.record({ + relations: fullAxisOrder(immediateOrderingAxes), + titles: fc.tuple(generatedTitle, generatedTitle), + }) + +const abortGraphHistory: fc.Arbitrary = fc.record({ + relationships: fullAxisOrder(abortGraphAxes), + titles: fc.tuple(generatedTitle, generatedTitle), +}) + +const ownerIsolationHistory: fc.Arbitrary = fc.record({ + operations: fullAxisOrder(ownerIsolationAxes), + queuedTitle: generatedTitle, +}) + +const reservationBoundaryHistory: fc.Arbitrary = + fc.record({ + boundaries: fullAxisOrder(reservationBoundaryAxes), + hydratedTitle: generatedTitle, + }) + +function generatedPersistenceGrammarSamples(): ReadonlyArray<{ + grammar: GeneratedPersistenceGrammar + histories: Array +}> { + const grammar = (property: string): GeneratedPersistenceGrammar => { + const match = generatedPersistenceGrammars.find( + (candidate) => candidate.property === property, + ) + if (match === undefined) throw new Error(`missing grammar: ${property}`) + return match + } + const sampleOptions = (seed: number) => ({ seed, numRuns: 4 }) + return [ + { + grammar: grammar(`sqlite-persistence.immediate-order`), + histories: fc + .sample( + immediateOrderingHistory, + sampleOptions(generatedPersistenceFixedSeeds.immediateOrdering), + ) + .map((history) => ({ + axes: history.relations, + titles: [...history.titles], + })), + }, + { + grammar: grammar(`sqlite-persistence.abort-graph`), + histories: fc + .sample( + abortGraphHistory, + sampleOptions(generatedPersistenceFixedSeeds.abortGraph), + ) + .map((history) => ({ + axes: history.relationships, + titles: [...history.titles], + })), + }, + { + grammar: grammar(`sqlite-persistence.owner-isolation`), + histories: fc + .sample( + ownerIsolationHistory, + sampleOptions(generatedPersistenceFixedSeeds.ownerIsolation), + ) + .map((history) => ({ + axes: history.operations, + titles: [history.queuedTitle], + })), + }, + { + grammar: grammar(`sqlite-persistence.reservation-boundary`), + histories: fc + .sample( + reservationBoundaryHistory, + sampleOptions(generatedPersistenceFixedSeeds.reservationBoundary), + ) + .map((history) => ({ + axes: history.boundaries, + titles: [history.hydratedTitle], + })), + }, + ] +} + +async function runGeneratedImmediateOrderingHistory( + history: ImmediateOrderingHistory, +): Promise { + for (const relation of history.relations) { + await runImmediateOrderingLaw( + relation, + history.titles[0], + history.titles[1], + ) + } +} + +async function runGeneratedAbortGraphHistory( + history: AbortGraphHistory, +): Promise { + for (const relationship of history.relationships) { + await runAbortRelationshipLaw( + relationship, + history.titles[0], + history.titles[1], + ) + } +} + +async function runGeneratedOwnerIsolationHistory( + history: OwnerIsolationHistory, +): Promise { + for (const operation of history.operations) { + await runBareOwnerOperationLaw(operation, history.queuedTitle) + } +} + +async function runGeneratedReservationBoundaryHistory( + history: ReservationBoundaryHistory, +): Promise { + for (const boundary of history.boundaries) { + await runInternalReservationBoundaryLaw(boundary, history.hydratedTitle) + } +} + +function registerGeneratedPersistenceProperty(options: { + property: string + title: string + arbitrary: fc.Arbitrary + fixedSeed: number + run: (value: Value) => Promise +}): void { + const { property, title, arbitrary, fixedSeed, run } = options + if (requestedOracleReplayProperty === undefined) { + fcTest.prop([arbitrary], { seed: fixedSeed, numRuns: oracleRuns(4) })( + `${title} (fixed)`, + run, + ) + fcTest.prop([arbitrary], oraclePropertyOptions(4, property))( + `${title} (random)`, + run, + ) + } else if (requestedOracleReplayProperty === property) { + fcTest.prop([arbitrary], oraclePropertyOptions(4, property))( + `${title} (replay)`, + run, + ) + } +} + +describe(`generated persistence durability oracles`, () => { + if (requestedOracleReplayProperty === undefined) { + it(`reconstructs the full grammar and rejects ablated, out-of-range, and foreign histories`, () => { + for (const { + grammar, + histories, + } of generatedPersistenceGrammarSamples()) { + for (const history of histories) { + expect( + reconstructGeneratedPersistenceHistory(grammar, history), + ).toEqual(history) + } + } + + for (const grammar of generatedPersistenceGrammars) { + expect(Object.keys(grammar.axisContribution).sort()).toEqual( + [...grammar.axes].sort(), + ) + for (const contribution of Object.values(grammar.axisContribution)) { + expect(contribution).not.toBe(``) + } + expect( + reconstructGeneratedPersistenceHistory(grammar, grammar.witness), + ).toEqual(grammar.witness) + + for (const omittedAxis of grammar.axes) { + expect(() => + reconstructGeneratedPersistenceHistory(grammar, { + ...grammar.witness, + axes: grammar.witness.axes.filter((axis) => axis !== omittedAxis), + }), + ).toThrow(`invalid generated persistence history`) + } + + expect( + reconstructGeneratedPersistenceHistory(grammar, { + axes: [...grammar.axes], + titles: Array.from({ length: grammar.titleCount }, () => + `x`.repeat(generatedTitleMaxLength), + ), + }).titles, + ).toEqual( + Array.from({ length: grammar.titleCount }, () => + `x`.repeat(generatedTitleMaxLength), + ), + ) + expect( + reconstructGeneratedPersistenceHistory(grammar, { + axes: [...grammar.axes], + titles: Array.from({ length: grammar.titleCount }, () => ``), + }).titles, + ).toEqual(Array.from({ length: grammar.titleCount }, () => ``)) + expect(() => + reconstructGeneratedPersistenceHistory(grammar, { + axes: [...grammar.axes], + titles: [ + `x`.repeat(generatedTitleMaxLength + 1), + ...Array.from( + { length: Math.max(0, grammar.titleCount - 1) }, + () => ``, + ), + ], + }), + ).toThrow(`invalid generated persistence history`) + expect(() => + reconstructGeneratedPersistenceHistory(grammar, { + ...grammar.witness, + axes: grammar.axes.map((axis, index) => + index === grammar.axes.length - 1 ? grammar.axes[0]! : axis, + ), + }), + ).toThrow(`invalid generated persistence history`) + expect(() => + reconstructGeneratedPersistenceHistory(grammar, { + ...grammar.witness, + axes: [...grammar.axes.slice(0, -1), `foreign-axis`], + }), + ).toThrow(`invalid generated persistence history`) + } + }) + + it.each([ + { + name: `immediate-order older same-key value wins`, + reject: () => + expectImmediateOrderingObservation( + { + publicRows: [{ id: `shared`, title: `older` }], + durableRows: [{ id: `shared`, title: `older` }], + status: `ready`, + pendingMarkers: 0, + }, + [{ id: `shared`, title: `newer` }], + ), + }, + { + name: `abort-graph independent sibling inherits the abort`, + reject: () => + expectAbortGraphObservation( + { + predecessorStatus: `rejected`, + predecessorName: `AbortError`, + successorStatus: `rejected`, + sameFailure: true, + status: `error`, + publicSuccessor: undefined, + durableSuccessor: undefined, + pendingMarkers: 0, + }, + `independent`, + `survivor`, + `must survive`, + ), + }, + { + name: `owner-isolation bare write steals and publishes a queued marker`, + reject: () => + expectOwnerIsolationAdmission({ + exactError: false, + strayVisible: true, + }), + }, + { + name: `reservation-boundary terminal failure leaks one marker`, + reject: () => + expectReservationBoundaryObservation( + { status: `error`, pendingMarkers: 1 }, + `terminal-failure`, + ), + }, + ])(`rejects named wrong answer: $name`, ({ reject }) => { + expect(reject).toThrow() + }) + } + + registerGeneratedPersistenceProperty({ + property: `sqlite-persistence.immediate-order`, + title: `immediate histories preserve source order and settle every receipt`, + arbitrary: immediateOrderingHistory, + fixedSeed: generatedPersistenceFixedSeeds.immediateOrdering, + run: runGeneratedImmediateOrderingHistory, + }) + registerGeneratedPersistenceProperty({ + property: `sqlite-persistence.abort-graph`, + title: `abort graphs preserve independent siblings and reject dependent suffixes`, + arbitrary: abortGraphHistory, + fixedSeed: generatedPersistenceFixedSeeds.abortGraph, + run: runGeneratedAbortGraphHistory, + }) + registerGeneratedPersistenceProperty({ + property: `sqlite-persistence.owner-isolation`, + title: `bare operations cannot capture another owner's reservation`, + arbitrary: ownerIsolationHistory, + fixedSeed: generatedPersistenceFixedSeeds.ownerIsolation, + run: runGeneratedOwnerIsolationHistory, + }) + registerGeneratedPersistenceProperty({ + property: `sqlite-persistence.reservation-boundary`, + title: `reservation boundaries release every internally owned marker`, + arbitrary: reservationBoundaryHistory, + fixedSeed: generatedPersistenceFixedSeeds.reservationBoundary, + run: runGeneratedReservationBoundaryHistory, + }) +}) + +async function createTerminalFailureHarness( + kind: `hydration` | `durability`, + id: string, +) { + const seed = { id: `seed`, title: `stable before terminal failure` } + const adapter = createRecordingAdapter([seed]) + adapter.rowMetadata.set(seed.id, `row-metadata-before-failure`) + adapter.collectionMetadata.set(`resume`, `collection-metadata-before-failure`) + const coordinator = createFailStopCoordinatorHarness(id) + const upstreamLoads: Array = [] + const upstreamUnloads: Array = [] + let sourceParams!: TodoSyncParams + const collection = createCollection( + persistedCollectionOptions({ + id, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() + return { + loadSubset: (options) => { + upstreamLoads.push(options) + return true + }, + unloadSubset: (options) => { + upstreamUnloads.push(options) + }, + } + }, + }, + persistence: { adapter, coordinator }, + }), + ) + const initialLease: LoadSubsetOptions = { limit: 1 } + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `${id} initially ready`, + ) + await atPersistedOracleCheckpoint( + Promise.resolve(collection._sync.loadSubset(initialLease)), + `${id} initial lease hydrated`, + ) + await flushAsyncWork() + + let terminalError: unknown + if (kind === `hydration`) { + const hydrationError = new Error(`${id} terminal hydration failure`) + adapter.loadSubset = () => Promise.reject(hydrationError) + const outcome = await atPersistedOracleCheckpoint( + Promise.resolve(collection._sync.loadSubset({ limit: 2 })).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `${id} hydration failure settled`, + ) + expect(outcome).toEqual({ status: `rejected`, reason: hydrationError }) + terminalError = hydrationError + } else { + const adapterError = Object.assign(new Error(`${id} storage failure`), { + code: `SQLITE_FULL`, + path: `adapter.applyCommittedTx`, + }) + adapter.applyCommittedTx = () => Promise.reject(adapterError) + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `first`, title: `published before terminal failure` }, + }) + const outcome = await atPersistedOracleCheckpoint( + Promise.resolve(sourceParams.commit()).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `${id} durability failure settled`, + ) + expect(outcome.status).toBe(`rejected`) + terminalError = outcome.status === `rejected` ? outcome.reason : undefined + expect(terminalError).toBeInstanceOf(PersistenceDurabilityError) expect( - collection._state.syncedCollectionMetadata.get(`queryCollection:gc:q1`), - ).toEqual({ - queryHash: `q1`, + terminalError instanceof Error ? terminalError.cause : undefined, + ).toBe(adapterError) + } + + expect(collection.status).toBe(`error`) + expect(collection._lifecycle.getSyncError()).toBe(terminalError) + + return { + adapter, + collection, + coordinator, + initialLease, + seed, + sourceParams, + terminalError, + upstreamLoads, + upstreamUnloads, + } +} + +describeUnlessOracleReplay(`persistedCollectionOptions`, () => { + it(`provides a sync-absent loopback configuration with persisted utils`, async () => { + const adapter = createRecordingAdapter() + const collection = createCollection( + persistedCollectionOptions({ + id: `persisted-loopback`, + getKey: (item) => item.id, + persistence: { + adapter, + }, + }), + ) + + const insertTx = collection.insert({ + id: `1`, + title: `Phase 0`, + }) + + await insertTx.isPersisted.promise + + expect(stripVirtualProps(collection.get(`1`))).toEqual({ + id: `1`, + title: `Phase 0`, + }) + expect(adapter.applyCommittedTxCalls).toHaveLength(1) + expect(adapter.applyCommittedTxCalls[0]?.tx.mutations[0]?.type).toBe( + `insert`, + ) + expect(typeof collection.utils.acceptMutations).toBe(`function`) + expect(collection.utils.getLeadershipState?.().isLeader).toBe(true) + }) + + it(`supports acceptMutations for manual transactions`, async () => { + const adapter = createRecordingAdapter() + const collection = createCollection( + persistedCollectionOptions({ + id: `persisted-manual`, + getKey: (item) => item.id, + persistence: { + adapter, + }, + }), + ) + + const tx = createTransaction({ + autoCommit: false, + mutationFn: async ({ transaction }) => { + await collection.utils.acceptMutations(transaction) + }, + }) + + tx.mutate(() => { + collection.insert({ + id: `manual-1`, + title: `Manual`, + }) + }) + + await tx.commit() + + expect(stripVirtualProps(collection.get(`manual-1`))).toEqual({ + id: `manual-1`, + title: `Manual`, + }) + expect(adapter.applyCommittedTxCalls).toHaveLength(1) + }) + + it(`loads collection metadata into collection state during startup`, async () => { + const adapter = createRecordingAdapter() + adapter.collectionMetadata.set(`electric:resume`, { + kind: `resume`, + offset: `10_0`, + handle: `handle-1`, + shapeId: `shape-1`, + updatedAt: 1, + }) + + const collection = createCollection( + persistedCollectionOptions({ + id: `persisted-startup-metadata`, + getKey: (item) => item.id, + persistence: { + adapter, + }, + }), + ) + + await collection.stateWhenReady() + + expect(adapter.loadCollectionMetadataCalls).toEqual([ + `persisted-startup-metadata`, + ]) + expect( + collection._state.syncedCollectionMetadata.get(`electric:resume`), + ).toEqual({ + kind: `resume`, + offset: `10_0`, + handle: `handle-1`, + shapeId: `shape-1`, + updatedAt: 1, + }) + }) + + it(`restores row and collection metadata after metadata-bearing full reload`, async () => { + const adapter = createRecordingAdapter([ + { + id: `1`, + title: `Tracked`, + }, + ]) + adapter.rowMetadata.set(`1`, { + source: `initial`, + }) + adapter.collectionMetadata.set(`electric:resume`, { + kind: `resume`, + offset: `10_0`, + handle: `handle-1`, + shapeId: `shape-1`, + updatedAt: 1, + }) + const coordinator = createCoordinatorHarness() + + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { + adapter, + coordinator, + }, + }), + ) + + await collection.preload() + await flushAsyncWork() + + expect(collection._state.syncedMetadata.get(`1`)).toEqual({ + source: `initial`, + }) + expect( + collection._state.syncedCollectionMetadata.get(`electric:resume`), + ).toEqual({ + kind: `resume`, + offset: `10_0`, + handle: `handle-1`, + shapeId: `shape-1`, + updatedAt: 1, + }) + + adapter.rowMetadata.set(`1`, { + source: `reloaded`, + }) + adapter.collectionMetadata.delete(`electric:resume`) + adapter.collectionMetadata.set(`queryCollection:gc:q1`, { + queryHash: `q1`, + mode: `until-revalidated`, + }) + + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 1, + txId: `tx-reload`, + latestRowVersion: 1, + requiresFullReload: true, + }) + + await flushAsyncWork() + await flushAsyncWork() + + expect(collection._state.syncedMetadata.get(`1`)).toEqual({ + source: `reloaded`, + }) + expect( + collection._state.syncedCollectionMetadata.has(`electric:resume`), + ).toBe(false) + expect( + collection._state.syncedCollectionMetadata.get(`queryCollection:gc:q1`), + ).toEqual({ + queryHash: `q1`, mode: `until-revalidated`, }) - }) + }) + + it(`persists metadata-only wrapped sync transactions`, async () => { + const adapter = createRecordingAdapter() + + const collection = createCollection( + persistedCollectionOptions({ + id: `persisted-metadata-only`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, commit, markReady, metadata }) => { + begin() + metadata?.collection.set(`runtime:key`, { persisted: true }) + commit() + markReady() + }, + }, + persistence: { + adapter, + }, + }), + ) + + await collection.stateWhenReady() + await flushAsyncWork() + + expect(adapter.applyCommittedTxCalls).toHaveLength(1) + expect(adapter.applyCommittedTxCalls[0]?.tx.mutations).toEqual([]) + expect(adapter.collectionMetadata.get(`runtime:key`)).toEqual({ + persisted: true, + }) + expect( + collection._state.syncedCollectionMetadata.get(`runtime:key`), + ).toEqual({ + persisted: true, + }) + }) + + it(`replays metadata-only tx:committed deltas without full reload`, async () => { + const adapter = createRecordingAdapter([ + { + id: `1`, + title: `Tracked`, + }, + ]) + adapter.rowMetadata.set(`1`, { source: `initial` }) + const coordinator = createCoordinatorHarness() + + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { + adapter, + coordinator, + }, + }), + ) + + await collection.preload() + await flushAsyncWork() + const loadSubsetCallsAfterPreload = adapter.loadSubsetCalls.length + + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 1, + txId: `tx-metadata-only`, + latestRowVersion: 2, + requiresFullReload: false, + changedRows: [], + deletedKeys: [], + rowMetadataMutations: [ + { + type: `set`, + key: `1`, + value: { source: `replayed` }, + }, + ], + collectionMetadataMutations: [ + { + type: `set`, + key: `electric:resume`, + value: { + kind: `reset`, + updatedAt: 2, + }, + }, + ], + }) + + await flushAsyncWork() + await flushAsyncWork() + + expect(adapter.loadSubsetCalls.length).toBe(loadSubsetCallsAfterPreload) + expect(collection._state.syncedMetadata.get(`1`)).toEqual({ + source: `replayed`, + }) + expect( + collection._state.syncedCollectionMetadata.get(`electric:resume`), + ).toEqual({ + kind: `reset`, + updatedAt: 2, + }) + }) + + it(`uses pullSince replay deltas for metadata-bearing seq-gap recovery`, async () => { + const adapter = createRecordingAdapter([ + { + id: `1`, + title: `Tracked`, + }, + ]) + adapter.rowMetadata.set(`1`, { source: `initial` }) + const coordinator = createCoordinatorHarness() + coordinator.setPullSinceResponse({ + type: `rpc:pullSince:res`, + rpcId: `pull-metadata`, + ok: true, + latestTerm: 1, + latestSeq: 3, + latestRowVersion: 3, + requiresFullReload: false, + changedKeys: [], + deletedKeys: [], + deltas: [ + { + txId: `tx-gap-1`, + latestRowVersion: 2, + changedRows: [], + deletedKeys: [], + rowMetadataMutations: [ + { + type: `set`, + key: `1`, + value: { source: `gap-replayed` }, + }, + ], + collectionMetadataMutations: [], + }, + { + txId: `tx-gap-2`, + latestRowVersion: 3, + changedRows: [], + deletedKeys: [], + rowMetadataMutations: [], + collectionMetadataMutations: [ + { + type: `set`, + key: `queryCollection:gc:q1`, + value: { + queryHash: `q1`, + mode: `until-revalidated`, + }, + }, + ], + }, + ], + }) + + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { + adapter, + coordinator, + }, + }), + ) + + await collection.preload() + await flushAsyncWork() + const loadSubsetCallsAfterPreload = adapter.loadSubsetCalls.length + + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 3, + txId: `tx-gap-trigger`, + latestRowVersion: 3, + requiresFullReload: false, + changedRows: [], + deletedKeys: [], + }) + + await flushAsyncWork() + await flushAsyncWork() + + expect(coordinator.pullSinceCalls).toBe(1) + expect(adapter.loadSubsetCalls.length).toBe(loadSubsetCallsAfterPreload) + expect(collection._state.syncedMetadata.get(`1`)).toEqual({ + source: `gap-replayed`, + }) + expect( + collection._state.syncedCollectionMetadata.get(`queryCollection:gc:q1`), + ).toEqual({ + queryHash: `q1`, + mode: `until-revalidated`, + }) + }) + + it(`throws InvalidSyncConfigError when sync key is present but null`, () => { + const invalidOptions = { + id: `invalid-sync-null`, + getKey: (item: Todo) => item.id, + sync: null, + persistence: { + adapter: createNoopAdapter(), + }, + } as unknown as PersistedSyncWrappedOptions + + expect(() => persistedCollectionOptions(invalidOptions)).toThrow( + InvalidSyncConfigError, + ) + }) + + it(`throws InvalidSyncConfigError when sync key is present but invalid`, () => { + const invalidOptions = { + id: `invalid-sync-shape`, + getKey: (item: Todo) => item.id, + sync: {} as unknown as SyncConfig, + persistence: { + adapter: createNoopAdapter(), + }, + } as PersistedSyncWrappedOptions + + expect(() => persistedCollectionOptions(invalidOptions)).toThrow( + InvalidSyncConfigError, + ) + }) + + it(`uses SingleProcessCoordinator when coordinator is omitted`, () => { + const options = persistedCollectionOptions({ + id: `default-coordinator`, + getKey: (item) => item.id, + persistence: { + adapter: createNoopAdapter(), + }, + }) + + expect(options.persistence.coordinator).toBeInstanceOf( + SingleProcessCoordinator, + ) + }) + + it(`resolves persistence per collection and forwards schemaVersion`, () => { + const baseAdapter = createNoopAdapter() + const syncAdapter = createNoopAdapter() + const localAdapter = createNoopAdapter() + const resolverCalls: Array<{ + collectionId: string + mode: `sync-present` | `sync-absent` + schemaVersion?: number + }> = [] + + const persistence: PersistedCollectionPersistence = { + adapter: baseAdapter, + resolvePersistenceForCollection: (options) => { + resolverCalls.push(options) + return { + adapter: options.mode === `sync-present` ? syncAdapter : localAdapter, + } + }, + } + + const syncOptions = persistedCollectionOptions({ + id: `sync-collection`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + schemaVersion: 7, + persistence, + }) + expect(syncOptions.persistence.adapter).toBe(syncAdapter) + + const localOptions = persistedCollectionOptions({ + id: `local-collection`, + getKey: (item) => item.id, + schemaVersion: 3, + persistence, + }) + expect(localOptions.persistence.adapter).toBe(localAdapter) + + expect(resolverCalls).toEqual([ + { + collectionId: `sync-collection`, + mode: `sync-present`, + schemaVersion: 7, + }, + { + collectionId: `local-collection`, + mode: `sync-absent`, + schemaVersion: 3, + }, + ]) + }) + + it(`throws for invalid coordinator implementations`, () => { + const invalidCoordinator = { + getNodeId: () => `node-1`, + subscribe: () => () => {}, + publish: () => {}, + isLeader: () => true, + ensureLeadership: async () => {}, + // requestEnsurePersistedIndex is intentionally missing + } as unknown as PersistedCollectionCoordinator + + expect(() => + persistedCollectionOptions({ + id: `invalid-coordinator`, + getKey: (item) => item.id, + persistence: { + adapter: createNoopAdapter(), + coordinator: invalidCoordinator, + }, + }), + ).toThrow(InvalidPersistedCollectionCoordinatorError) + }) + + it(`preserves valid sync config in sync-present mode`, async () => { + const adapter = createRecordingAdapter() + const sync: SyncConfig = { + sync: ({ begin, write, commit, markReady }) => { + begin() + write({ + type: `insert`, + value: { + id: `remote-1`, + title: `From remote`, + }, + }) + commit() + markReady() + }, + } + + const options = persistedCollectionOptions({ + id: `sync-present`, + getKey: (item: Todo) => item.id, + sync, + persistence: { + adapter, + }, + }) + + const collection = createCollection(options) + await collection.stateWhenReady() + await flushAsyncWork() + + expect(stripVirtualProps(collection.get(`remote-1`))).toEqual({ + id: `remote-1`, + title: `From remote`, + }) + expect(adapter.applyCommittedTxCalls).toHaveLength(1) + expect(adapter.applyCommittedTxCalls[0]?.tx.mutations[0]?.type).toBe( + `update`, + ) + }) + + it(`does not apply or persist a wrapped sync transaction committed with an aborted signal`, async () => { + const adapter = createRecordingAdapter() + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: + | ((signal?: AbortSignal) => true | Promise) + | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present-aborted-commit`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit + markReady() + }, + }, + persistence: { adapter }, + }), + ) + try { + await collection.stateWhenReady() + const abortController = new AbortController() + abortController.abort() + remoteBegin?.() + remoteWrite?.({ + type: `insert`, + value: { id: `aborted`, title: `Must not publish` }, + }) + await expect( + remoteCommit?.(abortController.signal), + ).rejects.toMatchObject({ name: `AbortError` }) + await flushAsyncWork() + + expect(collection.get(`aborted`)).toBeUndefined() + expect(adapter.applyCommittedTxCalls).toHaveLength(0) + expect(collection._state.pendingSyncedTransactions).toHaveLength(0) + + remoteBegin?.() + remoteWrite?.({ + type: `insert`, + value: { id: `recovered`, title: `Later transaction` }, + }) + await remoteCommit?.() + + expect(stripVirtualProps(collection.get(`recovered`))).toEqual({ + id: `recovered`, + title: `Later transaction`, + }) + expect(adapter.rows.get(`recovered`)).toEqual({ + id: `recovered`, + title: `Later transaction`, + }) + expect(collection._state.pendingSyncedTransactions).toHaveLength(0) + } finally { + await collection.cleanup() + } + }) + + it(`persists a wrapped sync transaction when abort follows application`, async () => { + const adapter = createRecordingAdapter() + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: + | ((signal?: AbortSignal) => true | Promise) + | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present-abort-after-application`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit + markReady() + }, + }, + persistence: { adapter }, + }), + ) + let releaseMutation!: () => void + const mutationGate = new Promise((resolve) => { + releaseMutation = resolve + }) + const transaction = createTransaction({ + mutationFn: () => mutationGate, + }) + + try { + await collection.stateWhenReady() + transaction.mutate(() => { + collection.insert({ id: `local`, title: `Optimistic gate` }) + }) + + const abortController = new AbortController() + let appliedPublications = 0 + const subscription = collection.subscribeChanges((changes) => { + if (changes.some((change) => change.key === `remote`)) { + appliedPublications++ + abortController.abort() + } + }) + remoteBegin?.() + remoteWrite?.({ + type: `insert`, + value: { id: `remote`, title: `Already visible` }, + }) + const receipt = remoteCommit?.(abortController.signal) + expect(receipt).toBeInstanceOf(Promise) + + releaseMutation() + await transaction.isPersisted.promise + await receipt + subscription.unsubscribe() + + expect(appliedPublications).toBe(1) + expect(abortController.signal.aborted).toBe(true) + expect(stripVirtualProps(collection.get(`remote`))).toEqual({ + id: `remote`, + title: `Already visible`, + }) + expect(adapter.applyCommittedTxCalls).toHaveLength(1) + } finally { + releaseMutation() + await transaction.isPersisted.promise.catch(() => undefined) + await collection.cleanup() + } + }) + + it(`fail-stops a queued suffix when its staged predecessor aborts before publication`, async () => { + const adapter = createRecordingAdapter() + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: + | ((signal?: AbortSignal) => true | Promise) + | undefined + let sourceParams!: TodoSyncParams + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present-abort-before-publication-suffix`, + getKey: (item) => item.id, + sync: { + sync: (params) => { + sourceParams = params + remoteBegin = params.begin + remoteWrite = params.write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = params.commit + params.markReady() + }, + }, + persistence: { adapter }, + }), + ) + const localPersistence = createEventGate() + const localTransaction = createTransaction({ + mutationFn: () => localPersistence.promise, + }) + const aborted = new AbortController() + let abortedReceipt: Promise | undefined + let dependentReceipt: Promise | undefined + let hasPrimaryFailure = false + + try { + await collection.stateWhenReady() + localTransaction.mutate(() => { + collection.insert({ id: `local`, title: `publication gate` }) + }) + + remoteBegin?.() + sourceParams.metadata!.row.set(`shared`, { owner: `aborted` }) + abortedReceipt = Promise.resolve(remoteCommit?.(aborted.signal)).then( + () => undefined, + ) + void abortedReceipt.catch(() => undefined) + + remoteBegin?.() + const stagedOwner = sourceParams.metadata!.row.get(`shared`) + remoteWrite?.({ + type: `insert`, + value: { + id: `dependent`, + title: + (stagedOwner as { owner?: string } | undefined)?.owner ?? `missing`, + }, + }) + dependentReceipt = Promise.resolve(remoteCommit?.()).then(() => undefined) + void dependentReceipt.catch(() => undefined) + + expect(stagedOwner).toEqual({ owner: `aborted` }) + aborted.abort() + const abortedOutcome = await atPersistedOracleCheckpoint( + abortedReceipt.then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `staged predecessor aborted before publication`, + ) + localPersistence.resolve() + await localTransaction.isPersisted.promise + const dependentOutcome = await atPersistedOracleCheckpoint( + dependentReceipt.then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `post-abort staged suffix settled`, + ) + const terminalError = + abortedOutcome.status === `rejected` ? abortedOutcome.reason : undefined + + expect({ + abortedStatus: abortedOutcome.status, + abortedName: + terminalError instanceof Error ? terminalError.name : undefined, + dependentStatus: dependentOutcome.status, + dependentExact: + dependentOutcome.status === `rejected` && + dependentOutcome.reason === terminalError, + status: collection.status, + exactPublicError: + collection._lifecycle.getSyncError() === terminalError, + visibleDependent: collection.get(`dependent`), + durableDependent: adapter.rows.get(`dependent`), + durableMetadata: adapter.rowMetadata.get(`shared`), + }).toEqual({ + abortedStatus: `rejected`, + abortedName: `AbortError`, + dependentStatus: `rejected`, + dependentExact: true, + status: `error`, + exactPublicError: true, + visibleDependent: undefined, + durableDependent: undefined, + durableMetadata: undefined, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + localPersistence.resolve() + await cleanupPersistedOracle( + [ + () => localTransaction.isPersisted.promise.catch(() => undefined), + () => abortedReceipt?.catch(() => undefined), + () => dependentReceipt?.catch(() => undefined), + () => collection.cleanup(), + ], + hasPrimaryFailure, + ) + } + }) + + it(`rejects a wrapped sync receipt with a named terminal persistence error`, async () => { + const adapter = createRecordingAdapter() + const adapterError = Object.assign( + new Error(`adapter write failed exactly`), + { + code: `SQLITE_FULL`, + path: `adapter.applyCommittedTx`, + }, + ) + const persistenceEntered = createEventGate() + const persistence = createEventGate() + adapter.applyCommittedTx = async () => { + persistenceEntered.resolve() + await persistence.promise + } + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: (() => true | Promise) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present-persistence-error`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit + markReady() + }, + }, + persistence: { adapter }, + }), + ) + let hasPrimaryFailure = false + + try { + await collection.stateWhenReady() + remoteBegin?.() + remoteWrite?.({ + type: `insert`, + value: { id: `failed`, title: `Not durable` }, + }) + + const receipt = Promise.resolve(remoteCommit?.()).then(() => undefined) + const settlement = observeSettlement(receipt) + await atPersistedOracleCheckpoint( + persistenceEntered.promise, + `terminal persistence entered`, + ) + await flushAsyncWork() + + expect({ + receipt: settlement.read(), + status: collection.status, + publicError: collection._lifecycle.getSyncError(), + visible: stripVirtualProps(collection.get(`failed`)), + durable: adapter.rows.get(`failed`), + }).toEqual({ + receipt: { status: `pending` }, + status: `ready`, + publicError: undefined, + visible: { id: `failed`, title: `Not durable` }, + durable: undefined, + }) + + persistence.reject(adapterError) + const outcome = await receipt.then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + + const publicError = collection._lifecycle.getSyncError() + const receiptError = + outcome.status === `rejected` && outcome.reason instanceof Error + ? (outcome.reason as Error & { + code?: unknown + path?: unknown + }) + : undefined + expect({ + receiptStatus: outcome.status, + receiptErrorName: receiptError?.name, + receiptHasNamedPersistenceSemantics: + outcome.status === `rejected` && + outcome.reason instanceof PersistenceDurabilityError, + receiptCauseIsAdapterError: receiptError?.cause === adapterError, + receiptErrorCode: receiptError?.code, + receiptErrorPath: receiptError?.path, + publicErrorIsReceipt: + outcome.status === `rejected` && publicError === outcome.reason, + status: collection.status, + visible: stripVirtualProps(collection.get(`failed`)), + durable: adapter.rows.get(`failed`), + }).toEqual({ + receiptStatus: `rejected`, + receiptErrorName: `PersistenceDurabilityError`, + receiptHasNamedPersistenceSemantics: true, + receiptCauseIsAdapterError: true, + receiptErrorCode: `SQLITE_FULL`, + receiptErrorPath: `adapter.applyCommittedTx`, + publicErrorIsReceipt: true, + status: `error`, + visible: { id: `failed`, title: `Not durable` }, + durable: undefined, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + persistence.reject(adapterError) + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }) + + it(`keeps durability failure terminal when publication aborts the signal`, async () => { + const adapter = createRecordingAdapter() + const storageError = Object.assign( + new Error(`post-publication durability failed exactly`), + { + code: `SQLITE_IOERR`, + path: `adapter.applyCommittedTx`, + }, + ) + const persistenceEntered = createEventGate() + const releasePersistence = createEventGate() + adapter.applyCommittedTx = async () => { + persistenceEntered.resolve() + await releasePersistence.promise + throw storageError + } + let sourceParams!: TodoSyncParams + const abortController = new AbortController() + const collection = createCollection( + persistedCollectionOptions({ + id: `post-publication-abort-durability`, + getKey: (row) => row.id, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() + }, + }, + persistence: { adapter }, + }), + ) + const publication = collection.subscribeChanges((changes) => { + if (changes.some((entry) => entry.key === `failed`)) { + abortController.abort() + } + }) + let receipt: Promise | undefined + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `post-publication abort collection ready`, + ) + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `failed`, title: `visible before durability` }, + }) + receipt = Promise.resolve( + sourceParams.commit(abortController.signal), + ).then(() => undefined) + void receipt.catch(() => undefined) + await atPersistedOracleCheckpoint( + persistenceEntered.promise, + `post-publication abort persistence entered`, + ) + + expect({ + signalAborted: abortController.signal.aborted, + visible: stripVirtualProps(collection.get(`failed`)), + status: collection.status, + }).toEqual({ + signalAborted: true, + visible: { id: `failed`, title: `visible before durability` }, + status: `ready`, + }) + + releasePersistence.resolve() + const outcome = await atPersistedOracleCheckpoint( + receipt.then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `post-publication abort durability receipt`, + ) + const terminalError = + outcome.status === `rejected` ? outcome.reason : undefined + + expect({ + receiptStatus: outcome.status, + namedDurability: terminalError instanceof PersistenceDurabilityError, + cause: terminalError instanceof Error ? terminalError.cause : undefined, + exactPublicError: + collection._lifecycle.getSyncError() === terminalError, + status: collection.status, + durable: adapter.rows.get(`failed`), + }).toEqual({ + receiptStatus: `rejected`, + namedDurability: true, + cause: storageError, + exactPublicError: true, + status: `error`, + durable: undefined, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releasePersistence.resolve() + publication.unsubscribe() + await cleanupPersistedOracle( + [() => receipt?.catch(() => undefined), () => collection.cleanup()], + hasPrimaryFailure, + ) + } + }) + + it(`reserves FIFO admission before a publication callback commits a sibling`, async () => { + const adapter = createRecordingAdapter() + const successfulApply = adapter.applyCommittedTx.bind(adapter) + const firstPersistenceEntered = createEventGate() + const releaseFirstPersistence = createEventGate() + const firstStorageError = new Error(`reentrant first durability failed`) + let applyCalls = 0 + adapter.applyCommittedTx = async (...args) => { + applyCalls++ + if (applyCalls === 1) { + firstPersistenceEntered.resolve() + await releaseFirstPersistence.promise + throw firstStorageError + } + await successfulApply(...args) + } + let sourceParams!: TodoSyncParams + const collection = createCollection( + persistedCollectionOptions({ + id: `reentrant-fifo-admission`, + getKey: (row) => row.id, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() + }, + }, + persistence: { adapter }, + }), + ) + let siblingReceipt: Promise | undefined + let publicationTriggers = 0 + const subscription = collection.subscribeChanges((changes) => { + if ( + publicationTriggers === 0 && + changes.some((entry) => entry.key === `first`) + ) { + publicationTriggers++ + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `sibling`, title: `must remain queued` }, + }) + siblingReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + void siblingReceipt.catch(() => undefined) + } + }) + let firstReceipt: Promise | undefined + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `reentrant FIFO collection ready`, + ) + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `first`, title: `publishes before durability` }, + }) + firstReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + void firstReceipt.catch(() => undefined) + await atPersistedOracleCheckpoint( + firstPersistenceEntered.promise, + `reentrant first persistence entered`, + ) + await Promise.resolve() + + expect(siblingReceipt).toBeInstanceOf(Promise) + const firstSettlement = observeSettlement(firstReceipt) + const siblingSettlement = observeSettlement(siblingReceipt!) + await Promise.resolve() + expect({ + publicationTriggers, + applyCalls, + firstReceipt: firstSettlement.read(), + siblingReceipt: siblingSettlement.read(), + visibleFirst: stripVirtualProps(collection.get(`first`)), + visibleSibling: collection.get(`sibling`), + durableFirst: adapter.rows.get(`first`), + durableSibling: adapter.rows.get(`sibling`), + }).toEqual({ + publicationTriggers: 1, + applyCalls: 1, + firstReceipt: { status: `pending` }, + siblingReceipt: { status: `pending` }, + visibleFirst: { id: `first`, title: `publishes before durability` }, + visibleSibling: undefined, + durableFirst: undefined, + durableSibling: undefined, + }) + + releaseFirstPersistence.resolve() + const firstOutcome = await atPersistedOracleCheckpoint( + firstReceipt.then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `reentrant first receipt rejected`, + ) + const siblingOutcome = await atPersistedOracleCheckpoint( + siblingReceipt!.then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `reentrant sibling receipt rejected`, + ) + const terminalError = + firstOutcome.status === `rejected` ? firstOutcome.reason : undefined + expect({ + firstStatus: firstOutcome.status, + siblingStatus: siblingOutcome.status, + sameTerminal: + siblingOutcome.status === `rejected` && + siblingOutcome.reason === terminalError, + status: collection.status, + publicErrorIsFirst: + collection._lifecycle.getSyncError() === terminalError, + visibleSibling: collection.get(`sibling`), + durableSibling: adapter.rows.get(`sibling`), + applyCalls, + }).toEqual({ + firstStatus: `rejected`, + siblingStatus: `rejected`, + sameTerminal: true, + status: `error`, + publicErrorIsFirst: true, + visibleSibling: undefined, + durableSibling: undefined, + applyCalls: 1, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseFirstPersistence.resolve() + subscription.unsubscribe() + await cleanupPersistedOracle( + [ + () => firstReceipt?.catch(() => undefined), + () => siblingReceipt?.catch(() => undefined), + () => collection.cleanup(), + ], + hasPrimaryFailure, + ) + } + }) + + it(`rejects a queued suffix derived from an aborted staged metadata layer`, async () => { + const adapter = createRecordingAdapter() + const successfulApply = adapter.applyCommittedTx.bind(adapter) + const firstPersistenceEntered = createEventGate() + const releaseFirstPersistence = createEventGate() + let applyCalls = 0 + adapter.applyCommittedTx = async (...args) => { + applyCalls++ + if (applyCalls === 1) { + firstPersistenceEntered.resolve() + await releaseFirstPersistence.promise + } + await successfulApply(...args) + } + let sourceParams!: TodoSyncParams + const collection = createCollection( + persistedCollectionOptions({ + id: `aborted-staged-metadata-suffix`, + getKey: (row) => row.id, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() + }, + }, + persistence: { adapter }, + }), + ) + const aborted = new AbortController() + let firstReceipt: Promise | undefined + let abortedReceipt: Promise | undefined + let dependentReceipt: Promise | undefined + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `aborted staged metadata collection ready`, + ) + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `first`, title: `durability gate` }, + }) + firstReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + void firstReceipt.catch(() => undefined) + await atPersistedOracleCheckpoint( + firstPersistenceEntered.promise, + `first persistence entered before staged suffix`, + ) + + sourceParams.begin() + sourceParams.metadata!.row.set(`shared`, { owner: `aborted` }) + abortedReceipt = Promise.resolve( + sourceParams.commit(aborted.signal), + ).then(() => undefined) + void abortedReceipt.catch(() => undefined) + + sourceParams.begin() + const stagedOwner = sourceParams.metadata!.row.get(`shared`) + sourceParams.write({ + type: `insert`, + value: { + id: `dependent`, + title: + (stagedOwner as { owner?: string } | undefined)?.owner ?? `missing`, + }, + }) + dependentReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + void dependentReceipt.catch(() => undefined) + + expect(stagedOwner).toEqual({ owner: `aborted` }) + aborted.abort() + releaseFirstPersistence.resolve() + await atPersistedOracleCheckpoint( + firstReceipt, + `first durability gate settled`, + ) + const abortedOutcome = await atPersistedOracleCheckpoint( + abortedReceipt.then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `aborted staged receipt settled`, + ) + const dependentOutcome = await atPersistedOracleCheckpoint( + dependentReceipt.then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `dependent staged receipt settled`, + ) + const terminalError = + abortedOutcome.status === `rejected` ? abortedOutcome.reason : undefined + + expect({ + abortedStatus: abortedOutcome.status, + abortedName: + terminalError instanceof Error ? terminalError.name : undefined, + dependentStatus: dependentOutcome.status, + dependentExact: + dependentOutcome.status === `rejected` && + dependentOutcome.reason === terminalError, + status: collection.status, + exactPublicError: + collection._lifecycle.getSyncError() === terminalError, + visibleDependent: collection.get(`dependent`), + durableDependent: adapter.rows.get(`dependent`), + durableMetadata: adapter.rowMetadata.get(`shared`), + applyCalls, + }).toEqual({ + abortedStatus: `rejected`, + abortedName: `AbortError`, + dependentStatus: `rejected`, + dependentExact: true, + status: `error`, + exactPublicError: true, + visibleDependent: undefined, + durableDependent: undefined, + durableMetadata: undefined, + applyCalls: 1, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseFirstPersistence.resolve() + await cleanupPersistedOracle( + [ + () => firstReceipt?.catch(() => undefined), + () => abortedReceipt?.catch(() => undefined), + () => dependentReceipt?.catch(() => undefined), + () => collection.cleanup(), + ], + hasPrimaryFailure, + ) + } + }) + + it(`does not fail-stop when an aborted staged collection metadata write was superseded`, async () => { + const adapter = createRecordingAdapter() + const successfulApply = adapter.applyCommittedTx.bind(adapter) + const firstPersistenceEntered = createEventGate() + const releaseFirstPersistence = createEventGate() + let applyCalls = 0 + adapter.applyCommittedTx = async (...args) => { + applyCalls++ + if (applyCalls === 1) { + firstPersistenceEntered.resolve() + await releaseFirstPersistence.promise + } + await successfulApply(...args) + } + let sourceParams!: TodoSyncParams + const collection = createCollection( + persistedCollectionOptions({ + id: `superseded-staged-collection-metadata`, + getKey: (row) => row.id, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() + }, + }, + persistence: { adapter }, + }), + ) + const olderAbort = new AbortController() + let firstReceipt: Promise | undefined + let olderReceipt: Promise | undefined + let newerReceipt: Promise | undefined + let dependentReceipt: Promise | undefined + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `superseded collection metadata owner ready`, + ) + + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `first`, title: `durability gate` }, + }) + firstReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + void firstReceipt.catch(() => undefined) + await atPersistedOracleCheckpoint( + firstPersistenceEntered.promise, + `first persistence entered before superseded metadata writes`, + ) + + sourceParams.begin() + sourceParams.metadata!.collection.set(`shared`, { owner: `older` }) + olderReceipt = Promise.resolve( + sourceParams.commit(olderAbort.signal), + ).then(() => undefined) + void olderReceipt.catch(() => undefined) + + sourceParams.begin() + sourceParams.metadata!.collection.set(`shared`, { owner: `newer` }) + newerReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + void newerReceipt.catch(() => undefined) + + sourceParams.begin() + const stagedOwner = sourceParams + .metadata!.collection.list() + .find(({ key }) => key === `shared`)?.value + sourceParams.write({ + type: `insert`, + value: { + id: `dependent`, + title: + (stagedOwner as { owner?: string } | undefined)?.owner ?? `missing`, + }, + }) + dependentReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + void dependentReceipt.catch(() => undefined) + + expect(stagedOwner).toEqual({ owner: `newer` }) + olderAbort.abort() + releaseFirstPersistence.resolve() + await atPersistedOracleCheckpoint( + firstReceipt, + `superseded metadata durability gate settled`, + ) + const olderOutcome = await atPersistedOracleCheckpoint( + olderReceipt.then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `superseded older metadata receipt settled`, + ) + const newerOutcome = await atPersistedOracleCheckpoint( + newerReceipt.then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `newer metadata receipt settled`, + ) + const dependentOutcome = await atPersistedOracleCheckpoint( + dependentReceipt.then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `newest-owner dependent receipt settled`, + ) + + expect({ + olderStatus: olderOutcome.status, + newerStatus: newerOutcome.status, + dependentStatus: dependentOutcome.status, + status: collection.status, + publicError: collection._lifecycle.getSyncError(), + visibleDependent: stripVirtualProps(collection.get(`dependent`)), + durableDependent: adapter.rows.get(`dependent`), + durableMetadata: adapter.collectionMetadata.get(`shared`), + applyCalls, + }).toEqual({ + olderStatus: `rejected`, + newerStatus: `fulfilled`, + dependentStatus: `fulfilled`, + status: `ready`, + publicError: undefined, + visibleDependent: { id: `dependent`, title: `newer` }, + durableDependent: { id: `dependent`, title: `newer` }, + durableMetadata: { owner: `newer` }, + applyCalls: 3, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseFirstPersistence.resolve() + await cleanupPersistedOracle( + [ + () => firstReceipt?.catch(() => undefined), + () => olderReceipt?.catch(() => undefined), + () => newerReceipt?.catch(() => undefined), + () => dependentReceipt?.catch(() => undefined), + () => collection.cleanup(), + ], + hasPrimaryFailure, + ) + } + }) + + it(`does not let an old external adapter success affect a restarted lifecycle`, async () => { + const id = `external-lifecycle-success` + const adapter = createRecordingAdapter() + const successfulApply = adapter.applyCommittedTx.bind(adapter) + const oldPersistenceEntered = createEventGate() + const releaseOldPersistence = createEventGate() + adapter.applyCommittedTx = async (...args) => { + oldPersistenceEntered.resolve() + await releaseOldPersistence.promise + await successfulApply(...args) + } + const coordinator = createFailStopCoordinatorHarness(id) + let sourceParams!: TodoSyncParams + const replacement = { + id: `replacement`, + title: `owned by restarted lifecycle`, + } + const collection = createCollection( + persistedCollectionOptions({ + id, + getKey: (row) => row.id, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + let oldSettlement: + | Promise< + { status: `fulfilled` } | { status: `rejected`; reason: unknown } + > + | undefined + let cleanedUp = false + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `${id} initial lifecycle ready`, + ) + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `old`, title: `must remain owned by old lifecycle` }, + }) + oldSettlement = Promise.resolve(sourceParams.commit()).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + await atPersistedOracleCheckpoint( + oldPersistenceEntered.promise, + `${id} old persistence entered`, + ) + + await collection.cleanup() + cleanedUp = true + adapter.loadSubset = () => + Promise.resolve([{ key: replacement.id, value: replacement }]) + collection.startSyncImmediate() + cleanedUp = false + const replacementSettlement = collection.stateWhenReady().then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + releaseOldPersistence.resolve() + + const oldOutcome = await atPersistedOracleCheckpoint( + oldSettlement, + `${id} old receipt settled`, + ) + const replacementOutcome = await atPersistedOracleCheckpoint( + replacementSettlement, + `${id} replacement lifecycle settled`, + ) + await flushAsyncWork() + + expect({ + oldReceiptStatus: oldOutcome.status, + replacementStatus: replacementOutcome.status, + replacementVisible: stripVirtualProps(collection.get(replacement.id)), + replacementPublicError: collection._lifecycle.getSyncError(), + staleBroadcasts: coordinator.publishCalls.length, + }).toEqual({ + oldReceiptStatus: `fulfilled`, + replacementStatus: `fulfilled`, + replacementVisible: replacement, + replacementPublicError: undefined, + staleBroadcasts: 0, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseOldPersistence.resolve() + await cleanupPersistedOracle( + [ + () => oldSettlement, + () => (cleanedUp ? undefined : collection.cleanup()), + ], + hasPrimaryFailure, + ) + } + }) + + it(`does not let an old external adapter failure poison a restarted lifecycle`, async () => { + const id = `external-lifecycle-failure` + const adapter = createRecordingAdapter() + const successfulApply = adapter.applyCommittedTx.bind(adapter) + const oldPersistenceEntered = createEventGate() + const releaseOldPersistence = createEventGate() + const oldAdapterError = new Error(`old lifecycle adapter failure`) + let applyCalls = 0 + adapter.applyCommittedTx = async (...args) => { + applyCalls++ + if (applyCalls === 1) { + oldPersistenceEntered.resolve() + await releaseOldPersistence.promise + throw oldAdapterError + } + await successfulApply(...args) + } + const coordinator = createFailStopCoordinatorHarness(id) + let sourceParams!: TodoSyncParams + const replacement = { + id: `replacement`, + title: `owned by restarted lifecycle`, + } + const collection = createCollection( + persistedCollectionOptions({ + id, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + const startAndWaitForReady = async (label: string) => { + const ready = createEventGate() + const unsubscribe = collection.on(`status:ready`, () => ready.resolve()) + collection.startSyncImmediate() + try { + await atPersistedOracleCheckpoint(ready.promise, label) + } finally { + unsubscribe() + } + } + let oldSettlement: + | Promise< + { status: `fulfilled` } | { status: `rejected`; reason: unknown } + > + | undefined + let replacementReceipt: + | Promise< + { status: `fulfilled` } | { status: `rejected`; reason: unknown } + > + | undefined + let hasPrimaryFailure = false + + try { + await startAndWaitForReady(`${id} initial lifecycle ready`) + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `old`, title: `must remain owned by old lifecycle` }, + }) + oldSettlement = Promise.resolve(sourceParams.commit()).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + await atPersistedOracleCheckpoint( + oldPersistenceEntered.promise, + `${id} old persistence entered`, + ) + + await collection.cleanup() + await startAndWaitForReady( + `${id} replacement lifecycle ready before old failure`, + ) + sourceParams.begin() + sourceParams.write({ type: `insert`, value: replacement }) + replacementReceipt = Promise.resolve(sourceParams.commit()).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + releaseOldPersistence.resolve() + + const oldOutcome = await atPersistedOracleCheckpoint( + oldSettlement, + `${id} old receipt settled`, + ) + const replacementOutcome = await atPersistedOracleCheckpoint( + replacementReceipt, + `${id} replacement transaction settled`, + ) + await flushAsyncWork() + + expect({ + oldReceiptStatus: oldOutcome.status, + replacementStatus: replacementOutcome.status, + replacementVisible: stripVirtualProps(collection.get(replacement.id)), + replacementPublicError: collection._lifecycle.getSyncError(), + broadcasts: coordinator.publishCalls.length, + applyCalls, + }).toEqual({ + oldReceiptStatus: `rejected`, + replacementStatus: `fulfilled`, + replacementVisible: replacement, + replacementPublicError: undefined, + broadcasts: 1, + applyCalls: 2, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseOldPersistence.resolve() + await cleanupPersistedOracle( + [ + () => oldSettlement, + () => replacementReceipt, + () => collection.cleanup(), + ], + hasPrimaryFailure, + ) + } + }) + + it(`does not deliver coordinator work accepted before lifecycle replacement into the new owner`, async () => { + const id = `queued-coordinator-lifecycle` + const adapter = createRecordingAdapter() + const oldReloadEntered = createEventGate() + const releaseOldReload = createEventGate() + const replacementHydrationEntered = createEventGate() + const releaseReplacementHydration = createEventGate() + let subsetCalls = 0 + adapter.loadSubset = async () => { + subsetCalls++ + if (subsetCalls === 2) { + oldReloadEntered.resolve() + await releaseOldReload.promise + return [] + } + if (subsetCalls === 3) { + replacementHydrationEntered.resolve() + await releaseReplacementHydration.promise + return [ + { + key: `replacement`, + value: { id: `replacement`, title: `new owner baseline` }, + }, + ] + } + return [] + } + const coordinator = createFailStopCoordinatorHarness(id) + const replacementControlsReady = createEventGate() + let sourceRuns = 0 + const collection = createCollection( + persistedCollectionOptions({ + id, + getKey: (row) => row.id, + sync: { + sync: ({ markReady }) => { + sourceRuns++ + if (sourceRuns === 2) replacementControlsReady.resolve() + markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + let replacementReady: + | Promise< + { status: `fulfilled` } | { status: `rejected`; reason: unknown } + > + | undefined + let cleanedUp = false + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `queued coordinator initial owner ready`, + ) + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 1, + txId: `old-reload-gate`, + latestRowVersion: 1, + requiresFullReload: true, + }) + await atPersistedOracleCheckpoint( + oldReloadEntered.promise, + `old coordinator reload entered`, + ) + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 2, + txId: `queued-before-restart`, + latestRowVersion: 2, + requiresFullReload: false, + changedRows: [ + { + key: `stale`, + value: { id: `stale`, title: `must stay with old owner` }, + }, + ], + deletedKeys: [], + }) + + await collection.cleanup() + cleanedUp = true + collection.startSyncImmediate() + cleanedUp = false + replacementReady = collection.stateWhenReady().then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + await atPersistedOracleCheckpoint( + replacementControlsReady.promise, + `replacement sync controls installed`, + ) + releaseOldReload.resolve() + await atPersistedOracleCheckpoint( + replacementHydrationEntered.promise, + `replacement hydration entered after queued coordinator work`, + ) + + expect({ + sourceRuns, + subsetCalls, + staleVisible: collection.get(`stale`), + staleDurable: adapter.rows.get(`stale`), + replacementPublicError: collection._lifecycle.getSyncError(), + }).toEqual({ + sourceRuns: 2, + subsetCalls: 3, + staleVisible: undefined, + staleDurable: undefined, + replacementPublicError: undefined, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseOldReload.resolve() + releaseReplacementHydration.resolve() + await cleanupPersistedOracle( + [ + () => replacementReady, + () => (cleanedUp ? undefined : collection.cleanup()), + ], + hasPrimaryFailure, + ) + } + }) + + it(`discards a queued collection reset when its owner is cleaned up`, async () => { + const id = `queued-reset-lifecycle` + const adapter = createRecordingAdapter() + const oldReloadEntered = createEventGate() + const releaseOldReload = createEventGate() + let subsetCalls = 0 + adapter.loadSubset = async () => { + subsetCalls++ + if (subsetCalls === 2) { + oldReloadEntered.resolve() + await releaseOldReload.promise + } + return [] + } + const coordinator = createFailStopCoordinatorHarness(id) + const collection = createCollection( + persistedCollectionOptions({ + id, + getKey: (row) => row.id, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + let cleanedUp = false + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `queued reset initial owner ready`, + ) + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 1, + txId: `old-reset-gate`, + latestRowVersion: 1, + requiresFullReload: true, + }) + await atPersistedOracleCheckpoint( + oldReloadEntered.promise, + `old reload entered before queued reset`, + ) + coordinator.emit({ + type: `collection:reset`, + schemaVersion: 1, + resetEpoch: 1, + }) + + await collection.cleanup() + cleanedUp = true + releaseOldReload.resolve() + await flushAsyncWork() + await flushAsyncWork() + + expect({ + subsetCalls, + status: collection.status, + publicError: collection._lifecycle.getSyncError(), + }).toEqual({ + subsetCalls: 2, + status: `cleaned-up`, + publicError: undefined, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseOldReload.resolve() + await cleanupPersistedOracle( + [() => (cleanedUp ? undefined : collection.cleanup())], + hasPrimaryFailure, + ) + } + }) + + it(`does not apply a delayed seq-gap replay response into a restarted lifecycle`, async () => { + const id = `delayed-pull-since-lifecycle` + const adapter = createRecordingAdapter() + const pullEntered = createEventGate() + const releasePull = createEventGate() + const replacementHydrationEntered = createEventGate() + const releaseReplacementHydration = createEventGate() + let subsetCalls = 0 + adapter.loadSubset = async () => { + subsetCalls++ + if (subsetCalls === 2) { + replacementHydrationEntered.resolve() + await releaseReplacementHydration.promise + return [ + { + key: `replacement`, + value: { id: `replacement`, title: `new owner baseline` }, + }, + ] + } + return [] + } + const coordinator = createFailStopCoordinatorHarness(id) + coordinator.pullSince = async () => { + pullEntered.resolve() + await releasePull.promise + return { + type: `rpc:pullSince:res`, + rpcId: `old-delayed-pull`, + ok: true, + latestTerm: 1, + latestSeq: 2, + latestRowVersion: 2, + requiresFullReload: false, + changedKeys: [`stale`], + deletedKeys: [], + deltas: [ + { + txId: `old-delayed-delta`, + latestRowVersion: 2, + changedRows: [ + { + key: `stale`, + value: { id: `stale`, title: `must stay with old owner` }, + }, + ], + deletedKeys: [], + rowMetadataMutations: [], + collectionMetadataMutations: [], + }, + ], + } + } + const replacementControlsReady = createEventGate() + let sourceRuns = 0 + const collection = createCollection( + persistedCollectionOptions({ + id, + getKey: (row) => row.id, + sync: { + sync: ({ markReady }) => { + sourceRuns++ + if (sourceRuns === 2) replacementControlsReady.resolve() + markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + let replacementReady: + | Promise< + { status: `fulfilled` } | { status: `rejected`; reason: unknown } + > + | undefined + let cleanedUp = false + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `delayed pull initial owner ready`, + ) + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 2, + txId: `gap-trigger`, + latestRowVersion: 2, + requiresFullReload: false, + changedRows: [], + deletedKeys: [], + }) + await atPersistedOracleCheckpoint( + pullEntered.promise, + `old pullSince entered`, + ) + + await collection.cleanup() + cleanedUp = true + collection.startSyncImmediate() + cleanedUp = false + replacementReady = collection.stateWhenReady().then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + await atPersistedOracleCheckpoint( + replacementControlsReady.promise, + `delayed pull replacement controls installed`, + ) + releasePull.resolve() + await atPersistedOracleCheckpoint( + replacementHydrationEntered.promise, + `replacement hydration entered after delayed pull replay`, + ) + + expect({ + sourceRuns, + subsetCalls, + staleVisible: collection.get(`stale`), + staleDurable: adapter.rows.get(`stale`), + replacementPublicError: collection._lifecycle.getSyncError(), + }).toEqual({ + sourceRuns: 2, + subsetCalls: 2, + staleVisible: undefined, + staleDurable: undefined, + replacementPublicError: undefined, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releasePull.resolve() + releaseReplacementHydration.resolve() + await cleanupPersistedOracle( + [ + () => replacementReady, + () => (cleanedUp ? undefined : collection.cleanup()), + ], + hasPrimaryFailure, + ) + } + }) + + it(`does not apply a later seq-gap replay delta through replacement controls`, async () => { + const id = `two-delta-replay-lifecycle` + const firstReceiptEntered = createEventGate() + const releaseFirstReceipt = createEventGate() + const replacementControlsInstalled = createEventGate() + const replacementHydrationEntered = createEventGate() + const releaseReplacementHydration = createEventGate() + let subsetCalls = 0 + const adapter = createRecordingAdapter() + adapter.loadSubset = async () => { + subsetCalls++ + if (subsetCalls === 2) { + replacementHydrationEntered.resolve() + await releaseReplacementHydration.promise + return [ + { + key: `replacement`, + value: { id: `replacement`, title: `B baseline` }, + }, + ] + } + return [] + } + const coordinator = createFailStopCoordinatorHarness(id) + const pullArguments: Array<{ + collectionId: string + fromRowVersion: number + }> = [] + coordinator.pullSince = async (collectionId, fromRowVersion) => { + pullArguments.push({ collectionId, fromRowVersion }) + return { + type: `rpc:pullSince:res`, + rpcId: `two-delta-replay`, + ok: true, + latestTerm: 1, + latestSeq: 3, + latestRowVersion: 3, + requiresFullReload: false, + changedKeys: [`first`, `second`], + deletedKeys: [], + deltas: [ + { + txId: `A-delta-1`, + latestRowVersion: 1, + changedRows: [ + { + key: `first`, + value: { id: `first`, title: `A first delta` }, + }, + ], + deletedKeys: [], + rowMetadataMutations: [ + { type: `set`, key: `first`, value: { owner: `A-first` } }, + ], + collectionMetadataMutations: [ + { type: `set`, key: `replay:first`, value: `A-first` }, + ], + }, + { + txId: `A-delta-2`, + latestRowVersion: 2, + changedRows: [ + { + key: `second`, + value: { id: `second`, title: `A second delta` }, + }, + ], + deletedKeys: [], + rowMetadataMutations: [ + { type: `set`, key: `second`, value: { owner: `A-second` } }, + ], + collectionMetadataMutations: [ + { type: `set`, key: `replay:second`, value: `A-second` }, + ], + }, + ], + } + } + + let gateNextApplicationReceipt = false + let controlsRun = 0 + const applicationReceipts: Array<{ + controlsOwner: `A` | `B` + returnedReceipt: `actual` | `held` + rowMetadataKeys: Array + collectionMetadataKeys: Array + }> = [] + const baseOptions = persistedCollectionOptions({ + id, + getKey: (row) => row.id, + sync: { sync: ({ markReady }) => markReady() }, + persistence: { adapter, coordinator }, + }) + const baseSync = baseOptions.sync.sync + const collection = createCollection({ + ...baseOptions, + sync: { + ...baseOptions.sync, + sync: (params) => { + controlsRun++ + const controlsOwner = + controlsRun === 1 ? (`A` as const) : (`B` as const) + if (controlsOwner === `B`) replacementControlsInstalled.resolve() + let rowMetadataKeys: Array = [] + let collectionMetadataKeys: Array = [] + return baseSync({ + ...params, + begin: (options) => { + rowMetadataKeys = [] + collectionMetadataKeys = [] + return params.begin(options) + }, + metadata: { + row: { + ...params.metadata!.row, + set: (key, value) => { + rowMetadataKeys.push(String(key)) + params.metadata!.row.set(key, value) + }, + delete: (key) => { + rowMetadataKeys.push(String(key)) + params.metadata!.row.delete(key) + }, + }, + collection: { + ...params.metadata!.collection, + set: (key, value) => { + collectionMetadataKeys.push(key) + params.metadata!.collection.set(key, value) + }, + delete: (key) => { + collectionMetadataKeys.push(key) + params.metadata!.collection.delete(key) + }, + }, + }, + commit: (signal) => { + const actualReceipt = params.commit(signal) + if ( + !gateNextApplicationReceipt && + applicationReceipts.length === 0 + ) { + return actualReceipt + } + const trace = { + controlsOwner, + returnedReceipt: `actual` as `actual` | `held`, + rowMetadataKeys: [...rowMetadataKeys], + collectionMetadataKeys: [...collectionMetadataKeys], + } + applicationReceipts.push(trace) + if (gateNextApplicationReceipt) { + gateNextApplicationReceipt = false + trace.returnedReceipt = `held` + firstReceiptEntered.resolve() + void Promise.resolve(actualReceipt).catch(() => undefined) + return releaseFirstReceipt.promise + } + return actualReceipt + }, + }) + }, + }, + }) + let replacementReady: Promise | undefined + let cleanedUp = false + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `two-delta A lifecycle ready`, + ) + applicationReceipts.length = 0 + gateNextApplicationReceipt = true + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 3, + txId: `gap-trigger`, + latestRowVersion: 3, + requiresFullReload: false, + changedRows: [], + deletedKeys: [], + }) + await atPersistedOracleCheckpoint( + firstReceiptEntered.promise, + `two-delta first A receipt held`, + ) + expect(collection.get(`first`)?.title).toBe(`A first delta`) + + await collection.cleanup() + cleanedUp = true + collection.startSyncImmediate() + cleanedUp = false + replacementReady = collection.stateWhenReady() + await atPersistedOracleCheckpoint( + replacementControlsInstalled.promise, + `two-delta B controls installed`, + ) + + releaseFirstReceipt.resolve() + await atPersistedOracleCheckpoint( + replacementHydrationEntered.promise, + `two-delta B hydration held`, + ) + await Promise.resolve() + + const secondReceipts = applicationReceipts.filter( + (receipt) => + receipt.rowMetadataKeys.includes(`second`) || + receipt.collectionMetadataKeys.includes(`replay:second`), + ) + expect({ + controlsRun, + pullArguments, + secondReceipts, + secondRowMetadata: collection._state.syncedMetadata.get(`second`), + secondCollectionMetadata: + collection._state.syncedCollectionMetadata.get(`replay:second`), + publicError: collection._lifecycle.getSyncError(), + }).toEqual({ + controlsRun: 2, + pullArguments: [{ collectionId: id, fromRowVersion: 0 }], + secondReceipts: [], + secondRowMetadata: undefined, + secondCollectionMetadata: undefined, + publicError: undefined, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseFirstReceipt.resolve() + releaseReplacementHydration.resolve() + await cleanupPersistedOracle( + [ + () => replacementReady, + () => (cleanedUp ? undefined : collection.cleanup()), + ], + hasPrimaryFailure, + ) + } + }) + + it(`does not confirm a delayed local coordinator response through a restarted owner`, async () => { + const id = `delayed-local-confirmation-lifecycle` + const adapter = createRecordingAdapter() + const replacementHydrationEntered = createEventGate() + const releaseReplacementHydration = createEventGate() + let subsetCalls = 0 + adapter.loadSubset = async () => { + subsetCalls++ + if (subsetCalls === 2) { + replacementHydrationEntered.resolve() + await releaseReplacementHydration.promise + return [ + { + key: `replacement`, + value: { id: `replacement`, title: `new owner baseline` }, + }, + ] + } + return [] + } + const coordinator = createFailStopCoordinatorHarness(id) + const requestEntered = createEventGate() + const releaseRequest = createEventGate() + coordinator.requestApplyLocalMutations = async ( + _collectionId, + mutations, + ) => { + requestEntered.resolve() + await releaseRequest.promise + return { + type: `rpc:applyLocalMutations:res`, + rpcId: `old-delayed-local`, + ok: true, + term: 1, + seq: 1, + latestRowVersion: 1, + acceptedMutationIds: mutations.map((mutation) => mutation.mutationId), + } + } + const collection = createCollection( + persistedCollectionOptions({ + id, + getKey: (row) => row.id, + persistence: { adapter, coordinator }, + }), + ) + const oldRow = { id: `old-local`, title: `must stay with old owner` } + let localTransaction: ReturnType | undefined + let replacementReady: + | Promise< + { status: `fulfilled` } | { status: `rejected`; reason: unknown } + > + | undefined + let cleanedUp = false + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `delayed local initial owner ready`, + ) + localTransaction = collection.insert(oldRow) + await atPersistedOracleCheckpoint( + requestEntered.promise, + `old local coordinator request entered`, + ) + + await collection.cleanup() + cleanedUp = true + collection.startSyncImmediate() + cleanedUp = false + replacementReady = collection.stateWhenReady().then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + releaseRequest.resolve() + await atPersistedOracleCheckpoint( + replacementHydrationEntered.promise, + `replacement hydration entered after delayed local confirmation`, + ) + + expect({ + subsetCalls, + staleVisible: collection.get(oldRow.id), + staleDurable: adapter.rows.get(oldRow.id), + replacementPublicError: collection._lifecycle.getSyncError(), + }).toEqual({ + subsetCalls: 2, + staleVisible: undefined, + staleDurable: undefined, + replacementPublicError: undefined, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseRequest.resolve() + releaseReplacementHydration.resolve() + await cleanupPersistedOracle( + [ + () => localTransaction?.isPersisted.promise.catch(() => undefined), + () => replacementReady, + () => (cleanedUp ? undefined : collection.cleanup()), + ], + hasPrimaryFailure, + ) + } + }) + + it(`does not admit an old pending user hook through a restarted coordinator`, async () => { + const id = `pending-user-hook-lifecycle` + const adapter = createRecordingAdapter() + const coordinator = createFailStopCoordinatorHarness(id) + const hookEntered = createEventGate() + const releaseHook = createEventGate() + let coordinatorRequests = 0 + coordinator.requestApplyLocalMutations = async ( + _collectionId, + mutations, + ) => { + coordinatorRequests++ + return { + type: `rpc:applyLocalMutations:res`, + rpcId: `stale-user-hook-request`, + ok: true, + term: 1, + seq: 1, + latestRowVersion: 1, + acceptedMutationIds: mutations.map((mutation) => mutation.mutationId), + } + } + const collection = createCollection( + persistedCollectionOptions({ + id, + getKey: (row) => row.id, + onInsert: async () => { + hookEntered.resolve() + await releaseHook.promise + return {} + }, + persistence: { adapter, coordinator }, + }), + ) + let transaction: ReturnType | undefined + let transactionOutcome: + | Promise< + { status: `fulfilled` } | { status: `rejected`; reason: unknown } + > + | undefined + let replacementReady: + | Promise< + { status: `fulfilled` } | { status: `rejected`; reason: unknown } + > + | undefined + let cleanedUp = false + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `pending user hook initial owner ready`, + ) + transaction = collection.insert({ + id: `old-hook`, + title: `must not enter replacement coordinator`, + }) + transactionOutcome = transaction.isPersisted.promise.then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + await atPersistedOracleCheckpoint( + hookEntered.promise, + `old user hook entered before coordinator admission`, + ) + expect(coordinatorRequests).toBe(0) + + await collection.cleanup() + cleanedUp = true + collection.startSyncImmediate() + cleanedUp = false + replacementReady = collection.stateWhenReady().then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + const replacementOutcome = await atPersistedOracleCheckpoint( + replacementReady, + `pending user hook replacement owner ready`, + ) + releaseHook.resolve() + const outcome = await atPersistedOracleCheckpoint( + transactionOutcome, + `pending user hook transaction settled`, + ) + + expect({ + replacementStatus: replacementOutcome.status, + transactionStatus: outcome.status, + coordinatorRequests, + durableOldRow: adapter.rows.get(`old-hook`), + }).toEqual({ + replacementStatus: `fulfilled`, + transactionStatus: `rejected`, + coordinatorRequests: 0, + durableOldRow: undefined, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseHook.resolve() + await cleanupPersistedOracle( + [ + () => transactionOutcome, + () => replacementReady, + () => (cleanedUp ? undefined : collection.cleanup()), + ], + hasPrimaryFailure, + ) + } + }) + + it(`does not let old loopback readiness complete a restarted hydration`, async () => { + const adapter = createRecordingAdapter() + const oldHydrationEntered = createEventGate() + const releaseOldHydration = createEventGate() + const replacementHydrationEntered = createEventGate() + const releaseReplacementHydration = createEventGate() + let subsetCalls = 0 + adapter.loadSubset = async () => { + subsetCalls++ + if (subsetCalls === 1) { + oldHydrationEntered.resolve() + await releaseOldHydration.promise + return [] + } + replacementHydrationEntered.resolve() + await releaseReplacementHydration.promise + return [ + { + key: `replacement`, + value: { id: `replacement`, title: `new owner baseline` }, + }, + ] + } + const collection = createCollection( + persistedCollectionOptions({ + id: `loopback-late-readiness-lifecycle`, + getKey: (row) => row.id, + persistence: { adapter }, + }), + ) + const oldReady = collection.stateWhenReady().then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + let replacementReady: + | Promise< + { status: `fulfilled` } | { status: `rejected`; reason: unknown } + > + | undefined + let cleanedUp = false + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + oldHydrationEntered.promise, + `old loopback hydration entered`, + ) + await collection.cleanup() + cleanedUp = true + collection.startSyncImmediate() + cleanedUp = false + replacementReady = collection.stateWhenReady().then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + + releaseOldHydration.resolve() + await atPersistedOracleCheckpoint( + replacementHydrationEntered.promise, + `replacement loopback hydration entered`, + ) + await flushAsyncWork() + + expect({ + subsetCalls, + replacementIsReady: collection.isReady(), + replacementVisible: collection.get(`replacement`), + publicError: collection._lifecycle.getSyncError(), + }).toEqual({ + subsetCalls: 2, + replacementIsReady: false, + replacementVisible: undefined, + publicError: undefined, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseOldHydration.resolve() + releaseReplacementHydration.resolve() + await cleanupPersistedOracle( + [ + () => oldReady, + () => replacementReady, + () => (cleanedUp ? undefined : collection.cleanup()), + ], + hasPrimaryFailure, + ) + } + }) + + it(`marks a targeted invalidation commit-receipt rejection terminal`, async () => { + const id = `targeted-invalidation-receipt` + const adapter = createRecordingAdapter() + const coordinator = createFailStopCoordinatorHarness(id) + const receiptEntered = createEventGate() + const rejectedReceipt = createEventGate() + const receiptError = new Error( + `targeted invalidation receipt failed exactly`, + ) + let rejectNextCommit = false + const baseOptions = persistedCollectionOptions({ + id, + getKey: (row) => row.id, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { adapter, coordinator }, + }) + const baseSync = baseOptions.sync.sync + const collection = createCollection({ + ...baseOptions, + sync: { + ...baseOptions.sync, + sync: (params) => + baseSync({ + ...params, + commit: (signal) => { + const actualReceipt = params.commit(signal) + if (!rejectNextCommit) return actualReceipt + rejectNextCommit = false + receiptEntered.resolve() + void Promise.resolve(actualReceipt).catch(() => undefined) + return rejectedReceipt.promise + }, + }), + }, + }) + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `targeted receipt collection ready`, + ) + rejectNextCommit = true + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 1, + txId: `targeted-receipt-failure`, + latestRowVersion: 1, + requiresFullReload: false, + changedRows: [ + { + key: `targeted`, + value: { id: `targeted`, title: `publication attempted` }, + }, + ], + deletedKeys: [], + }) + await atPersistedOracleCheckpoint( + receiptEntered.promise, + `targeted invalidation receipt entered`, + ) + rejectedReceipt.reject(receiptError) + await flushAsyncWork() + await flushAsyncWork() + + expect({ + status: collection.status, + exactPublicError: collection._lifecycle.getSyncError() === receiptError, + }).toEqual({ + status: `error`, + exactPublicError: true, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + rejectedReceipt.reject(receiptError) + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }) + + it(`waits for a reset truncate receipt before reloading`, async () => { + const id = `reset-truncate-receipt` + const seed = { id: `seed`, title: `must survive a failed reset` } + const adapter = createRecordingAdapter([seed]) + const coordinator = createFailStopCoordinatorHarness(id) + const receiptEntered = createEventGate() + const rejectedReceipt = createEventGate() + const receiptError = new Error(`reset truncate receipt failed exactly`) + let rejectNextCommit = false + const baseOptions = persistedCollectionOptions({ + id, + getKey: (row) => row.id, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { adapter, coordinator }, + }) + const baseSync = baseOptions.sync.sync + const collection = createCollection({ + ...baseOptions, + sync: { + ...baseOptions.sync, + sync: (params) => + baseSync({ + ...params, + commit: (signal) => { + const actualReceipt = params.commit(signal) + if (!rejectNextCommit) return actualReceipt + rejectNextCommit = false + receiptEntered.resolve() + void Promise.resolve(actualReceipt).catch(() => undefined) + return rejectedReceipt.promise + }, + }), + }, + }) + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `reset receipt collection ready`, + ) + const baselineLoadCount = adapter.loadSubsetCalls.length + rejectNextCommit = true + coordinator.emit({ + type: `collection:reset`, + schemaVersion: 1, + resetEpoch: 1, + }) + await atPersistedOracleCheckpoint( + receiptEntered.promise, + `reset truncate receipt entered`, + ) + await flushAsyncWork() + await flushAsyncWork() + + expect(adapter.loadSubsetCalls.length).toBe(baselineLoadCount) + + rejectedReceipt.reject(receiptError) + await flushAsyncWork() + await flushAsyncWork() + + expect({ + status: collection.status, + exactPublicError: collection._lifecycle.getSyncError() === receiptError, + }).toEqual({ + status: `error`, + exactPublicError: true, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + rejectedReceipt.reject(receiptError) + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }) + + it(`keeps local persistence pending through confirmation receipt settlement`, async () => { + const id = `local-confirmation-receipt` + const adapter = createRecordingAdapter() + const coordinator = createFailStopCoordinatorHarness(id) + const receiptEntered = createEventGate() + const rejectedReceipt = createEventGate() + const receiptError = new Error(`local confirmation receipt failed exactly`) + let rejectNextCommit = false + coordinator.requestApplyLocalMutations = async ( + _collectionId, + mutations, + ) => { + rejectNextCommit = true + return { + type: `rpc:applyLocalMutations:res`, + rpcId: `local-confirmation`, + ok: true, + term: 1, + seq: 1, + latestRowVersion: 1, + acceptedMutationIds: mutations.map((mutation) => mutation.mutationId), + } + } + const baseOptions = persistedCollectionOptions({ + id, + getKey: (row) => row.id, + persistence: { adapter, coordinator }, + }) + const baseSync = baseOptions.sync.sync + const collection = createCollection({ + ...baseOptions, + sync: { + ...baseOptions.sync, + sync: (params) => + baseSync({ + ...params, + commit: (signal) => { + const actualReceipt = params.commit(signal) + if (!rejectNextCommit) return actualReceipt + rejectNextCommit = false + receiptEntered.resolve() + void Promise.resolve(actualReceipt).catch(() => undefined) + return rejectedReceipt.promise + }, + }), + }, + }) + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `local confirmation receipt collection ready`, + ) + const transaction = collection.insert({ + id: `local`, + title: `durable before confirmation settles`, + }) + const transactionOutcome = transaction.isPersisted.promise.then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + await atPersistedOracleCheckpoint( + receiptEntered.promise, + `local confirmation receipt entered`, + ) + const pending = observeSettlement( + transactionOutcome.then(() => undefined), + ) + await Promise.resolve() + expect(pending.read()).toEqual({ status: `pending` }) + + rejectedReceipt.reject(receiptError) + const outcome = await atPersistedOracleCheckpoint( + transactionOutcome, + `local confirmation transaction settled`, + ) + await flushAsyncWork() + + expect({ + transactionStatus: outcome.status, + transactionExact: + outcome.status === `rejected` && outcome.reason === receiptError, + status: collection.status, + exactPublicError: collection._lifecycle.getSyncError() === receiptError, + }).toEqual({ + transactionStatus: `rejected`, + transactionExact: true, + status: `error`, + exactPublicError: true, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + rejectedReceipt.reject(receiptError) + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }) + + it(`does not continue an old hydration into a replacement after its applied receipt settles`, async () => { + const id = `hydration-applied-receipt-lifecycle` + const adapter = createRecordingAdapter() + const receiptEntered = createEventGate() + const releaseReceipt = createEventGate() + let gateNextCommit = false + let subsetCalls = 0 + adapter.loadSubset = async () => { + subsetCalls++ + if (subsetCalls === 1) { + gateNextCommit = true + return [ + { + key: `old`, + value: { id: `old`, title: `old owner hydration` }, + }, + ] + } + return [ + { + key: `replacement`, + value: { id: `replacement`, title: `new owner hydration` }, + }, + ] + } + const baseOptions = persistedCollectionOptions({ + id, + getKey: (row) => row.id, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { adapter }, + }) + const baseSync = baseOptions.sync.sync + const collection = createCollection({ + ...baseOptions, + sync: { + ...baseOptions.sync, + sync: (params) => + baseSync({ + ...params, + commit: (signal) => { + const actualReceipt = params.commit(signal) + if (!gateNextCommit) return actualReceipt + gateNextCommit = false + receiptEntered.resolve() + void Promise.resolve(actualReceipt).catch(() => undefined) + return releaseReceipt.promise + }, + }), + }, + }) + const oldReady = collection.stateWhenReady().then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + let replacementReady: + | Promise< + { status: `fulfilled` } | { status: `rejected`; reason: unknown } + > + | undefined + let cleanedUp = false + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + receiptEntered.promise, + `old hydration applied receipt entered`, + ) + expect(stripVirtualProps(collection.get(`old`))).toEqual({ + id: `old`, + title: `old owner hydration`, + }) + + await collection.cleanup() + cleanedUp = true + collection.startSyncImmediate() + cleanedUp = false + replacementReady = collection.stateWhenReady().then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + releaseReceipt.resolve() + const replacementOutcome = await atPersistedOracleCheckpoint( + replacementReady, + `replacement hydration after old applied receipt`, + ) + + expect({ + replacementStatus: replacementOutcome.status, + subsetCalls, + oldVisible: collection.get(`old`), + replacementVisible: stripVirtualProps(collection.get(`replacement`)), + status: collection.status, + publicError: collection._lifecycle.getSyncError(), + }).toEqual({ + replacementStatus: `fulfilled`, + subsetCalls: 2, + oldVisible: undefined, + replacementVisible: { + id: `replacement`, + title: `new owner hydration`, + }, + status: `ready`, + publicError: undefined, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseReceipt.resolve() + await cleanupPersistedOracle( + [ + () => oldReady, + () => replacementReady, + () => (cleanedUp ? undefined : collection.cleanup()), + ], + hasPrimaryFailure, + ) + } + }) + + it(`does not let an old post-receipt continuation flush replacement coordinator work`, async () => { + const id = `post-receipt-queue-lifecycle` + const adapter = createRecordingAdapter() + const coordinator = createFailStopCoordinatorHarness(id) + const receiptEntered = createEventGate() + const releaseReceipt = createEventGate() + const replacementSourceStarted = createEventGate() + const replacementHydrationEntered = createEventGate() + const releaseReplacementHydration = createEventGate() + let gateNextCommit = false + let subsetCalls = 0 + let sourceRuns = 0 + adapter.loadSubset = async () => { + subsetCalls++ + if (subsetCalls === 1) { + gateNextCommit = true + return [ + { + key: `old`, + value: { id: `old`, title: `old owner hydration` }, + }, + ] + } + replacementHydrationEntered.resolve() + await releaseReplacementHydration.promise + return [ + { + key: `replacement`, + value: { id: `replacement`, title: `new owner hydration` }, + }, + ] + } + const baseOptions = persistedCollectionOptions({ + id, + getKey: (row) => row.id, + sync: { + sync: ({ markReady }) => { + sourceRuns++ + if (sourceRuns === 2) replacementSourceStarted.resolve() + markReady() + }, + }, + persistence: { adapter, coordinator }, + }) + const baseSync = baseOptions.sync.sync + const collection = createCollection({ + ...baseOptions, + sync: { + ...baseOptions.sync, + sync: (params) => + baseSync({ + ...params, + commit: (signal) => { + const actualReceipt = params.commit(signal) + if (!gateNextCommit) return actualReceipt + gateNextCommit = false + receiptEntered.resolve() + void Promise.resolve(actualReceipt).catch(() => undefined) + return releaseReceipt.promise + }, + }), + }, + }) + const oldReady = collection.stateWhenReady().then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + let replacementReady: + | Promise< + { status: `fulfilled` } | { status: `rejected`; reason: unknown } + > + | undefined + let cleanedUp = false + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + receiptEntered.promise, + `old post-receipt hydration entered`, + ) + await collection.cleanup() + cleanedUp = true + collection.startSyncImmediate() + cleanedUp = false + replacementReady = collection.stateWhenReady().then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + await atPersistedOracleCheckpoint( + replacementSourceStarted.promise, + `replacement source subscribed behind old receipt`, + ) + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 1, + txId: `replacement-queued-work`, + latestRowVersion: 1, + requiresFullReload: false, + changedRows: [ + { + key: `queued`, + value: { + id: `queued`, + title: `belongs after replacement hydrate`, + }, + }, + ], + deletedKeys: [], + }) + + releaseReceipt.resolve() + await atPersistedOracleCheckpoint( + replacementHydrationEntered.promise, + `replacement hydration entered after old receipt`, + ) + await flushAsyncWork() + + expect({ + sourceRuns, + subsetCalls, + queuedVisible: collection.get(`queued`), + }).toEqual({ + sourceRuns: 2, + subsetCalls: 2, + queuedVisible: undefined, + }) + + releaseReplacementHydration.resolve() + const replacementOutcome = await atPersistedOracleCheckpoint( + replacementReady, + `replacement hydration settled before coordinator work`, + ) + await flushAsyncWork() + await flushAsyncWork() + + expect({ + replacementStatus: replacementOutcome.status, + oldVisible: collection.get(`old`), + replacementVisible: stripVirtualProps(collection.get(`replacement`)), + queuedVisible: stripVirtualProps(collection.get(`queued`)), + publicError: collection._lifecycle.getSyncError(), + }).toEqual({ + replacementStatus: `fulfilled`, + oldVisible: undefined, + replacementVisible: { + id: `replacement`, + title: `new owner hydration`, + }, + queuedVisible: { + id: `queued`, + title: `belongs after replacement hydrate`, + }, + publicError: undefined, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseReceipt.resolve() + releaseReplacementHydration.resolve() + await cleanupPersistedOracle( + [ + () => oldReady, + () => replacementReady, + () => (cleanedUp ? undefined : collection.cleanup()), + ], + hasPrimaryFailure, + ) + } + }) + + it(`rejects new source transactions after a terminal persistence failure`, async () => { + const adapter = createRecordingAdapter() + const adapterError = Object.assign( + new Error(`terminal adapter write failed exactly`), + { + code: `SQLITE_FULL`, + path: `adapter.applyCommittedTx`, + }, + ) + const successfulApply = adapter.applyCommittedTx.bind(adapter) + let applyCalls = 0 + adapter.applyCommittedTx = async (collectionId, tx) => { + applyCalls++ + if (applyCalls === 1) throw adapterError + await successfulApply(collectionId, tx) + } + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: (() => true | Promise) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present-terminal-persistence-admission`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit + markReady() + }, + }, + persistence: { adapter }, + }), + ) + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `terminal persistence admission collection ready`, + ) + remoteBegin!() + remoteWrite!({ + type: `insert`, + value: { id: `first`, title: `published before persistence` }, + }) + const firstOutcome = await atPersistedOracleCheckpoint( + Promise.resolve(remoteCommit!()).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `first terminal persistence receipt`, + ) + const terminalError = + firstOutcome.status === `rejected` ? firstOutcome.reason : undefined + + expect({ + receiptStatus: firstOutcome.status, + errorName: + terminalError instanceof Error ? terminalError.name : undefined, + cause: terminalError instanceof Error ? terminalError.cause : undefined, + publicErrorIsReceipt: + collection._lifecycle.getSyncError() === terminalError, + status: collection.status, + firstVisible: stripVirtualProps(collection.get(`first`)), + firstDurable: adapter.rows.get(`first`), + applyCalls, + }).toEqual({ + receiptStatus: `rejected`, + errorName: `PersistenceDurabilityError`, + cause: adapterError, + publicErrorIsReceipt: true, + status: `error`, + firstVisible: { + id: `first`, + title: `published before persistence`, + }, + firstDurable: undefined, + applyCalls: 1, + }) + + remoteBegin!() + remoteWrite!({ + type: `insert`, + value: { id: `late`, title: `must not be admitted` }, + }) + const lateOutcome = await atPersistedOracleCheckpoint( + Promise.resolve(remoteCommit!()).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `late terminal persistence receipt`, + ) + + expect({ + receiptStatus: lateOutcome.status, + exactReceipt: + lateOutcome.status === `rejected` && + lateOutcome.reason === terminalError, + publicErrorIsTerminal: + collection._lifecycle.getSyncError() === terminalError, + status: collection.status, + lateVisible: collection.get(`late`), + lateDurable: adapter.rows.get(`late`), + applyCalls, + }).toEqual({ + receiptStatus: `rejected`, + exactReceipt: true, + publicErrorIsTerminal: true, + status: `error`, + lateVisible: undefined, + lateDurable: undefined, + applyCalls: 1, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }) + + it.each([`hydration`, `durability`] as const)( + `fences metadata and truncate transactions after a terminal %s failure`, + async (failureKind) => { + const harness = await createTerminalFailureHarness( + failureKind, + `terminal-transaction-surface-${failureKind}`, + ) + const { + adapter, + collection, + coordinator, + seed, + sourceParams, + terminalError, + } = harness + const applyCallsBefore = adapter.applyCommittedTxCalls.length + const publishCallsBefore = coordinator.publishCalls.length + const operations: Array<{ name: string; apply: () => void }> = [ + { + name: `row metadata set`, + apply: () => + sourceParams.metadata!.row.set(seed.id, `must-not-replace-row`), + }, + { + name: `row metadata delete`, + apply: () => sourceParams.metadata!.row.delete(seed.id), + }, + { + name: `collection metadata set`, + apply: () => + sourceParams.metadata!.collection.set( + `resume`, + `must-not-replace-collection`, + ), + }, + { + name: `collection metadata delete`, + apply: () => sourceParams.metadata!.collection.delete(`resume`), + }, + { + name: `truncate`, + apply: () => sourceParams.truncate(), + }, + ] + let hasPrimaryFailure = false + + try { + for (const operation of operations) { + sourceParams.begin() + const operationOutcome = await Promise.resolve() + .then(operation.apply) + .then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + const receiptOutcome = await atPersistedOracleCheckpoint( + Promise.resolve(sourceParams.commit()).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `${failureKind} ${operation.name} terminal receipt`, + ) + + expect({ + operation: operationOutcome.status, + receipt: receiptOutcome.status, + exactReceipt: + receiptOutcome.status === `rejected` && + receiptOutcome.reason === terminalError, + visibleSeed: stripVirtualProps(collection.get(seed.id)), + durableSeed: adapter.rows.get(seed.id), + rowMetadata: sourceParams.metadata!.row.get(seed.id), + collectionMetadata: sourceParams.metadata!.collection.get(`resume`), + applyCalls: adapter.applyCommittedTxCalls.length, + publishCalls: coordinator.publishCalls.length, + status: collection.status, + exactPublicError: + collection._lifecycle.getSyncError() === terminalError, + }).toEqual({ + operation: `fulfilled`, + receipt: `rejected`, + exactReceipt: true, + visibleSeed: seed, + durableSeed: seed, + rowMetadata: `row-metadata-before-failure`, + collectionMetadata: `collection-metadata-before-failure`, + applyCalls: applyCallsBefore, + publishCalls: publishCallsBefore, + status: `error`, + exactPublicError: true, + }) + } + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }, + ) + + it(`fences reload, upstream hydration, and late readiness after terminal persistence failure`, async () => { + const harness = await createTerminalFailureHarness( + `durability`, + `terminal-persistence-reload-surfaces`, + ) + const { + adapter, + collection, + coordinator, + sourceParams, + terminalError, + upstreamLoads, + } = harness + const late = { id: `late-reload`, title: `must not hydrate` } + const loadCallsBefore = adapter.loadSubsetCalls.length + const upstreamCallsBefore = upstreamLoads.length + const publishCallsBefore = coordinator.publishCalls.length + adapter.loadSubset = (collectionId, options, context) => { + adapter.loadSubsetCalls.push({ + collectionId, + options, + requiredIndexSignatures: context?.requiredIndexSignatures ?? [], + }) + return Promise.resolve([{ key: late.id, value: late }]) + } + let hasPrimaryFailure = false + + try { + const loadOutcome = await atPersistedOracleCheckpoint( + Promise.resolve(collection._sync.loadSubset({ limit: 3 })).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `post-persistence terminal loadSubset`, + ) + sourceParams.markReady() + await flushAsyncWork() + + expect({ + loadStatus: loadOutcome.status, + loadExact: + loadOutcome.status === `rejected` && + loadOutcome.reason === terminalError, + adapterLoads: adapter.loadSubsetCalls.length, + upstreamLoads: upstreamLoads.length, + publishCalls: coordinator.publishCalls.length, + visibleLate: collection.get(late.id), + durableLate: adapter.rows.get(late.id), + status: collection.status, + exactPublicError: + collection._lifecycle.getSyncError() === terminalError, + }).toEqual({ + loadStatus: `rejected`, + loadExact: true, + adapterLoads: loadCallsBefore, + upstreamLoads: upstreamCallsBefore, + publishCalls: publishCallsBefore, + visibleLate: undefined, + durableLate: undefined, + status: `error`, + exactPublicError: true, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }) + + it(`rejects an already-admitted sibling when the preceding persistence fails`, async () => { + const adapter = createRecordingAdapter() + const firstPersistenceEntered = createEventGate() + const firstPersistence = createEventGate() + const adapterError = new Error(`first pending persistence failed exactly`) + const successfulApply = adapter.applyCommittedTx.bind(adapter) + let applyCalls = 0 + adapter.applyCommittedTx = async (...args) => { + applyCalls++ + if (applyCalls === 1) { + firstPersistenceEntered.resolve() + await firstPersistence.promise + return + } + await successfulApply(...args) + } + const coordinator = createFailStopCoordinatorHarness( + `terminal-pending-sibling`, + ) + let sourceParams!: TodoSyncParams + const collection = createCollection( + persistedCollectionOptions({ + id: `terminal-pending-sibling`, + getKey: (row) => row.id, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `pending sibling collection ready`, + ) + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `a`, title: `first publishes before persistence` }, + }) + const firstReceipt = Promise.resolve(sourceParams.commit()).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + await atPersistedOracleCheckpoint( + firstPersistenceEntered.promise, + `first persistence entered before sibling admission`, + ) + + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `b`, title: `must not publish or persist` }, + }) + const secondReceipt = Promise.resolve(sourceParams.commit()).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + const secondSettlement = observeSettlement( + secondReceipt.then( + () => undefined, + () => undefined, + ), + ) + expect(secondSettlement.read()).toEqual({ status: `pending` }) + + firstPersistence.reject(adapterError) + const firstOutcome = await atPersistedOracleCheckpoint( + firstReceipt, + `first failed persistence receipt`, + ) + const terminalError = + firstOutcome.status === `rejected` ? firstOutcome.reason : undefined + const secondOutcome = await atPersistedOracleCheckpoint( + secondReceipt, + `already-admitted sibling terminal receipt`, + ) + + expect({ + firstStatus: firstOutcome.status, + terminalName: + terminalError instanceof Error ? terminalError.name : undefined, + terminalCause: + terminalError instanceof Error ? terminalError.cause : undefined, + secondStatus: secondOutcome.status, + secondExact: + secondOutcome.status === `rejected` && + secondOutcome.reason === terminalError, + applyCalls, + firstVisible: stripVirtualProps(collection.get(`a`)), + firstDurable: adapter.rows.get(`a`), + secondVisible: collection.get(`b`), + secondDurable: adapter.rows.get(`b`), + publishCalls: coordinator.publishCalls.length, + status: collection.status, + exactPublicError: + collection._lifecycle.getSyncError() === terminalError, + }).toEqual({ + firstStatus: `rejected`, + terminalName: `PersistenceDurabilityError`, + terminalCause: adapterError, + secondStatus: `rejected`, + secondExact: true, + applyCalls: 1, + firstVisible: { + id: `a`, + title: `first publishes before persistence`, + }, + firstDurable: undefined, + secondVisible: undefined, + secondDurable: undefined, + publishCalls: 0, + status: `error`, + exactPublicError: true, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + firstPersistence.reject(adapterError) + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }) + + it(`marks direct adapter failure terminal and fences later local mutations`, async () => { + const adapter = createRecordingAdapter() + const adapterError = Object.assign( + new Error(`local adapter failed exactly`), + { + code: `SQLITE_IOERR`, + path: `adapter.applyCommittedTx`, + }, + ) + const successfulApply = adapter.applyCommittedTx.bind(adapter) + let applyCalls = 0 + adapter.applyCommittedTx = async (...args) => { + applyCalls++ + if (applyCalls === 1) throw adapterError + await successfulApply(...args) + } + const collection = createCollection( + persistedCollectionOptions({ + id: `terminal-loopback-local-durability`, + getKey: (row) => row.id, + persistence: { adapter }, + }), + ) + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `local durability collection ready`, + ) + const firstTransaction = collection.insert({ + id: `first-local`, + title: `must roll back`, + }) + const firstOutcome = await atPersistedOracleCheckpoint( + firstTransaction.isPersisted.promise.then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `first local durability failure`, + ) + const terminalError = + firstOutcome.status === `rejected` ? firstOutcome.reason : undefined + + const loadCallsBefore = adapter.loadSubsetCalls.length + adapter.loadSubset = (collectionId, options, context) => { + adapter.loadSubsetCalls.push({ + collectionId, + options, + requiredIndexSignatures: context?.requiredIndexSignatures ?? [], + }) + return Promise.resolve([ + { + key: `forced-late`, + value: { id: `forced-late`, title: `must not hydrate` }, + }, + ]) + } + const forceOutcome = await atPersistedOracleCheckpoint( + Promise.resolve(collection.utils.forceReloadSubset!({ limit: 4 })).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `force reload after local durability failure`, + ) + + const lateOutcome = await atPersistedOracleCheckpoint( + Promise.resolve() + .then(async () => { + const lateTransaction = collection.insert({ + id: `late-local`, + title: `must not be admitted`, + }) + await lateTransaction.isPersisted.promise + }) + .then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `late local terminal receipt`, + ) + + expect({ + firstStatus: firstOutcome.status, + errorName: + terminalError instanceof Error ? terminalError.name : undefined, + errorCause: + terminalError instanceof Error ? terminalError.cause : undefined, + errorCode: + terminalError instanceof Error + ? (terminalError as Error & { code?: unknown }).code + : undefined, + forceStatus: forceOutcome.status, + forceExact: + forceOutcome.status === `rejected` && + forceOutcome.reason === terminalError, + loadCalls: adapter.loadSubsetCalls.length, + forceVisible: collection.get(`forced-late`), + lateStatus: lateOutcome.status, + lateErrorName: + lateOutcome.status === `rejected` && + lateOutcome.reason instanceof Error + ? lateOutcome.reason.name + : undefined, + status: collection.status, + exactPublicError: + collection._lifecycle.getSyncError() === terminalError, + applyCalls, + firstVisible: collection.get(`first-local`), + firstDurable: adapter.rows.get(`first-local`), + lateVisible: collection.get(`late-local`), + lateDurable: adapter.rows.get(`late-local`), + }).toEqual({ + firstStatus: `rejected`, + errorName: `PersistenceDurabilityError`, + errorCause: adapterError, + errorCode: `SQLITE_IOERR`, + forceStatus: `rejected`, + forceExact: true, + loadCalls: loadCallsBefore, + forceVisible: undefined, + lateStatus: `rejected`, + lateErrorName: `CollectionStateError`, + status: `error`, + exactPublicError: true, + applyCalls: 1, + firstVisible: undefined, + firstDurable: undefined, + lateVisible: undefined, + lateDurable: undefined, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }) + + it(`fences direct acceptMutations and local mutations after hydration failure`, async () => { + const adapter = createRecordingAdapter() + const hydrationError = new Error(`loopback hydration failed exactly`) + adapter.loadSubset = () => Promise.reject(hydrationError) + const collection = createCollection( + persistedCollectionOptions({ + id: `terminal-loopback-hydration`, + getKey: (row) => row.id, + persistence: { adapter }, + }), + ) + let hasPrimaryFailure = false + + try { + await expect( + atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `loopback hydration rejected`, + ), + ).rejects.toBe(hydrationError) + expect(collection.status).toBe(`error`) + expect(collection._lifecycle.getSyncError()).toBe(hydrationError) + + adapter.loadSubset = () => Promise.resolve([]) + const now = new Date() + const directRow = { id: `direct`, title: `must not be accepted` } + const directMutation: PendingMutation = { + mutationId: `direct-after-hydration`, + original: {}, + modified: directRow, + changes: directRow, + globalKey: `terminal-loopback-hydration:direct`, + key: directRow.id, + type: `insert`, + metadata: undefined, + syncMetadata: {}, + optimistic: true, + createdAt: now, + updatedAt: now, + collection, + } + const directOutcome = await atPersistedOracleCheckpoint( + Promise.resolve( + collection.utils.acceptMutations({ + mutations: [ + directMutation as unknown as PendingMutation< + Record + >, + ], + }), + ).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `direct acceptMutations after hydration failure`, + ) + const localOutcome = await Promise.resolve() + .then(() => + collection.insert({ + id: `late-local-hydration`, + title: `must not be admitted`, + }), + ) + .then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + + expect({ + directStatus: directOutcome.status, + directExact: + directOutcome.status === `rejected` && + directOutcome.reason === hydrationError, + localStatus: localOutcome.status, + localErrorName: + localOutcome.status === `rejected` && + localOutcome.reason instanceof Error + ? localOutcome.reason.name + : undefined, + status: collection.status, + exactPublicError: + collection._lifecycle.getSyncError() === hydrationError, + applyCalls: adapter.applyCommittedTxCalls.length, + directVisible: collection.get(directRow.id), + directDurable: adapter.rows.get(directRow.id), + localVisible: collection.get(`late-local-hydration`), + localDurable: adapter.rows.get(`late-local-hydration`), + }).toEqual({ + directStatus: `rejected`, + directExact: true, + localStatus: `rejected`, + localErrorName: `CollectionStateError`, + status: `error`, + exactPublicError: true, + applyCalls: 0, + directVisible: undefined, + directDurable: undefined, + localVisible: undefined, + localDurable: undefined, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }) + + it(`preserves row metadata set before a metadata-less insert in the same sync transaction`, async () => { + const adapter = createRecordingAdapter() + const ownership = { queryCollection: { owners: [`gc:q1`] } } + const sync: SyncConfig = { + sync: ({ begin, write, commit, markReady, metadata }) => { + begin() + metadata?.row.set(`remote-1`, ownership) + write({ + type: `insert`, + value: { + id: `remote-1`, + title: `From remote`, + }, + }) + commit() + markReady() + }, + } + + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present`, + getKey: (item: Todo) => item.id, + sync, + persistence: { + adapter, + }, + }), + ) + + await collection.stateWhenReady() + await flushAsyncWork() + + expect(adapter.rowMetadata.get(`remote-1`)).toEqual(ownership) + expect(collection._state.syncedMetadata.get(`remote-1`)).toEqual(ownership) + }) + + it(`resets stale row metadata for a metadata-less insert with no queued metadata`, async () => { + const adapter = createRecordingAdapter() + adapter.rowMetadata.set(`remote-1`, { stale: true }) + const sync: SyncConfig = { + sync: ({ begin, write, commit, markReady }) => { + begin() + write({ + type: `insert`, + value: { + id: `remote-1`, + title: `From remote`, + }, + }) + commit() + markReady() + }, + } + + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present`, + getKey: (item: Todo) => item.id, + sync, + persistence: { + adapter, + }, + }), + ) + + await collection.stateWhenReady() + await flushAsyncWork() + + expect(adapter.rowMetadata.has(`remote-1`)).toBe(false) + }) + + it(`uses a stable generated collection id in sync-present mode when id is omitted`, async () => { + const adapter = createRecordingAdapter() + const options = persistedCollectionOptions({ + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { + adapter, + }, + }) - it(`persists metadata-only wrapped sync transactions`, async () => { - const adapter = createRecordingAdapter() + expect(options.id).toBeDefined() - const collection = createCollection( + const collection = createCollection(options) + await collection.preload() + await flushAsyncWork() + + expect(collection.id).toBe(options.id) + expect(adapter.loadSubsetCalls[0]?.collectionId).toBe(collection.id) + }) + + it(`keeps hydrated rows ahead of persisted startup rows`, async () => { + const adapter = createRecordingAdapter([ + { id: `1`, title: `Persisted title` }, + ]) + const descriptor = collectionOptions( persistedCollectionOptions({ - id: `persisted-metadata-only`, + id: `hydration-precedence`, getKey: (item) => item.id, sync: { - sync: ({ begin, commit, markReady, metadata }) => { - begin() - metadata?.collection.set(`runtime:key`, { persisted: true }) - commit() + sync: ({ markReady }) => { markReady() }, }, @@ -488,36 +5908,40 @@ describe(`persistedCollectionOptions`, () => { }, }), ) + const client = new DbClient() + client.hydrate({ + collections: [ + { + collectionId: descriptor.id, + rows: [ + { + key: `1`, + value: { id: `1`, title: `SSR title` }, + }, + ], + }, + ], + }) + const collection = client.collection(descriptor) await collection.stateWhenReady() await flushAsyncWork() - expect(adapter.applyCommittedTxCalls).toHaveLength(1) - expect(adapter.applyCommittedTxCalls[0]?.tx.mutations).toEqual([]) - expect(adapter.collectionMetadata.get(`runtime:key`)).toEqual({ - persisted: true, - }) - expect( - collection._state.syncedCollectionMetadata.get(`runtime:key`), - ).toEqual({ - persisted: true, + expect(collection.get(`1`)).toMatchObject({ + id: `1`, + title: `SSR title`, }) - }) - it(`replays metadata-only tx:committed deltas without full reload`, async () => { - const adapter = createRecordingAdapter([ - { - id: `1`, - title: `Tracked`, - }, - ]) - adapter.rowMetadata.set(`1`, { source: `initial` }) - const coordinator = createCoordinatorHarness() + await client.cleanup() + }) + it(`bootstraps and tracks persisted index lifecycle in sync-present mode`, async () => { + const adapter = createRecordingAdapter() const collection = createCollection( persistedCollectionOptions({ - id: `sync-present`, + id: `sync-present-indexes`, getKey: (item) => item.id, + defaultIndexType: BasicIndex, sync: { sync: ({ markReady }) => { markReady() @@ -525,322 +5949,658 @@ describe(`persistedCollectionOptions`, () => { }, persistence: { adapter, - coordinator, }, }), ) + const preSyncIndex = collection.createIndex((row) => row.title, { + name: `pre-sync-title`, + }) + const expectedPreSyncSignature = collection.getIndexMetadata()[0]?.signature + await collection.preload() await flushAsyncWork() - const loadSubsetCallsAfterPreload = adapter.loadSubsetCalls.length - coordinator.emit({ - type: `tx:committed`, - term: 1, - seq: 1, - txId: `tx-metadata-only`, - latestRowVersion: 2, - requiresFullReload: false, - changedRows: [], - deletedKeys: [], - rowMetadataMutations: [ - { - type: `set`, - key: `1`, - value: { source: `replayed` }, - }, - ], - collectionMetadataMutations: [ - { - type: `set`, - key: `electric:resume`, - value: { - kind: `reset`, - updatedAt: 2, - }, - }, - ], - }) + expect(expectedPreSyncSignature).toBeDefined() + expect( + adapter.ensureIndexCalls.some( + (call) => call.signature === expectedPreSyncSignature, + ), + ).toBe(true) - await flushAsyncWork() + const runtimeIndex = collection.createIndex((row) => row.id, { + name: `runtime-id`, + }) await flushAsyncWork() - expect(adapter.loadSubsetCalls.length).toBe(loadSubsetCallsAfterPreload) - expect(collection._state.syncedMetadata.get(`1`)).toEqual({ - source: `replayed`, - }) + const runtimeSignature = collection + .getIndexMetadata() + .find((index) => index.indexId === runtimeIndex.id)?.signature + expect(runtimeSignature).toBeDefined() expect( - collection._state.syncedCollectionMetadata.get(`electric:resume`), - ).toEqual({ - kind: `reset`, - updatedAt: 2, - }) + adapter.ensureIndexCalls.some( + (call) => call.signature === runtimeSignature, + ), + ).toBe(true) + + collection.removeIndex(preSyncIndex) + await flushAsyncWork() + expect( + adapter.markIndexRemovedCalls.some( + (call) => call.signature === expectedPreSyncSignature, + ), + ).toBe(true) }) - it(`uses pullSince replay deltas for metadata-bearing seq-gap recovery`, async () => { + it(`queues remote sync writes that arrive during hydration`, async () => { const adapter = createRecordingAdapter([ { - id: `1`, - title: `Tracked`, + id: `cached-1`, + title: `Cached row`, }, ]) - adapter.rowMetadata.set(`1`, { source: `initial` }) - const coordinator = createCoordinatorHarness() - coordinator.setPullSinceResponse({ - type: `rpc:pullSince:res`, - rpcId: `pull-metadata`, - ok: true, - latestTerm: 1, - latestSeq: 3, - latestRowVersion: 3, - requiresFullReload: false, - changedKeys: [], - deletedKeys: [], - deltas: [ + let resolveLoadSubset: (() => void) | undefined + adapter.loadSubset = async () => { + await new Promise((resolve) => { + resolveLoadSubset = resolve + }) + return [ { - txId: `tx-gap-1`, - latestRowVersion: 2, - changedRows: [], - deletedKeys: [], - rowMetadataMutations: [ - { - type: `set`, - key: `1`, - value: { source: `gap-replayed` }, - }, - ], - collectionMetadataMutations: [], + key: `cached-1`, + value: { + id: `cached-1`, + title: `Cached row`, + }, }, - { - txId: `tx-gap-2`, - latestRowVersion: 3, - changedRows: [], - deletedKeys: [], - rowMetadataMutations: [], - collectionMetadataMutations: [ - { - type: `set`, - key: `queryCollection:gc:q1`, - value: { - queryHash: `q1`, - mode: `until-revalidated`, - }, - }, - ], + ] + } + + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: (() => void) | undefined + + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit + markReady() + return {} + }, }, - ], + persistence: { + adapter, + }, + }), + ) + + const readyPromise = collection.stateWhenReady() + for (let attempt = 0; attempt < 20 && !resolveLoadSubset; attempt++) { + await flushAsyncWork() + } + + expect(resolveLoadSubset).toBeDefined() + expect(remoteBegin).toBeDefined() + + remoteBegin?.() + remoteWrite?.({ + type: `insert`, + value: { + id: `during-hydrate`, + title: `During hydrate`, + }, + }) + remoteCommit?.() + + resolveLoadSubset?.() + await readyPromise + await flushAsyncWork() + + expect(stripVirtualProps(collection.get(`cached-1`))).toEqual({ + id: `cached-1`, + title: `Cached row`, + }) + expect(stripVirtualProps(collection.get(`during-hydrate`))).toEqual({ + id: `during-hydrate`, + title: `During hydrate`, }) + }) + it(`preserves a source transaction committed before the next persisted hydration`, async () => { + const hydrationEntered = createEventGate() + const hydration = createEventGate() + const adapter = createRecordingAdapter() + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: (() => true | Promise) | undefined const collection = createCollection( persistedCollectionOptions({ - id: `sync-present`, + id: `before-hydration-boundary`, getKey: (item) => item.id, + syncMode: `on-demand`, sync: { - sync: ({ markReady }) => { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit markReady() }, }, - persistence: { - adapter, - coordinator, - }, + persistence: { adapter }, }), ) + let hasPrimaryFailure = false - await collection.preload() - await flushAsyncWork() - const loadSubsetCallsAfterPreload = adapter.loadSubsetCalls.length - - coordinator.emit({ - type: `tx:committed`, - term: 1, - seq: 3, - txId: `tx-gap-trigger`, - latestRowVersion: 3, - requiresFullReload: false, - changedRows: [], - deletedKeys: [], - }) - - await flushAsyncWork() - await flushAsyncWork() + try { + collection.startSyncImmediate() + await vi.waitFor(() => { + expect(remoteBegin).toBeTypeOf(`function`) + expect(remoteWrite).toBeTypeOf(`function`) + expect(remoteCommit).toBeTypeOf(`function`) + }) + const row = { id: `before`, title: `before` } + remoteBegin?.() + remoteWrite?.({ type: `insert`, value: row }) + const receipt = remoteCommit?.() + expect(receipt).toBeInstanceOf(Promise) + await receipt + adapter.loadSubset = async () => { + hydrationEntered.resolve() + await hydration.promise + return Array.from(adapter.rows, ([key, value]) => ({ key, value })) + } + const rehydrating = Promise.resolve(collection._sync.loadSubset({})) + await atPersistedOracleCheckpoint( + hydrationEntered.promise, + `on-demand hydration entered`, + ) - expect(coordinator.pullSinceCalls).toBe(1) - expect(adapter.loadSubsetCalls.length).toBe(loadSubsetCallsAfterPreload) - expect(collection._state.syncedMetadata.get(`1`)).toEqual({ - source: `gap-replayed`, - }) - expect( - collection._state.syncedCollectionMetadata.get(`queryCollection:gc:q1`), - ).toEqual({ - queryHash: `q1`, - mode: `until-revalidated`, - }) + expect(stripVirtualProps(collection.get(`before`))).toEqual(row) + expect(adapter.rows.get(`before`)).toEqual(row) + hydration.resolve() + await atPersistedOracleCheckpoint( + rehydrating, + `on-demand hydration completed`, + ) + expect(stripVirtualProps(collection.get(`before`))).toEqual(row) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + hydration.resolve() + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } }) - it(`throws InvalidSyncConfigError when sync key is present but null`, () => { - const invalidOptions = { - id: `invalid-sync-null`, - getKey: (item: Todo) => item.id, - sync: null, - persistence: { - adapter: createNoopAdapter(), - }, - } as unknown as PersistedSyncWrappedOptions - - expect(() => persistedCollectionOptions(invalidOptions)).toThrow( - InvalidSyncConfigError, - ) - }) + it.each([`during`, `straddles`, `after`] as const)( + `durably settles a source transaction that commits %s persisted hydration`, + async (position) => { + const hydrationEntered = createEventGate() + const hydration = createEventGate() + const adapter = createRecordingAdapter() + adapter.loadSubset = async () => { + hydrationEntered.resolve() + await hydration.promise + return [] + } - it(`throws InvalidSyncConfigError when sync key is present but invalid`, () => { - const invalidOptions = { - id: `invalid-sync-shape`, - getKey: (item: Todo) => item.id, - sync: {} as unknown as SyncConfig, - persistence: { - adapter: createNoopAdapter(), - }, - } as PersistedSyncWrappedOptions + const persistedTransactionOrder: Array = [] + const applyCommittedTx = adapter.applyCommittedTx.bind(adapter) + adapter.applyCommittedTx = async (...args) => { + const title = args[1].mutations.find( + (mutation) => mutation.type !== `delete`, + )?.value as Todo | undefined + if (title) persistedTransactionOrder.push(title.title) + await applyCommittedTx(...args) + } - expect(() => persistedCollectionOptions(invalidOptions)).toThrow( - InvalidSyncConfigError, - ) - }) + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: (() => true | Promise) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `hydration-boundary-${position}`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit + markReady() + }, + }, + persistence: { adapter }, + }), + ) + const ledger: Array = [] + const transactionId = position + const row = { id: `shared`, title: position } + let receipt: Promise | undefined + let hasPrimaryFailure = false + + const beginAndWrite = () => { + ledger.push({ type: `begin`, transactionId }) + remoteBegin?.() + ledger.push({ type: `write`, transactionId, row }) + remoteWrite?.({ type: `insert`, value: row }) + } + const commit = () => { + ledger.push({ type: `commit`, transactionId }) + const applied = remoteCommit?.() + expect(applied).toBeInstanceOf(Promise) + receipt = Promise.resolve(applied).then(() => undefined) + } - it(`uses SingleProcessCoordinator when coordinator is omitted`, () => { - const options = persistedCollectionOptions({ - id: `default-coordinator`, - getKey: (item) => item.id, - persistence: { - adapter: createNoopAdapter(), - }, - }) + try { + const ready = collection.stateWhenReady() + await atPersistedOracleCheckpoint( + hydrationEntered.promise, + `${position} hydration entered`, + ) + expect(remoteBegin).toBeTypeOf(`function`) + expect(remoteCommit).toBeTypeOf(`function`) + + if (position === `during`) { + beginAndWrite() + commit() + hydration.resolve() + await atPersistedOracleCheckpoint( + ready, + `${position} hydration completed`, + ) + } else { + if (position === `straddles`) beginAndWrite() + hydration.resolve() + await atPersistedOracleCheckpoint( + ready, + `${position} hydration completed`, + ) + if (position === `after`) beginAndWrite() + commit() + } - expect(options.persistence.coordinator).toBeInstanceOf( - SingleProcessCoordinator, - ) - }) + const settlement = observeSettlement(receipt!) + const checkpointId = `checkpoint-${position}` + const checkpointRow = { id: checkpointId, title: checkpointId } + ledger.push({ type: `begin`, transactionId: checkpointId }) + remoteBegin?.() + ledger.push({ + type: `write`, + transactionId: checkpointId, + row: checkpointRow, + }) + remoteWrite?.({ type: `insert`, value: checkpointRow }) + ledger.push({ type: `commit`, transactionId: checkpointId }) + const checkpointReceipt = remoteCommit?.() + expect(checkpointReceipt).toBeInstanceOf(Promise) + await atPersistedOracleCheckpoint( + Promise.resolve(checkpointReceipt), + `${position} later durable receipt`, + ) - it(`resolves persistence per collection and forwards schemaVersion`, () => { - const baseAdapter = createNoopAdapter() - const syncAdapter = createNoopAdapter() - const localAdapter = createNoopAdapter() - const resolverCalls: Array<{ - collectionId: string - mode: `sync-present` | `sync-absent` - schemaVersion?: number - }> = [] + // A settled later durable receipt is the exact cut: FIFO forbids it + // from overtaking an earlier commit opened in the old phase. + expect(settlement.read()).toEqual({ status: `fulfilled` }) + const expected = foldDurabilityLedger(ledger) + expect(persistedTransactionOrder).toEqual(expected.commitOrder) + expect(stripVirtualProps(collection.get(`shared`))).toEqual(row) + expect(adapter.rows.get(`shared`)).toEqual(row) + expect(adapter.rows.get(checkpointId)).toEqual(checkpointRow) - const persistence: PersistedCollectionPersistence = { - adapter: baseAdapter, - resolvePersistenceForCollection: (options) => { - resolverCalls.push(options) - return { - adapter: options.mode === `sync-present` ? syncAdapter : localAdapter, + await collection.cleanup() + adapter.loadSubset = () => + Promise.resolve( + Array.from(adapter.rows, ([key, value]) => ({ key, value })), + ) + const reopened = createCollection( + persistedCollectionOptions({ + id: `hydration-boundary-${position}`, + getKey: (item) => item.id, + sync: { sync: ({ markReady }) => markReady() }, + persistence: { adapter }, + }), + ) + let hasReopenFailure = false + try { + await atPersistedOracleCheckpoint( + reopened.stateWhenReady(), + `${position} reopened hydration completed`, + ) + expect(stripVirtualProps(reopened.get(`shared`))).toEqual( + expected.committedRows.get(`shared`), + ) + } catch (error) { + hasReopenFailure = true + throw error + } finally { + await cleanupPersistedOracle( + [() => reopened.cleanup()], + hasReopenFailure, + ) } - }, + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + hydration.resolve() + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }, + ) + + it(`does not let a later same-key transaction overtake a hydration-straddling commit`, async () => { + const hydrationEntered = createEventGate() + const hydration = createEventGate() + const adapter = createRecordingAdapter() + adapter.loadSubset = async () => { + hydrationEntered.resolve() + await hydration.promise + return [] + } + const persistedTransactionOrder: Array = [] + const applyCommittedTx = adapter.applyCommittedTx.bind(adapter) + adapter.applyCommittedTx = async (...args) => { + const row = args[1].mutations.find( + (mutation) => mutation.type !== `delete`, + )?.value as Todo | undefined + if (row) persistedTransactionOrder.push(row.title) + await applyCommittedTx(...args) } - const syncOptions = persistedCollectionOptions({ - id: `sync-collection`, - getKey: (item) => item.id, - sync: { - sync: ({ markReady }) => { - markReady() + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: (() => true | Promise) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `hydration-straddle-sibling-fifo`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit + markReady() + }, }, - }, - schemaVersion: 7, - persistence, - }) - expect(syncOptions.persistence.adapter).toBe(syncAdapter) + persistence: { adapter }, + }), + ) + const ledger: Array = [] + let firstReceipt: Promise | undefined + let secondReceipt: Promise | undefined + let hasPrimaryFailure = false + + const sourceTransaction = (transactionId: string, title: string) => { + const row = { id: `shared`, title } + ledger.push({ type: `begin`, transactionId }) + remoteBegin?.() + ledger.push({ type: `write`, transactionId, row }) + remoteWrite?.({ type: `insert`, value: row }) + ledger.push({ type: `commit`, transactionId }) + return Promise.resolve(remoteCommit?.()).then(() => undefined) + } - const localOptions = persistedCollectionOptions({ - id: `local-collection`, - getKey: (item) => item.id, - schemaVersion: 3, - persistence, - }) - expect(localOptions.persistence.adapter).toBe(localAdapter) + try { + const ready = collection.stateWhenReady() + await atPersistedOracleCheckpoint( + hydrationEntered.promise, + `FIFO hydration entered`, + ) - expect(resolverCalls).toEqual([ - { - collectionId: `sync-collection`, - mode: `sync-present`, - schemaVersion: 7, - }, - { - collectionId: `local-collection`, - mode: `sync-absent`, - schemaVersion: 3, - }, - ]) - }) + const firstRow = { id: `shared`, title: `first` } + ledger.push({ type: `begin`, transactionId: `first` }) + remoteBegin?.() + ledger.push({ type: `write`, transactionId: `first`, row: firstRow }) + remoteWrite?.({ type: `insert`, value: firstRow }) + hydration.resolve() + await atPersistedOracleCheckpoint(ready, `FIFO hydration completed`) + ledger.push({ type: `commit`, transactionId: `first` }) + firstReceipt = Promise.resolve(remoteCommit?.()).then(() => undefined) + const firstSettlement = observeSettlement(firstReceipt) + + secondReceipt = sourceTransaction(`second`, `second`) + const secondSettlement = observeSettlement(secondReceipt) + await atPersistedOracleCheckpoint( + secondReceipt, + `later same-key receipt settled`, + ) - it(`throws for invalid coordinator implementations`, () => { - const invalidCoordinator = { - getNodeId: () => `node-1`, - subscribe: () => () => {}, - publish: () => {}, - isLeader: () => true, - ensureLeadership: async () => {}, - // requestEnsurePersistedIndex is intentionally missing - } as unknown as PersistedCollectionCoordinator + const expected = foldDurabilityLedger(ledger) + expect(persistedTransactionOrder).toEqual(expected.commitOrder) + expect(firstSettlement.read()).toEqual({ status: `fulfilled` }) + expect(secondSettlement.read()).toEqual({ status: `fulfilled` }) + expect(adapter.rows.get(`shared`)).toEqual( + expected.committedRows.get(`shared`), + ) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + hydration.resolve() + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }) - expect(() => - persistedCollectionOptions({ - id: `invalid-coordinator`, - getKey: (item) => item.id, - persistence: { - adapter: createNoopAdapter(), - coordinator: invalidCoordinator, + it(`settles an aborted hydration-straddling commit without applying it`, async () => { + const hydrationEntered = createEventGate() + const hydration = createEventGate() + const adapter = createRecordingAdapter() + adapter.loadSubset = async () => { + hydrationEntered.resolve() + await hydration.promise + return [] + } + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: + | ((signal?: AbortSignal) => true | Promise) + | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `aborted-hydration-straddle`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit + markReady() + }, }, + persistence: { adapter }, }), - ).toThrow(InvalidPersistedCollectionCoordinatorError) - }) + ) + const ledger: Array = [] + const controller = new AbortController() + let receipt: Promise | undefined + let hasPrimaryFailure = false - it(`preserves valid sync config in sync-present mode`, async () => { - const adapter = createRecordingAdapter() - const sync: SyncConfig = { - sync: ({ begin, write, commit, markReady }) => { - begin() - write({ - type: `insert`, - value: { - id: `remote-1`, - title: `From remote`, - }, - }) - commit() - markReady() - }, + try { + const ready = collection.stateWhenReady() + await atPersistedOracleCheckpoint( + hydrationEntered.promise, + `abort hydration entered`, + ) + ledger.push({ type: `begin`, transactionId: `aborted` }) + remoteBegin?.() + const row = { id: `aborted`, title: `must not apply` } + ledger.push({ type: `write`, transactionId: `aborted`, row }) + remoteWrite?.({ type: `insert`, value: row }) + controller.abort() + ledger.push({ type: `abort`, transactionId: `aborted` }) + hydration.resolve() + await atPersistedOracleCheckpoint(ready, `abort hydration completed`) + receipt = Promise.resolve(remoteCommit?.(controller.signal)).then( + () => undefined, + ) + await expect( + atPersistedOracleCheckpoint(receipt, `aborted receipt settled`), + ).rejects.toMatchObject({ name: `AbortError` }) + expect(foldDurabilityLedger(ledger).committedRows.size).toBe(0) + expect(collection.get(`aborted`)).toBeUndefined() + expect(adapter.rows.get(`aborted`)).toBeUndefined() + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + hydration.resolve() + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) } + }) - const options = persistedCollectionOptions({ - id: `sync-present`, - getKey: (item: Todo) => item.id, - sync, - persistence: { - adapter, - }, - }) + it(`rejects every buffered receipt with the exact persisted hydration failure`, async () => { + await runRejectedHydrationBufferWitness(`rejected-hydration-buffer-fixed`, [ + { id: `first`, title: `first buffered commit` }, + { id: `second`, title: `second buffered commit` }, + ]) + }) - const collection = createCollection(options) - await collection.stateWhenReady() - await flushAsyncWork() + it(`rejects a hydration-straddling transaction committed after hydration fails`, async () => { + await runRejectedHydrationBufferWitness( + `rejected-hydration-late-commit-fixed`, + [{ id: `late`, title: `must not cross a failed baseline` }], + { commitAfterHydrationFailure: true }, + ) + }) - expect(stripVirtualProps(collection.get(`remote-1`))).toEqual({ - id: `remote-1`, - title: `From remote`, + it.each([ + { + schedule: `one late commit`, + rows: [{ id: `one-late`, title: `` }], + }, + { + schedule: `two sibling late commits`, + rows: [ + { id: `late-a`, title: `a` }, + { id: `late-b`, title: `b` }, + ], + }, + { + schedule: `same-key late replacements`, + rows: [ + { id: `late-shared`, title: `old` }, + { id: `late-shared`, title: `new` }, + ], + }, + ])( + `rejects generated late commits after hydration failure: $schedule`, + async ({ schedule, rows }) => { + await runRejectedHydrationBufferWitness( + `rejected-hydration-late-${schedule.replaceAll(` `, `-`)}`, + rows, + { commitAfterHydrationFailure: true }, + ) + }, + ) + + it.each([ + { + schedule: `one buffered commit`, + rows: [{ id: `one`, title: `one` }], + }, + { + schedule: `three sibling commits`, + rows: [ + { id: `a`, title: `a` }, + { id: `b`, title: `b` }, + { id: `c`, title: `c` }, + ], + }, + { + schedule: `same-key replacement commits`, + rows: [ + { id: `shared`, title: `old` }, + { id: `shared`, title: `new` }, + ], + }, + ])( + `rejects all receipts and adopts no rows after hydration failure: $schedule`, + async ({ schedule, rows }) => { + await runRejectedHydrationBufferWitness( + `rejected-hydration-${schedule.replaceAll(` `, `-`)}`, + rows, + ) + }, + ) + + it(`append-only transaction judgment rejects omission and reordering`, () => { + const events: Array = [ + { type: `begin`, transactionId: `first` }, + { + type: `write`, + transactionId: `first`, + row: { id: `shared`, title: `first` }, + }, + { type: `commit`, transactionId: `first` }, + { type: `begin`, transactionId: `second` }, + { + type: `write`, + transactionId: `second`, + row: { id: `shared`, title: `second` }, + }, + { type: `commit`, transactionId: `second` }, + ] + const expected = foldDurabilityLedger(events) + + expect(expected.commitOrder).toEqual([`first`, `second`]) + expect(expected.commitOrder).not.toEqual([`second`]) + expect(expected.commitOrder).not.toEqual([`second`, `first`]) + expect(expected.committedRows.get(`shared`)).toEqual({ + id: `shared`, + title: `second`, }) - expect(adapter.applyCommittedTxCalls).toHaveLength(1) - expect(adapter.applyCommittedTxCalls[0]?.tx.mutations[0]?.type).toBe( - `update`, - ) }) - it(`does not apply or persist a wrapped sync transaction committed with an aborted signal`, async () => { + it(`discards a hydration-buffered transaction aborted before replay`, async () => { const adapter = createRecordingAdapter() + let resolveLoadSubset: (() => void) | undefined + adapter.loadSubset = async () => { + await new Promise((resolve) => { + resolveLoadSubset = resolve + }) + return [] + } let remoteBegin: (() => void) | undefined let remoteWrite: | ((message: { type: `insert`; value: Todo }) => void) @@ -850,7 +6610,7 @@ describe(`persistedCollectionOptions`, () => { | undefined const collection = createCollection( persistedCollectionOptions({ - id: `sync-present-aborted-commit`, + id: `sync-present-aborted-hydration-queue`, getKey: (item) => item.id, sync: { sync: ({ begin, write, commit, markReady }) => { @@ -868,38 +6628,190 @@ describe(`persistedCollectionOptions`, () => { ) try { - await collection.stateWhenReady() + const ready = collection.stateWhenReady() + for (let attempt = 0; attempt < 20 && !resolveLoadSubset; attempt++) { + await flushAsyncWork() + } const abortController = new AbortController() - abortController.abort() remoteBegin?.() remoteWrite?.({ type: `insert`, - value: { id: `aborted`, title: `Must not publish` }, + value: { id: `aborted`, title: `Must not replay` }, }) - await expect( - remoteCommit?.(abortController.signal), - ).rejects.toMatchObject({ name: `AbortError` }) + const applied = remoteCommit?.(abortController.signal) + abortController.abort() + resolveLoadSubset?.() + await ready + if (applied !== true) { + await expect(applied).rejects.toMatchObject({ name: `AbortError` }) + } await flushAsyncWork() expect(collection.get(`aborted`)).toBeUndefined() expect(adapter.applyCommittedTxCalls).toHaveLength(0) } finally { + resolveLoadSubset?.() await collection.cleanup() } }) - it(`persists a wrapped sync transaction when abort follows application`, async () => { + it(`continues an independent hydration queue after an in-flight sibling aborts`, async () => { + const adapter = createRecordingAdapter() + const hydrationEntered = createEventGate() + const hydration = createEventGate() + adapter.loadSubset = async () => { + hydrationEntered.resolve() + await hydration.promise + return [] + } + let sourceParams!: TodoSyncParams + const collection = createCollection( + persistedCollectionOptions({ + id: `independent-hydration-abort`, + getKey: (row) => row.id, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() + }, + }, + persistence: { adapter }, + }), + ) + const localPersistence = createEventGate() + const localTransaction = createTransaction({ + mutationFn: () => localPersistence.promise, + }) + const aborted = new AbortController() + let abortedReceipt: Promise | undefined + let independentReceipt: Promise | undefined + const ready = collection.stateWhenReady() + void ready.catch(() => undefined) + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + hydrationEntered.promise, + `independent abort hydration entered`, + ) + localTransaction.mutate(() => { + collection.insert({ id: `local-gate`, title: `local pending` }) + }) + expect(localTransaction.state).toBe(`persisting`) + + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `aborted`, title: `must not publish` }, + }) + abortedReceipt = Promise.resolve( + sourceParams.commit(aborted.signal), + ).then(() => undefined) + void abortedReceipt.catch(() => undefined) + + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `independent`, title: `must survive sibling abort` }, + }) + independentReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + void independentReceipt.catch(() => undefined) + + hydration.resolve() + await vi.waitFor(() => + expect( + collection._state.pendingSyncedTransactions.some( + (transaction) => transaction.committed, + ), + ).toBe(true), + ) + aborted.abort() + await expect( + atPersistedOracleCheckpoint( + abortedReceipt, + `in-flight hydration sibling aborted`, + ), + ).rejects.toMatchObject({ name: `AbortError` }) + + const independentSettlement = observeSettlement(independentReceipt) + await Promise.resolve() + expect({ + independent: independentSettlement.read(), + status: collection.status, + publicError: collection._lifecycle.getSyncError(), + }).toEqual({ + independent: { status: `pending` }, + status: `loading`, + publicError: undefined, + }) + + localPersistence.resolve() + await atPersistedOracleCheckpoint( + localTransaction.isPersisted.promise, + `independent abort local gate settled`, + ) + await atPersistedOracleCheckpoint( + independentReceipt, + `independent hydration sibling applied`, + ) + await atPersistedOracleCheckpoint(ready, `independent abort ready`) + expect({ + status: collection.status, + independent: stripVirtualProps(collection.get(`independent`)), + durable: adapter.rows.get(`independent`), + }).toEqual({ + status: `ready`, + independent: { + id: `independent`, + title: `must survive sibling abort`, + }, + durable: { + id: `independent`, + title: `must survive sibling abort`, + }, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + hydration.resolve() + aborted.abort() + localPersistence.resolve() + await localTransaction.isPersisted.promise.catch(() => undefined) + await cleanupPersistedOracle( + [ + () => abortedReceipt?.catch(() => undefined), + () => independentReceipt?.catch(() => undefined), + () => ready.catch(() => undefined), + () => collection.cleanup(), + ], + hasPrimaryFailure, + ) + } + }) + + it(`rejects every hydration-buffered receipt when replay fails`, async () => { const adapter = createRecordingAdapter() + let resolveLoadSubset: (() => void) | undefined + adapter.loadSubset = async () => { + await new Promise((resolve) => { + resolveLoadSubset = resolve + }) + return [] + } + + const replayError = new Error(`replay application failed`) let remoteBegin: (() => void) | undefined let remoteWrite: | ((message: { type: `insert`; value: Todo }) => void) | undefined - let remoteCommit: - | ((signal?: AbortSignal) => true | Promise) - | undefined + let remoteCommit: (() => true | Promise) | undefined + const collection = createCollection( persistedCollectionOptions({ - id: `sync-present-abort-after-application`, + id: `sync-present-replay-failure-receipt`, getKey: (item) => item.id, sync: { sync: ({ begin, write, commit, markReady }) => { @@ -910,534 +6822,1230 @@ describe(`persistedCollectionOptions`, () => { }) => void remoteCommit = commit markReady() + return {} + }, + }, + persistence: { + adapter, + }, + }), + ) + + const readyOutcome = collection.stateWhenReady().then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + for (let attempt = 0; attempt < 20 && !resolveLoadSubset; attempt++) { + await flushAsyncWork() + } + + remoteBegin?.() + remoteWrite?.({ + type: `insert`, + value: { id: `during-hydrate`, title: `During hydrate` }, + }) + const failingReceipt = remoteCommit?.() + remoteBegin?.() + remoteWrite?.({ + type: `insert`, + value: { id: `sibling`, title: `Sibling` }, + }) + const siblingReceipt = remoteCommit?.() + expect(failingReceipt).toBeInstanceOf(Promise) + expect(siblingReceipt).toBeInstanceOf(Promise) + const setSyncedRow = collection._state.syncedData.set.bind( + collection._state.syncedData, + ) + vi.spyOn(collection._state.syncedData, `set`).mockImplementation( + (key, value) => { + if (key === `during-hydrate`) throw replayError + return setSyncedRow(key, value) + }, + ) + const failingExpectation = expect( + Promise.resolve(failingReceipt), + ).rejects.toBe(replayError) + const siblingExpectation = expect( + Promise.resolve(siblingReceipt), + ).rejects.toBe(replayError) + + resolveLoadSubset?.() + const ready = await atPersistedOracleCheckpoint( + readyOutcome, + `replay-failed hydration readiness settled`, + ) + await failingExpectation + await siblingExpectation + remoteBegin?.() + remoteWrite?.({ + type: `insert`, + value: { id: `after-replay-failure`, title: `must not be admitted` }, + }) + const lateOutcome = await atPersistedOracleCheckpoint( + Promise.resolve(remoteCommit?.()).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `post-replay-failure source receipt`, + ) + + expect({ + readyStatus: ready.status, + readyExact: ready.status === `rejected` && ready.reason === replayError, + lateStatus: lateOutcome.status, + lateExact: + lateOutcome.status === `rejected` && lateOutcome.reason === replayError, + status: collection.status, + exactPublicError: collection._lifecycle.getSyncError() === replayError, + visibleLate: collection.get(`after-replay-failure`), + durableLate: adapter.rows.get(`after-replay-failure`), + applyCalls: adapter.applyCommittedTxCalls.length, + }).toEqual({ + readyStatus: `rejected`, + readyExact: true, + lateStatus: `rejected`, + lateExact: true, + status: `error`, + exactPublicError: true, + visibleLate: undefined, + durableLate: undefined, + applyCalls: 0, + }) + + await collection.cleanup() + }) + + it.each([`stream-position`, `collection-metadata`, `row-apply`] as const)( + `fail-stops startup when the %s phase fails`, + async (phase) => { + const phaseError = new Error(`${phase} failed exactly`) + const adapter = createRecordingAdapter( + phase === `row-apply` + ? [{ id: `row-apply-failure`, title: `must not apply` }] + : [], + ) + if (phase === `stream-position`) { + adapter.getStreamPosition = () => Promise.reject(phaseError) + } else if (phase === `collection-metadata`) { + adapter.loadCollectionMetadata = () => Promise.reject(phaseError) + } + const collection = createCollection( + persistedCollectionOptions({ + id: `terminal-startup-${phase}`, + getKey: (row) => { + if (phase === `row-apply` && row.id === `row-apply-failure`) { + throw phaseError + } + return row.id + }, + persistence: { adapter }, + }), + ) + let hasPrimaryFailure = false + + try { + const startupOutcome = await atPersistedOracleCheckpoint( + collection.stateWhenReady().then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `${phase} startup boundary settled`, + ) + + expect({ + startupStatus: startupOutcome.status, + startupExact: + startupOutcome.status === `rejected` && + startupOutcome.reason === phaseError, + status: collection.status, + exactPublicError: collection._lifecycle.getSyncError() === phaseError, + visibleRows: Array.from(collection.values()).map(stripVirtualProps), + durableRows: Array.from(adapter.rows.values()), + applyCalls: adapter.applyCommittedTxCalls.length, + }).toEqual({ + startupStatus: `rejected`, + startupExact: true, + status: `error`, + exactPublicError: true, + visibleRows: [], + durableRows: + phase === `row-apply` + ? [{ id: `row-apply-failure`, title: `must not apply` }] + : [], + applyCalls: 0, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }, + ) + + it(`fail-stops a coordinator transaction flushed after hydration`, async () => { + const adapter = createRecordingAdapter() + const hydrationEntered = createEventGate() + const hydration = createEventGate() + adapter.loadSubset = async () => { + hydrationEntered.resolve() + await hydration.promise + return [] + } + const flushError = new Error(`coordinator hydration flush failed exactly`) + const coordinator = createFailStopCoordinatorHarness( + `terminal-coordinator-hydration-flush`, + ) + let sourceParams!: TodoSyncParams + const collection = createCollection( + persistedCollectionOptions({ + id: `terminal-coordinator-hydration-flush`, + getKey: (row) => { + if (row.id === `coordinator-bad`) throw flushError + return row.id + }, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() }, }, - persistence: { adapter }, + persistence: { adapter, coordinator }, }), ) - let releaseMutation!: () => void - const mutationGate = new Promise((resolve) => { - releaseMutation = resolve - }) - const transaction = createTransaction({ - mutationFn: () => mutationGate, - }) + const readyOutcome = collection.stateWhenReady().then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + let hasPrimaryFailure = false try { - await collection.stateWhenReady() - transaction.mutate(() => { - collection.insert({ id: `local`, title: `Optimistic gate` }) - }) - - const abortController = new AbortController() - let appliedPublications = 0 - const subscription = collection.subscribeChanges((changes) => { - if (changes.some((change) => change.key === `remote`)) { - appliedPublications++ - abortController.abort() - } + await atPersistedOracleCheckpoint( + hydrationEntered.promise, + `coordinator flush hydration entered`, + ) + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 1, + txId: `coordinator-flush-failure`, + latestRowVersion: 1, + requiresFullReload: false, + changedRows: [ + { + key: `coordinator-bad`, + value: { id: `coordinator-bad`, title: `must not publish` }, + }, + ], + deletedKeys: [], }) - remoteBegin?.() - remoteWrite?.({ + hydration.resolve() + const ready = await atPersistedOracleCheckpoint( + readyOutcome, + `coordinator flush readiness settled`, + ) + sourceParams.begin() + sourceParams.write({ type: `insert`, - value: { id: `remote`, title: `Already visible` }, + value: { id: `late`, title: `must not be admitted` }, }) - const receipt = remoteCommit?.(abortController.signal) - expect(receipt).toBeInstanceOf(Promise) - - releaseMutation() - await transaction.isPersisted.promise - await receipt - subscription.unsubscribe() + const late = await atPersistedOracleCheckpoint( + Promise.resolve(sourceParams.commit()).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `coordinator flush late receipt`, + ) - expect(appliedPublications).toBe(1) - expect(abortController.signal.aborted).toBe(true) - expect(stripVirtualProps(collection.get(`remote`))).toEqual({ - id: `remote`, - title: `Already visible`, + expect({ + readyStatus: ready.status, + readyExact: ready.status === `rejected` && ready.reason === flushError, + lateStatus: late.status, + lateExact: late.status === `rejected` && late.reason === flushError, + status: collection.status, + exactPublicError: collection._lifecycle.getSyncError() === flushError, + visibleBad: collection.get(`coordinator-bad`), + visibleLate: collection.get(`late`), + durableLate: adapter.rows.get(`late`), + applyCalls: adapter.applyCommittedTxCalls.length, + }).toEqual({ + readyStatus: `rejected`, + readyExact: true, + lateStatus: `rejected`, + lateExact: true, + status: `error`, + exactPublicError: true, + visibleBad: undefined, + visibleLate: undefined, + durableLate: undefined, + applyCalls: 0, }) - expect(adapter.applyCommittedTxCalls).toHaveLength(1) + } catch (error) { + hasPrimaryFailure = true + throw error } finally { - releaseMutation() - await transaction.isPersisted.promise.catch(() => undefined) - await collection.cleanup() + hydration.resolve() + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) } }) - it(`rejects a wrapped sync receipt when persistence fails`, async () => { + it(`applies hydration-queued sync work after an unrelated local persistence rollback`, async () => { const adapter = createRecordingAdapter() - const persistenceError = new Error(`persistence failed`) - adapter.applyCommittedTx = () => Promise.reject(persistenceError) - let remoteBegin: (() => void) | undefined - let remoteWrite: - | ((message: { type: `insert`; value: Todo }) => void) - | undefined - let remoteCommit: (() => true | Promise) | undefined + const hydrationEntered = createEventGate() + const hydration = createEventGate() + adapter.loadSubset = async () => { + hydrationEntered.resolve() + await hydration.promise + return [] + } + const appliedError = new Error(`hydration applied receipt failed exactly`) + const localPersistence = createEventGate() + let sourceParams!: TodoSyncParams const collection = createCollection( persistedCollectionOptions({ - id: `sync-present-persistence-error`, - getKey: (item) => item.id, + id: `terminal-hydration-applied-receipt`, + getKey: (row) => row.id, sync: { - sync: ({ begin, write, commit, markReady }) => { - remoteBegin = begin - remoteWrite = write as (message: { - type: `insert` - value: Todo - }) => void - remoteCommit = commit - markReady() + sync: (params) => { + sourceParams = params + params.markReady() }, }, persistence: { adapter }, }), ) + const localTransaction = createTransaction({ + mutationFn: () => localPersistence.promise, + }) + const readyOutcome = collection.stateWhenReady().then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + let remoteReceipt: true | Promise | undefined + let hasPrimaryFailure = false try { - await collection.stateWhenReady() - remoteBegin?.() - remoteWrite?.({ + await atPersistedOracleCheckpoint( + hydrationEntered.promise, + `applied-receipt hydration entered`, + ) + localTransaction.mutate(() => { + collection.insert({ id: `local-gate`, title: `local pending` }) + }) + sourceParams.begin() + sourceParams.write({ type: `insert`, - value: { id: `failed`, title: `Not durable` }, + value: { + id: `remote-applied`, + title: `applies after local rollback`, + }, }) + remoteReceipt = sourceParams.commit() + expect(remoteReceipt).toBeInstanceOf(Promise) + hydration.resolve() + await flushAsyncWork() + localPersistence.reject(appliedError) - await expect(Promise.resolve(remoteCommit?.())).rejects.toBe( - persistenceError, + const ready = await atPersistedOracleCheckpoint( + readyOutcome, + `applied-receipt readiness settled`, + ) + const remote = await atPersistedOracleCheckpoint( + Promise.resolve(remoteReceipt).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `hydration-applied remote receipt settled`, ) + + expect({ + readyStatus: ready.status, + remoteStatus: remote.status, + status: collection.status, + publicError: collection._lifecycle.getSyncError(), + durableRemote: adapter.rows.get(`remote-applied`), + applyCalls: adapter.applyCommittedTxCalls.length, + }).toEqual({ + readyStatus: `fulfilled`, + remoteStatus: `fulfilled`, + status: `ready`, + publicError: undefined, + durableRemote: { + id: `remote-applied`, + title: `applies after local rollback`, + }, + applyCalls: 1, + }) + } catch (error) { + hasPrimaryFailure = true + throw error } finally { - await collection.cleanup() + hydration.resolve() + localPersistence.reject(appliedError) + await localTransaction.isPersisted.promise.catch(() => undefined) + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) } }) - it(`preserves row metadata set before a metadata-less insert in the same sync transaction`, async () => { - const adapter = createRecordingAdapter() - const ownership = { queryCollection: { owners: [`gc:q1`] } } - const sync: SyncConfig = { - sync: ({ begin, write, commit, markReady, metadata }) => { - begin() - metadata?.row.set(`remote-1`, ownership) - write({ - type: `insert`, - value: { - id: `remote-1`, - title: `From remote`, + it.each([ + { + label: `immediate source transaction`, + options: { immediate: true }, + expectedVisibleBeforeLocalSettlement: true, + expectedApplyCallsBeforeLocalSettlement: 2, + }, + { + label: `non-immediate control`, + options: undefined, + expectedVisibleBeforeLocalSettlement: false, + expectedApplyCallsBeforeLocalSettlement: 1, + }, + ] as const)( + `preserves $label admission through the persisted wrapper`, + async ({ + options, + expectedVisibleBeforeLocalSettlement, + expectedApplyCallsBeforeLocalSettlement, + }) => { + const adapter = createRecordingAdapter() + let sourceParams!: TodoSyncParams + const collection = createCollection( + persistedCollectionOptions({ + id: `begin-options-${options?.immediate ? `immediate` : `normal`}`, + getKey: (row) => row.id, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() + }, }, - }) - commit() - markReady() - }, - } - - const collection = createCollection( - persistedCollectionOptions({ - id: `sync-present`, - getKey: (item: Todo) => item.id, - sync, - persistence: { - adapter, - }, - }), - ) + persistence: { adapter }, + }), + ) + const localPersistence = createEventGate() + const localTransaction = createTransaction({ + autoCommit: false, + mutationFn: () => localPersistence.promise, + }) + let localCommit: Promise | undefined + let remoteReceipt: true | Promise | undefined + let hasPrimaryFailure = false - await collection.stateWhenReady() - await flushAsyncWork() + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `begin-options collection ready`, + ) + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `seed`, title: `baseline` }, + }) + await atPersistedOracleCheckpoint( + Promise.resolve(sourceParams.commit()), + `begin-options seed persisted`, + ) - expect(adapter.rowMetadata.get(`remote-1`)).toEqual(ownership) - expect(collection._state.syncedMetadata.get(`remote-1`)).toEqual(ownership) - }) + localTransaction.mutate(() => { + collection.update(`seed`, (draft) => { + draft.title = `local pending` + }) + }) + localCommit = localTransaction.commit() + void localCommit.catch(() => undefined) + expect(localTransaction.state).toBe(`persisting`) - it(`resets stale row metadata for a metadata-less insert with no queued metadata`, async () => { - const adapter = createRecordingAdapter() - adapter.rowMetadata.set(`remote-1`, { stale: true }) - const sync: SyncConfig = { - sync: ({ begin, write, commit, markReady }) => { - begin() - write({ + sourceParams.begin(options) + sourceParams.write({ type: `insert`, - value: { - id: `remote-1`, - title: `From remote`, - }, + value: { id: `remote`, title: `source commit` }, }) - commit() - markReady() - }, - } + remoteReceipt = sourceParams.commit() + expect(remoteReceipt).toBeInstanceOf(Promise) + + expect({ + localState: localTransaction.state, + remoteVisible: collection.has(`remote`), + applyCalls: adapter.applyCommittedTxCalls.length, + }).toEqual({ + localState: `persisting`, + remoteVisible: expectedVisibleBeforeLocalSettlement, + applyCalls: expectedApplyCallsBeforeLocalSettlement, + }) + + if (options?.immediate) { + await atPersistedOracleCheckpoint( + Promise.resolve(remoteReceipt), + `immediate source receipt before local settlement`, + ) + expect(localTransaction.state).toBe(`persisting`) + } else { + const settlement = observeSettlement( + Promise.resolve(remoteReceipt).then(() => undefined), + ) + await Promise.resolve() + expect(settlement.read()).toEqual({ status: `pending` }) + } + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + localPersistence.resolve() + await cleanupPersistedOracle( + [ + () => localCommit, + () => + remoteReceipt === true + ? undefined + : Promise.resolve(remoteReceipt), + () => collection.cleanup(), + ], + hasPrimaryFailure, + ) + } + }, + ) + it(`lets an immediate source dependency release a persisting user transaction`, async () => { + const adapter = createRecordingAdapter() + let sourceParams!: TodoSyncParams const collection = createCollection( persistedCollectionOptions({ - id: `sync-present`, - getKey: (item: Todo) => item.id, - sync, - persistence: { - adapter, + id: `immediate-source-dependency`, + getKey: (row) => row.id, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() + }, }, + persistence: { adapter }, }), ) - - await collection.stateWhenReady() - await flushAsyncWork() - - expect(adapter.rowMetadata.has(`remote-1`)).toBe(false) - }) - - it(`uses a stable generated collection id in sync-present mode when id is omitted`, async () => { - const adapter = createRecordingAdapter() - const options = persistedCollectionOptions({ - getKey: (item) => item.id, - sync: { - sync: ({ markReady }) => { - markReady() - }, - }, - persistence: { - adapter, + const mutationEntered = createEventGate() + const startDependency = createEventGate() + const dependencySubmitted = createEventGate() + let dependencyReceipt: true | Promise | undefined + const localTransaction = createTransaction({ + mutationFn: async () => { + mutationEntered.resolve() + await startDependency.promise + sourceParams.begin({ immediate: true }) + sourceParams.write({ + type: `insert`, + value: { id: `dependency`, title: `releases user persistence` }, + }) + dependencyReceipt = sourceParams.commit() + dependencySubmitted.resolve() + await dependencyReceipt }, }) + const normalAbort = new AbortController() + let normalReceipt: Promise | undefined + let hasPrimaryFailure = false - expect(options.id).toBeDefined() + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `immediate dependency collection ready`, + ) + localTransaction.mutate(() => { + collection.insert({ id: `local`, title: `user persistence pending` }) + }) + await atPersistedOracleCheckpoint( + mutationEntered.promise, + `user persistence entered`, + ) + expect(localTransaction.state).toBe(`persisting`) - const collection = createCollection(options) - await collection.preload() - await flushAsyncWork() + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `parked`, title: `waits for user persistence` }, + }) + normalReceipt = Promise.resolve( + sourceParams.commit(normalAbort.signal), + ).then(() => undefined) + void normalReceipt.catch(() => undefined) + expect( + collection._state.pendingSyncedTransactions.some( + (transaction) => transaction.committed, + ), + ).toBe(true) - expect(collection.id).toBe(options.id) - expect(adapter.loadSubsetCalls[0]?.collectionId).toBe(collection.id) + startDependency.resolve() + await atPersistedOracleCheckpoint( + dependencySubmitted.promise, + `immediate dependency submitted`, + ) + expect({ + localState: localTransaction.state, + dependencyVisible: collection.has(`dependency`), + }).toEqual({ + localState: `persisting`, + dependencyVisible: true, + }) + + await atPersistedOracleCheckpoint( + localTransaction.isPersisted.promise, + `user persistence released by immediate dependency`, + ) + await atPersistedOracleCheckpoint( + normalReceipt, + `parked predecessor applied with immediate dependency`, + ) + if (dependencyReceipt !== true) { + await atPersistedOracleCheckpoint( + Promise.resolve(dependencyReceipt), + `immediate dependency receipt settled`, + ) + } + expect(adapter.rows.get(`dependency`)).toEqual({ + id: `dependency`, + title: `releases user persistence`, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + startDependency.resolve() + normalAbort.abort() + await cleanupPersistedOracle( + [ + () => localTransaction.isPersisted.promise.catch(() => undefined), + () => normalReceipt?.catch(() => undefined), + () => + dependencyReceipt === true + ? undefined + : Promise.resolve(dependencyReceipt).catch(() => undefined), + () => collection.cleanup(), + ], + hasPrimaryFailure, + ) + } }) - it(`keeps hydrated rows ahead of persisted startup rows`, async () => { - const adapter = createRecordingAdapter([ - { id: `1`, title: `Persisted title` }, - ]) - const descriptor = collectionOptions( + it(`preserves source order when an immediate commit overtakes a held predecessor`, async () => { + const adapter = createRecordingAdapter() + const firstPersistenceEntered = createEventGate() + const releaseFirstPersistence = createEventGate() + const applyCommittedTx = adapter.applyCommittedTx.bind(adapter) + const persistedSharedTitles: Array = [] + let applyCalls = 0 + adapter.applyCommittedTx = async (...args) => { + applyCalls++ + for (const mutation of args[1].mutations) { + if (mutation.type !== `delete` && mutation.key === `shared`) { + persistedSharedTitles.push((mutation.value as Todo).title) + } + } + if (applyCalls === 1) { + firstPersistenceEntered.resolve() + await releaseFirstPersistence.promise + } + await applyCommittedTx(...args) + } + let sourceParams!: TodoSyncParams + const collection = createCollection( persistedCollectionOptions({ - id: `hydration-precedence`, - getKey: (item) => item.id, - sync: { - sync: ({ markReady }) => { - markReady() - }, - }, - persistence: { - adapter, - }, - }), - ) - const client = new DbClient() - client.hydrate({ - collections: [ - { - collectionId: descriptor.id, - rows: [ - { - key: `1`, - value: { id: `1`, title: `SSR title` }, - }, - ], + id: `immediate-source-order`, + getKey: (row) => row.id, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() + }, }, - ], - }) + persistence: { adapter }, + }), + ) + let gateReceipt: Promise | undefined + let olderReceipt: Promise | undefined + let newerReceipt: Promise | undefined + let hasPrimaryFailure = false - const collection = client.collection(descriptor) - await collection.stateWhenReady() - await flushAsyncWork() + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `immediate ordering collection ready`, + ) + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `gate`, title: `holds adapter` }, + }) + gateReceipt = Promise.resolve(sourceParams.commit()).then(() => undefined) + void gateReceipt.catch(() => undefined) + await atPersistedOracleCheckpoint( + firstPersistenceEntered.promise, + `immediate ordering predecessor persistence entered`, + ) - expect(collection.get(`1`)).toMatchObject({ - id: `1`, - title: `SSR title`, - }) + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `shared`, title: `older normal` }, + }) + olderReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + void olderReceipt.catch(() => undefined) - await client.cleanup() + sourceParams.begin({ immediate: true }) + sourceParams.write({ + type: `update`, + value: { id: `shared`, title: `newer immediate` }, + }) + newerReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + void newerReceipt.catch(() => undefined) + + expect(stripVirtualProps(collection.get(`shared`))).toEqual({ + id: `shared`, + title: `newer immediate`, + }) + + releaseFirstPersistence.resolve() + await atPersistedOracleCheckpoint(gateReceipt, `ordering gate persisted`) + await atPersistedOracleCheckpoint( + olderReceipt, + `older normal source receipt`, + ) + await atPersistedOracleCheckpoint( + newerReceipt, + `newer immediate source receipt`, + ) + + expect({ + status: collection.status, + publicRow: stripVirtualProps(collection.get(`shared`)), + durableRow: adapter.rows.get(`shared`), + persistedTitles: persistedSharedTitles, + }).toEqual({ + status: `ready`, + publicRow: { id: `shared`, title: `newer immediate` }, + durableRow: { id: `shared`, title: `newer immediate` }, + persistedTitles: [`older normal`, `newer immediate`], + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseFirstPersistence.resolve() + await cleanupPersistedOracle( + [ + () => gateReceipt?.catch(() => undefined), + () => olderReceipt?.catch(() => undefined), + () => newerReceipt?.catch(() => undefined), + () => collection.cleanup(), + ], + hasPrimaryFailure, + ) + } }) - it(`bootstraps and tracks persisted index lifecycle in sync-present mode`, async () => { + it(`fail-stops a same-key partial suffix when its staged predecessor aborts`, async () => { const adapter = createRecordingAdapter() + const firstPersistenceEntered = createEventGate() + const releaseFirstPersistence = createEventGate() + const applyCommittedTx = adapter.applyCommittedTx.bind(adapter) + let applyCalls = 0 + adapter.applyCommittedTx = async (...args) => { + applyCalls++ + if (applyCalls === 1) { + firstPersistenceEntered.resolve() + await releaseFirstPersistence.promise + } + await applyCommittedTx(...args) + } + let sourceParams!: TodoSyncParams const collection = createCollection( persistedCollectionOptions({ - id: `sync-present-indexes`, - getKey: (item) => item.id, - defaultIndexType: BasicIndex, + id: `same-key-abort-dependency`, + getKey: (row) => row.id, sync: { - sync: ({ markReady }) => { - markReady() + rowUpdateMode: `partial`, + sync: (params) => { + sourceParams = params + params.markReady() }, }, - persistence: { - adapter, - }, + persistence: { adapter }, }), ) + const aborted = new AbortController() + let gateReceipt: Promise | undefined + let abortedReceipt: Promise | undefined + let dependentReceipt: Promise | undefined + let hasPrimaryFailure = false - const preSyncIndex = collection.createIndex((row) => row.title, { - name: `pre-sync-title`, - }) - const expectedPreSyncSignature = collection.getIndexMetadata()[0]?.signature - - await collection.preload() - await flushAsyncWork() - - expect(expectedPreSyncSignature).toBeDefined() - expect( - adapter.ensureIndexCalls.some( - (call) => call.signature === expectedPreSyncSignature, - ), - ).toBe(true) - - const runtimeIndex = collection.createIndex((row) => row.id, { - name: `runtime-id`, - }) - await flushAsyncWork() + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `same-key dependency collection ready`, + ) + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `gate`, title: `holds adapter` }, + }) + gateReceipt = Promise.resolve(sourceParams.commit()).then(() => undefined) + void gateReceipt.catch(() => undefined) + await atPersistedOracleCheckpoint( + firstPersistenceEntered.promise, + `same-key dependency persistence entered`, + ) - const runtimeSignature = collection - .getIndexMetadata() - .find((index) => index.indexId === runtimeIndex.id)?.signature - expect(runtimeSignature).toBeDefined() - expect( - adapter.ensureIndexCalls.some( - (call) => call.signature === runtimeSignature, - ), - ).toBe(true) + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { + id: `shared`, + title: `predecessor`, + detail: `required inherited field`, + }, + }) + abortedReceipt = Promise.resolve( + sourceParams.commit(aborted.signal), + ).then(() => undefined) + void abortedReceipt.catch(() => undefined) + + sourceParams.begin() + sourceParams.write({ + type: `update`, + value: { id: `shared`, title: `dependent suffix` }, + }) + dependentReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + void dependentReceipt.catch(() => undefined) - collection.removeIndex(preSyncIndex) - await flushAsyncWork() - expect( - adapter.markIndexRemovedCalls.some( - (call) => call.signature === expectedPreSyncSignature, - ), - ).toBe(true) + aborted.abort() + releaseFirstPersistence.resolve() + await atPersistedOracleCheckpoint( + gateReceipt, + `dependency gate persisted`, + ) + const abortedOutcome = await atPersistedOracleCheckpoint( + abortedReceipt.then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `same-key predecessor aborted`, + ) + const dependentOutcome = await atPersistedOracleCheckpoint( + dependentReceipt.then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `same-key dependent suffix settled`, + ) + const terminalError = + abortedOutcome.status === `rejected` ? abortedOutcome.reason : undefined + + expect({ + abortedStatus: abortedOutcome.status, + abortedName: + terminalError instanceof Error ? terminalError.name : undefined, + dependentStatus: dependentOutcome.status, + dependentExact: + dependentOutcome.status === `rejected` && + dependentOutcome.reason === terminalError, + status: collection.status, + exactPublicError: + collection._lifecycle.getSyncError() === terminalError, + publicRow: collection.get(`shared`), + durableRow: adapter.rows.get(`shared`), + pendingMarkers: collection._state.pendingSyncedTransactions.length, + }).toEqual({ + abortedStatus: `rejected`, + abortedName: `AbortError`, + dependentStatus: `rejected`, + dependentExact: true, + status: `error`, + exactPublicError: true, + publicRow: undefined, + durableRow: undefined, + pendingMarkers: 0, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseFirstPersistence.resolve() + await cleanupPersistedOracle( + [ + () => gateReceipt?.catch(() => undefined), + () => abortedReceipt?.catch(() => undefined), + () => dependentReceipt?.catch(() => undefined), + () => collection.cleanup(), + ], + hasPrimaryFailure, + ) + } }) - it(`queues remote sync writes that arrive during hydration`, async () => { + it(`releases a reentrant internal reservation after a terminal failure`, async () => { const adapter = createRecordingAdapter([ - { - id: `cached-1`, - title: `Cached row`, - }, + { id: `hydrated`, title: `starts reentrant reservation` }, ]) - let resolveLoadSubset: (() => void) | undefined + const hydrationEntered = createEventGate() + const releaseHydration = createEventGate() adapter.loadSubset = async () => { - await new Promise((resolve) => { - resolveLoadSubset = resolve - }) + hydrationEntered.resolve() + await releaseHydration.promise return [ { - key: `cached-1`, - value: { - id: `cached-1`, - title: `Cached row`, - }, + key: `hydrated`, + value: { id: `hydrated`, title: `starts reentrant reservation` }, }, ] } - - let remoteBegin: (() => void) | undefined - let remoteWrite: - | ((message: { type: `insert`; value: Todo }) => void) - | undefined - let remoteCommit: (() => void) | undefined - + let sourceParams!: TodoSyncParams const collection = createCollection( persistedCollectionOptions({ - id: `sync-present`, - getKey: (item) => item.id, + id: `terminal-internal-reservation`, + getKey: (row) => row.id, sync: { - sync: ({ begin, write, commit, markReady }) => { - remoteBegin = begin - remoteWrite = write as (message: { - type: `insert` - value: Todo - }) => void - remoteCommit = commit - markReady() - return {} + sync: (params) => { + sourceParams = params + params.markReady() }, }, - persistence: { - adapter, - }, + persistence: { adapter }, }), ) - - const readyPromise = collection.stateWhenReady() - for (let attempt = 0; attempt < 20 && !resolveLoadSubset; attempt++) { - await flushAsyncWork() - } - - expect(resolveLoadSubset).toBeDefined() - expect(remoteBegin).toBeDefined() - - remoteBegin?.() - remoteWrite?.({ - type: `insert`, - value: { - id: `during-hydrate`, - title: `During hydrate`, - }, + let internalReservationStarted = false + const subscription = collection.subscribeChanges((changes) => { + if ( + !internalReservationStarted && + changes.some((change) => change.key === `hydrated`) + ) { + internalReservationStarted = true + sourceParams.begin() + } }) - remoteCommit?.() + const terminalError = new Error(`terminal reservation failure`) + let externalReceipt: Promise | undefined + let internalReceipt: Promise | undefined + let hasPrimaryFailure = false - resolveLoadSubset?.() - await readyPromise - await flushAsyncWork() + try { + const ready = collection.stateWhenReady() + await atPersistedOracleCheckpoint( + hydrationEntered.promise, + `reentrant reservation hydration entered`, + ) + await vi.waitFor(() => expect(sourceParams).toBeDefined()) + releaseHydration.resolve() + await atPersistedOracleCheckpoint(ready, `reentrant reservation ready`) + expect({ + internalReservationStarted, + pendingMarkers: collection._state.pendingSyncedTransactions.length, + }).toEqual({ + internalReservationStarted: true, + pendingMarkers: 1, + }) - expect(stripVirtualProps(collection.get(`cached-1`))).toEqual({ - id: `cached-1`, - title: `Cached row`, - }) - expect(stripVirtualProps(collection.get(`during-hydrate`))).toEqual({ - id: `during-hydrate`, - title: `During hydrate`, - }) + adapter.applyCommittedTx = () => Promise.reject(terminalError) + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `terminal`, title: `published before failure` }, + }) + externalReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + void externalReceipt.catch(() => undefined) + await expect( + atPersistedOracleCheckpoint( + externalReceipt, + `external durability failure`, + ), + ).rejects.toMatchObject({ + name: `PersistenceDurabilityError`, + cause: terminalError, + }) + + internalReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + void internalReceipt.catch(() => undefined) + await expect( + atPersistedOracleCheckpoint( + internalReceipt, + `terminal internal reservation rejected`, + ), + ).rejects.toMatchObject({ name: `PersistenceDurabilityError` }) + expect({ + status: collection.status, + pendingMarkers: collection._state.pendingSyncedTransactions.length, + }).toEqual({ status: `error`, pendingMarkers: 0 }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseHydration.resolve() + subscription.unsubscribe() + await cleanupPersistedOracle( + [ + () => externalReceipt?.catch(() => undefined), + () => internalReceipt?.catch(() => undefined), + () => collection.cleanup(), + ], + hasPrimaryFailure, + ) + } }) - it(`discards a hydration-buffered transaction aborted before replay`, async () => { + it(`keeps bare sync operations away from another owner's queued marker`, async () => { const adapter = createRecordingAdapter() - let resolveLoadSubset: (() => void) | undefined - adapter.loadSubset = async () => { - await new Promise((resolve) => { - resolveLoadSubset = resolve - }) - return [] + const firstPersistenceEntered = createEventGate() + const releaseFirstPersistence = createEventGate() + const applyCommittedTx = adapter.applyCommittedTx.bind(adapter) + let applyCalls = 0 + adapter.applyCommittedTx = async (...args) => { + applyCalls++ + if (applyCalls === 1) { + firstPersistenceEntered.resolve() + await releaseFirstPersistence.promise + } + await applyCommittedTx(...args) } - let remoteBegin: (() => void) | undefined - let remoteWrite: - | ((message: { type: `insert`; value: Todo }) => void) - | undefined - let remoteCommit: - | ((signal?: AbortSignal) => true | Promise) - | undefined + let sourceParams!: TodoSyncParams const collection = createCollection( persistedCollectionOptions({ - id: `sync-present-aborted-hydration-queue`, - getKey: (item) => item.id, + id: `two-owner-marker-isolation`, + getKey: (row) => row.id, sync: { - sync: ({ begin, write, commit, markReady }) => { - remoteBegin = begin - remoteWrite = write as (message: { - type: `insert` - value: Todo - }) => void - remoteCommit = commit - markReady() + sync: (params) => { + sourceParams = params + params.markReady() }, }, persistence: { adapter }, }), ) + let firstReceipt: Promise | undefined + let queuedReceipt: Promise | undefined + let strayCommitReceipt: Promise | undefined + let hasPrimaryFailure = false try { - const ready = collection.stateWhenReady() - for (let attempt = 0; attempt < 20 && !resolveLoadSubset; attempt++) { - await flushAsyncWork() - } - const abortController = new AbortController() - remoteBegin?.() - remoteWrite?.({ + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `two-owner marker collection ready`, + ) + sourceParams.begin() + sourceParams.write({ type: `insert`, - value: { id: `aborted`, title: `Must not replay` }, + value: { id: `first`, title: `holds adapter` }, }) - const applied = remoteCommit?.(abortController.signal) - abortController.abort() - resolveLoadSubset?.() - await ready - if (applied !== true) { - await expect(applied).rejects.toMatchObject({ name: `AbortError` }) + firstReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + void firstReceipt.catch(() => undefined) + await atPersistedOracleCheckpoint( + firstPersistenceEntered.promise, + `first owner persistence entered`, + ) + + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `queued`, title: `owns queued marker` }, + }) + queuedReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + void queuedReceipt.catch(() => undefined) + + let writeError: unknown + try { + sourceParams.write({ + type: `insert`, + value: { id: `stray`, title: `must not steal marker` }, + }) + } catch (error) { + writeError = error + } + let commitError: unknown + try { + const receipt = sourceParams.commit() + if (receipt !== true) { + strayCommitReceipt = Promise.resolve(receipt).then(() => undefined) + void strayCommitReceipt.catch(() => undefined) + } + } catch (error) { + commitError = error } - await flushAsyncWork() - expect(collection.get(`aborted`)).toBeUndefined() - expect(adapter.applyCommittedTxCalls).toHaveLength(0) - } finally { - resolveLoadSubset?.() - await collection.cleanup() - } - }) + expect({ + writeErrorIsOwned: + writeError instanceof NoPendingSyncTransactionWriteError, + commitErrorIsOwned: + commitError instanceof NoPendingSyncTransactionCommitError, + queuedVisible: collection.has(`queued`), + strayVisible: collection.has(`stray`), + }).toEqual({ + writeErrorIsOwned: true, + commitErrorIsOwned: true, + queuedVisible: false, + strayVisible: false, + }) - it(`rejects every hydration-buffered receipt when replay fails`, async () => { - const adapter = createRecordingAdapter() - let resolveLoadSubset: (() => void) | undefined - adapter.loadSubset = async () => { - await new Promise((resolve) => { - resolveLoadSubset = resolve + releaseFirstPersistence.resolve() + await atPersistedOracleCheckpoint(firstReceipt, `first owner persisted`) + await atPersistedOracleCheckpoint(queuedReceipt, `queued owner persisted`) + expect({ + queued: stripVirtualProps(collection.get(`queued`)), + stray: collection.get(`stray`), + }).toEqual({ + queued: { id: `queued`, title: `owns queued marker` }, + stray: undefined, }) - return [] + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseFirstPersistence.resolve() + await cleanupPersistedOracle( + [ + () => firstReceipt?.catch(() => undefined), + () => queuedReceipt?.catch(() => undefined), + () => strayCommitReceipt?.catch(() => undefined), + () => collection.cleanup(), + ], + hasPrimaryFailure, + ) } + }) - const replayError = new Error(`replay key failed`) - let bufferedRowKeyReads = 0 - let remoteBegin: (() => void) | undefined - let remoteWrite: - | ((message: { type: `insert`; value: Todo }) => void) - | undefined - let remoteCommit: (() => true | Promise) | undefined - + it(`keeps a bare truncate away from another owner's queued marker`, async () => { + const adapter = createRecordingAdapter() + const firstPersistenceEntered = createEventGate() + const releaseFirstPersistence = createEventGate() + const applyCommittedTx = adapter.applyCommittedTx.bind(adapter) + let applyCalls = 0 + adapter.applyCommittedTx = async (...args) => { + applyCalls++ + if (applyCalls === 1) { + firstPersistenceEntered.resolve() + await releaseFirstPersistence.promise + } + await applyCommittedTx(...args) + } + let sourceParams!: TodoSyncParams const collection = createCollection( persistedCollectionOptions({ - id: `sync-present-replay-failure-receipt`, - getKey: (item) => { - if (item.id === `during-hydrate`) { - bufferedRowKeyReads++ - if (bufferedRowKeyReads === 2) { - throw replayError - } - } - return item.id - }, + id: `bare-truncate-owner-isolation`, + getKey: (row) => row.id, sync: { - sync: ({ begin, write, commit, markReady }) => { - remoteBegin = begin - remoteWrite = write as (message: { - type: `insert` - value: Todo - }) => void - remoteCommit = commit - markReady() - return {} + sync: (params) => { + sourceParams = params + params.markReady() }, }, - persistence: { - adapter, - }, + persistence: { adapter }, }), ) + let firstReceipt: Promise | undefined + let queuedReceipt: Promise | undefined + let hasPrimaryFailure = false - const readyPromise = collection.stateWhenReady() - for (let attempt = 0; attempt < 20 && !resolveLoadSubset; attempt++) { - await flushAsyncWork() - } + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `bare truncate collection ready`, + ) + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `first`, title: `must survive` }, + }) + firstReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + void firstReceipt.catch(() => undefined) + await atPersistedOracleCheckpoint( + firstPersistenceEntered.promise, + `bare truncate first owner persistence entered`, + ) - remoteBegin?.() - remoteWrite?.({ - type: `insert`, - value: { id: `during-hydrate`, title: `During hydrate` }, - }) - const failingReceipt = remoteCommit?.() - remoteBegin?.() - remoteWrite?.({ - type: `insert`, - value: { id: `sibling`, title: `Sibling` }, - }) - const siblingReceipt = remoteCommit?.() - expect(failingReceipt).toBeInstanceOf(Promise) - expect(siblingReceipt).toBeInstanceOf(Promise) - const failingExpectation = expect( - Promise.resolve(failingReceipt), - ).rejects.toBe(replayError) - const siblingExpectation = expect( - Promise.resolve(siblingReceipt), - ).rejects.toBe(replayError) + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `queued`, title: `owns queued marker` }, + }) + queuedReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + void queuedReceipt.catch(() => undefined) - resolveLoadSubset?.() - await readyPromise - await failingExpectation - await siblingExpectation + let truncateError: unknown + try { + sourceParams.truncate() + } catch (error) { + truncateError = error + } + expect({ + truncateErrorIsOwned: + truncateError instanceof NoPendingSyncTransactionWriteError, + firstVisible: stripVirtualProps(collection.get(`first`)), + queuedVisible: collection.has(`queued`), + }).toEqual({ + truncateErrorIsOwned: true, + firstVisible: { id: `first`, title: `must survive` }, + queuedVisible: false, + }) - await collection.cleanup() + releaseFirstPersistence.resolve() + await atPersistedOracleCheckpoint(firstReceipt, `first owner persisted`) + await atPersistedOracleCheckpoint(queuedReceipt, `queued owner persisted`) + expect({ + publicRows: Array.from(collection.values()).map(stripVirtualProps), + durableRows: Array.from(adapter.rows.values()), + }).toEqual({ + publicRows: [ + { id: `first`, title: `must survive` }, + { id: `queued`, title: `owns queued marker` }, + ], + durableRows: [ + { id: `first`, title: `must survive` }, + { id: `queued`, title: `owns queued marker` }, + ], + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseFirstPersistence.resolve() + await cleanupPersistedOracle( + [ + () => firstReceipt?.catch(() => undefined), + () => queuedReceipt?.catch(() => undefined), + () => collection.cleanup(), + ], + hasPrimaryFailure, + ) + } }) - it(`marks ready even when persisted startup fails before markReady`, async () => { + it(`marks the collection errored with the exact persisted startup hydration failure`, async () => { const adapter = createRecordingAdapter() + const startupError = new Error(`startup hydration failed exactly`) adapter.loadSubset = async () => { - throw new Error(`startup failure`) + throw startupError } const collection = createCollection( @@ -1456,9 +8064,22 @@ describe(`persistedCollectionOptions`, () => { }), ) - await collection.stateWhenReady() - await flushAsyncWork() - expect(collection.status).toBe(`ready`) + const warning = vi.spyOn(console, `warn`).mockImplementation(() => {}) + let hasPrimaryFailure = false + try { + await expect(collection.stateWhenReady()).rejects.toBe(startupError) + expect(collection.status).toBe(`error`) + expect(collection._lifecycle.getSyncError()).toBe(startupError) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + warning.mockRestore() + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } }) it(`reads staged metadata writes during hydration-queued transactions`, async () => { @@ -1609,40 +8230,502 @@ describe(`persistedCollectionOptions`, () => { title: `Post truncate`, }, }) - remoteCommit?.() - await flushAsyncWork() - - expect(adapter.applyCommittedTxCalls.at(-1)?.tx.truncate).toBe(true) + remoteCommit?.() + await flushAsyncWork() + + expect(adapter.applyCommittedTxCalls.at(-1)?.tx.truncate).toBe(true) + + const reloadedCollection = createCollection( + persistedCollectionOptions({ + id: `sync-present-truncate`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { + adapter, + }, + }), + ) + + await reloadedCollection.preload() + await flushAsyncWork() + + expect(reloadedCollection.get(`pre-truncate`)).toBeUndefined() + expect(stripVirtualProps(reloadedCollection.get(`post-truncate`))).toEqual({ + id: `post-truncate`, + title: `Post truncate`, + }) + expect( + reloadedCollection._state.syncedCollectionMetadata.get(`electric:resume`), + ).toEqual({ + kind: `reset`, + updatedAt: 1, + }) + }) + + it(`fail-stops an OK seq-gap delta when its application receipt rejects`, async () => { + const id = `gap-targeted-receipt` + const receiptError = new Error(`gap targeted receipt failed exactly`) + const firstApplied = createEventGate() + const receiptEntered = createEventGate() + const terminal = createEventGate() + const fallbackReload = createEventGate() + const barrierApplied = createEventGate() + let loadCalls = 0 + const adapter = createRecordingAdapter() + adapter.loadSubset = async () => { + loadCalls++ + if (loadCalls > 1) fallbackReload.resolve() + return [] + } + const coordinator = createFailStopCoordinatorHarness(id) + coordinator.pullSince = async () => ({ + type: `rpc:pullSince:res`, + rpcId: `targeted-replay`, + ok: true, + latestTerm: 1, + latestSeq: 3, + latestRowVersion: 3, + requiresFullReload: false, + changedKeys: [`replayed`], + deletedKeys: [], + deltas: [ + { + txId: `replayed-delta`, + latestRowVersion: 2, + changedRows: [ + { + key: `replayed`, + value: { id: `replayed`, title: `targeted replay` }, + }, + ], + deletedKeys: [], + rowMetadataMutations: [], + collectionMetadataMutations: [], + }, + ], + }) + let rejectNextCommit = false + let observeNextCommit = false + let observeBarrierCommit = false + const baseOptions = persistedCollectionOptions({ + id, + getKey: (row) => row.id, + sync: { sync: ({ markReady }) => markReady() }, + persistence: { adapter, coordinator }, + }) + const baseSync = baseOptions.sync.sync + const collection = createCollection({ + ...baseOptions, + sync: { + ...baseOptions.sync, + sync: (params) => + baseSync({ + ...params, + commit: (signal) => { + const actualReceipt = params.commit(signal) + if (rejectNextCommit) { + rejectNextCommit = false + receiptEntered.resolve() + void Promise.resolve(actualReceipt).catch(() => undefined) + return Promise.reject(receiptError) + } + if (observeNextCommit) { + observeNextCommit = false + void Promise.resolve(actualReceipt).then(() => + firstApplied.resolve(), + ) + } + if (observeBarrierCommit) { + observeBarrierCommit = false + void Promise.resolve(actualReceipt).then(() => + barrierApplied.resolve(), + ) + } + return actualReceipt + }, + }), + }, + }) + const unsubscribeError = collection.on(`status:error`, () => { + terminal.resolve() + }) + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `targeted gap initial ready`, + ) + observeNextCommit = true + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 1, + txId: `first`, + latestRowVersion: 1, + requiresFullReload: false, + changedRows: [ + { key: `first`, value: { id: `first`, title: `first committed` } }, + ], + deletedKeys: [], + }) + await atPersistedOracleCheckpoint( + firstApplied.promise, + `targeted gap first transaction applied`, + ) + + rejectNextCommit = true + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 3, + txId: `gap`, + latestRowVersion: 3, + requiresFullReload: false, + changedRows: [], + deletedKeys: [], + }) + await atPersistedOracleCheckpoint( + receiptEntered.promise, + `targeted replay receipt entered`, + ) + const outcome = await atPersistedOracleCheckpoint( + Promise.race([ + terminal.promise.then(() => ({ kind: `terminal` as const })), + fallbackReload.promise.then(() => ({ kind: `fallback` as const })), + ]), + `targeted terminal error or fallback reload`, + ) + if (outcome.kind === `fallback`) { + observeBarrierCommit = true + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 4, + txId: `barrier`, + latestRowVersion: 4, + requiresFullReload: false, + changedRows: [], + deletedKeys: [], + }) + await atPersistedOracleCheckpoint( + barrierApplied.promise, + `targeted fallback barrier`, + ) + } + + expect({ + outcome: outcome.kind, + status: collection.status, + publicError: collection._lifecycle.getSyncError(), + loadCalls, + }).toEqual({ + outcome: `terminal`, + status: `error`, + publicError: receiptError, + loadCalls: 1, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + unsubscribeError() + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }) + + it(`fail-stops an OK seq-gap full reload when adapter loading rejects`, async () => { + const id = `gap-full-reload-load` + const loadError = new Error(`gap full reload load failed exactly`) + const firstApplied = createEventGate() + const failingLoadEntered = createEventGate() + const terminal = createEventGate() + const fallbackReload = createEventGate() + const barrierApplied = createEventGate() + let loadCalls = 0 + const adapter = createRecordingAdapter() + adapter.loadSubset = async () => { + loadCalls++ + if (loadCalls === 2) { + failingLoadEntered.resolve() + throw loadError + } + if (loadCalls === 3) fallbackReload.resolve() + return [] + } + const coordinator = createFailStopCoordinatorHarness(id) + coordinator.pullSince = async () => ({ + type: `rpc:pullSince:res`, + rpcId: `full-reload`, + ok: true, + latestTerm: 1, + latestSeq: 3, + latestRowVersion: 3, + requiresFullReload: true, + }) + let observeNextCommit = false + let observeBarrierCommit = false + const baseOptions = persistedCollectionOptions({ + id, + getKey: (row) => row.id, + sync: { sync: ({ markReady }) => markReady() }, + persistence: { adapter, coordinator }, + }) + const baseSync = baseOptions.sync.sync + const collection = createCollection({ + ...baseOptions, + sync: { + ...baseOptions.sync, + sync: (params) => + baseSync({ + ...params, + commit: (signal) => { + const actualReceipt = params.commit(signal) + if (observeNextCommit) { + observeNextCommit = false + void Promise.resolve(actualReceipt).then(() => + firstApplied.resolve(), + ) + } + if (observeBarrierCommit) { + observeBarrierCommit = false + void Promise.resolve(actualReceipt).then(() => + barrierApplied.resolve(), + ) + } + return actualReceipt + }, + }), + }, + }) + const unsubscribeError = collection.on(`status:error`, () => { + terminal.resolve() + }) + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `full reload gap initial ready`, + ) + observeNextCommit = true + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 1, + txId: `first`, + latestRowVersion: 1, + requiresFullReload: false, + changedRows: [ + { key: `first`, value: { id: `first`, title: `first committed` } }, + ], + deletedKeys: [], + }) + await atPersistedOracleCheckpoint( + firstApplied.promise, + `full reload gap first transaction applied`, + ) + + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 3, + txId: `gap`, + latestRowVersion: 3, + requiresFullReload: false, + changedRows: [], + deletedKeys: [], + }) + await atPersistedOracleCheckpoint( + failingLoadEntered.promise, + `full reload gap failing load entered`, + ) + const outcome = await atPersistedOracleCheckpoint( + Promise.race([ + terminal.promise.then(() => ({ kind: `terminal` as const })), + fallbackReload.promise.then(() => ({ kind: `fallback` as const })), + ]), + `full reload terminal error or second fallback reload`, + ) + if (outcome.kind === `fallback`) { + observeBarrierCommit = true + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 4, + txId: `barrier`, + latestRowVersion: 4, + requiresFullReload: false, + changedRows: [], + deletedKeys: [], + }) + await atPersistedOracleCheckpoint( + barrierApplied.promise, + `full reload fallback barrier`, + ) + } + + expect({ + outcome: outcome.kind, + status: collection.status, + publicError: collection._lifecycle.getSyncError(), + loadCalls, + }).toEqual({ + outcome: `terminal`, + status: `error`, + publicError: loadError, + loadCalls: 2, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + unsubscribeError() + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }) + + it(`keeps seq-gap pull transport rejection on the fallback path`, async () => { + const id = `gap-transport-control` + const transportError = new Error(`gap pull transport failed exactly`) + const firstApplied = createEventGate() + const fallbackReload = createEventGate() + const barrierApplied = createEventGate() + let loadCalls = 0 + const adapter = createRecordingAdapter() + adapter.loadSubset = async () => { + loadCalls++ + if (loadCalls === 2) fallbackReload.resolve() + return [] + } + const coordinator = createFailStopCoordinatorHarness(id) + coordinator.pullSince = async () => { + throw transportError + } + let observeNextCommit = false + let observeBarrierCommit = false + const baseOptions = persistedCollectionOptions({ + id, + getKey: (row) => row.id, + sync: { sync: ({ markReady }) => markReady() }, + persistence: { adapter, coordinator }, + }) + const baseSync = baseOptions.sync.sync + const collection = createCollection({ + ...baseOptions, + sync: { + ...baseOptions.sync, + sync: (params) => + baseSync({ + ...params, + commit: (signal) => { + const actualReceipt = params.commit(signal) + if (observeNextCommit) { + observeNextCommit = false + void Promise.resolve(actualReceipt).then(() => + firstApplied.resolve(), + ) + } + if (observeBarrierCommit) { + observeBarrierCommit = false + void Promise.resolve(actualReceipt).then(() => + barrierApplied.resolve(), + ) + } + return actualReceipt + }, + }), + }, + }) + const warning = vi.spyOn(console, `warn`).mockImplementation(() => {}) + let hasPrimaryFailure = false - const reloadedCollection = createCollection( - persistedCollectionOptions({ - id: `sync-present-truncate`, - getKey: (item) => item.id, - sync: { - sync: ({ markReady }) => { - markReady() - }, - }, - persistence: { - adapter, - }, - }), - ) + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `transport gap initial ready`, + ) + observeNextCommit = true + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 1, + txId: `first`, + latestRowVersion: 1, + requiresFullReload: false, + changedRows: [ + { key: `first`, value: { id: `first`, title: `first committed` } }, + ], + deletedKeys: [], + }) + await atPersistedOracleCheckpoint( + firstApplied.promise, + `transport gap first transaction applied`, + ) - await reloadedCollection.preload() - await flushAsyncWork() + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 3, + txId: `gap`, + latestRowVersion: 3, + requiresFullReload: false, + changedRows: [], + deletedKeys: [], + }) + await atPersistedOracleCheckpoint( + fallbackReload.promise, + `transport gap fallback reload`, + ) + observeBarrierCommit = true + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 4, + txId: `barrier`, + latestRowVersion: 4, + requiresFullReload: false, + changedRows: [], + deletedKeys: [], + }) + await atPersistedOracleCheckpoint( + barrierApplied.promise, + `transport fallback barrier`, + ) - expect(reloadedCollection.get(`pre-truncate`)).toBeUndefined() - expect(stripVirtualProps(reloadedCollection.get(`post-truncate`))).toEqual({ - id: `post-truncate`, - title: `Post truncate`, - }) - expect( - reloadedCollection._state.syncedCollectionMetadata.get(`electric:resume`), - ).toEqual({ - kind: `reset`, - updatedAt: 1, - }) + expect(warning).toHaveBeenCalledWith( + `Failed pullSince recovery attempt:`, + transportError, + ) + expect({ + status: collection.status, + publicError: collection._lifecycle.getSyncError(), + loadCalls, + }).toEqual({ + status: `ready`, + publicError: undefined, + loadCalls: 2, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + warning.mockRestore() + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } }) it(`uses pullSince recovery when tx sequence gaps are detected`, async () => { @@ -2069,6 +9152,295 @@ describe(`persistedCollectionOptions`, () => { } }) + it(`marks the collection errored with the exact later on-demand hydration failure`, async () => { + const hydrationFailure = new Error(`later on-demand hydration failed`) + const adapter = createRecordingAdapter() + const collection = createCollection( + persistedCollectionOptions({ + id: `later-on-demand-hydration-failure`, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: ({ markReady }) => { + markReady() + return { loadSubset: () => true } + }, + }, + persistence: { adapter }, + }), + ) + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `on-demand collection initially ready`, + ) + expect(collection.status).toBe(`ready`) + adapter.loadSubset = () => Promise.reject(hydrationFailure) + + await expect( + atPersistedOracleCheckpoint( + Promise.resolve(collection._sync.loadSubset({ limit: 1 })), + `later on-demand hydration rejected`, + ), + ).rejects.toBe(hydrationFailure) + + expect(collection.status).toBe(`error`) + expect(collection._lifecycle.getSyncError()).toBe(hydrationFailure) + expect([...collection.values()]).toEqual([]) + expect(adapter.rows.size).toBe(0) + expect(adapter.applyCommittedTxCalls).toEqual([]) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }) + + it(`marks the collection errored and rejects later forced work after a hydration failure`, async () => { + const hydrationFailure = new Error(`later forced hydration failed`) + const adapter = createRecordingAdapter() + const collection = createCollection( + persistedCollectionOptions({ + id: `later-forced-hydration-failure`, + getKey: (row) => row.id, + syncMode: `on-demand`, + persistence: { adapter }, + }), + ) + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `forced-hydration collection initially ready`, + ) + expect(collection.status).toBe(`ready`) + adapter.loadSubset = () => Promise.reject(hydrationFailure) + + await expect( + atPersistedOracleCheckpoint( + Promise.resolve(collection.utils.forceReloadSubset!({ limit: 1 })), + `later forced hydration rejected`, + ), + ).rejects.toBe(hydrationFailure) + + expect(collection.status).toBe(`error`) + expect(collection._lifecycle.getSyncError()).toBe(hydrationFailure) + expect([...collection.values()]).toEqual([]) + expect(adapter.rows.size).toBe(0) + expect(adapter.applyCommittedTxCalls).toEqual([]) + + let laterAdapterCalls = 0 + adapter.loadSubset = () => { + laterAdapterCalls++ + return Promise.resolve([ + { + key: `late-forced`, + value: { id: `late-forced`, title: `must not be admitted` }, + }, + ]) + } + await expect( + atPersistedOracleCheckpoint( + Promise.resolve(collection.utils.forceReloadSubset!({ limit: 2 })), + `later forced subset work settled`, + ), + ).rejects.toBe(hydrationFailure) + expect({ + laterAdapterCalls, + visible: collection.get(`late-forced`), + durable: adapter.rows.get(`late-forced`), + durabilityCalls: adapter.applyCommittedTxCalls.length, + }).toEqual({ + laterAdapterCalls: 0, + visible: undefined, + durable: undefined, + durabilityCalls: 0, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }) + + it(`rejects subset work admitted after a terminal hydration failure`, async () => { + const hydrationFailure = new Error(`terminal hydration failed`) + const adapter = createRecordingAdapter() + const collection = createCollection( + persistedCollectionOptions({ + id: `terminal-hydration-rejects-later-work`, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: ({ markReady }) => { + markReady() + return { loadSubset: () => true } + }, + }, + persistence: { adapter }, + }), + ) + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `terminal-hydration collection initially ready`, + ) + adapter.loadSubset = () => Promise.reject(hydrationFailure) + await expect( + atPersistedOracleCheckpoint( + Promise.resolve(collection._sync.loadSubset({ limit: 1 })), + `terminal hydration rejected`, + ), + ).rejects.toBe(hydrationFailure) + expect(collection.status).toBe(`error`) + + let laterAdapterCalls = 0 + adapter.loadSubset = () => { + laterAdapterCalls++ + return Promise.resolve([ + { + key: `late`, + value: { id: `late`, title: `must not be admitted` }, + }, + ]) + } + const laterOutcome = await atPersistedOracleCheckpoint( + Promise.resolve(collection._sync.loadSubset({ limit: 2 })).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `later subset work settled`, + ) + + expect({ + status: laterOutcome.status, + exactReason: + laterOutcome.status === `rejected` && + laterOutcome.reason === hydrationFailure, + laterAdapterCalls, + visible: collection.get(`late`), + durable: adapter.rows.get(`late`), + durabilityCalls: adapter.applyCommittedTxCalls.length, + }).toEqual({ + status: `rejected`, + exactReason: true, + laterAdapterCalls: 0, + visible: undefined, + durable: undefined, + durabilityCalls: 0, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }) + + it(`rejects new source transactions after a terminal hydration failure`, async () => { + const hydrationFailure = new Error(`terminal source hydration failed`) + const adapter = createRecordingAdapter() + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: (() => true | Promise) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: `terminal-hydration-rejects-new-source-work`, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit + markReady() + }, + }, + persistence: { adapter }, + }), + ) + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `terminal source-hydration collection initially ready`, + ) + adapter.loadSubset = () => Promise.reject(hydrationFailure) + await expect( + atPersistedOracleCheckpoint( + Promise.resolve(collection._sync.loadSubset({ limit: 1 })), + `terminal source hydration rejected`, + ), + ).rejects.toBe(hydrationFailure) + expect(collection.status).toBe(`error`) + expect(collection._lifecycle.getSyncError()).toBe(hydrationFailure) + + remoteBegin!() + remoteWrite!({ + type: `insert`, + value: { id: `late`, title: `must not be admitted` }, + }) + const lateOutcome = await atPersistedOracleCheckpoint( + Promise.resolve(remoteCommit!()).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `late terminal hydration source receipt`, + ) + + expect({ + receiptStatus: lateOutcome.status, + exactReceipt: + lateOutcome.status === `rejected` && + lateOutcome.reason === hydrationFailure, + publicErrorIsHydration: + collection._lifecycle.getSyncError() === hydrationFailure, + status: collection.status, + lateVisible: collection.get(`late`), + lateDurable: adapter.rows.get(`late`), + applyCalls: adapter.applyCommittedTxCalls.length, + }).toEqual({ + receiptStatus: `rejected`, + exactReceipt: true, + publicErrorIsHydration: true, + status: `error`, + lateVisible: undefined, + lateDurable: undefined, + applyCalls: 0, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + await cleanupPersistedOracle( + [() => collection.cleanup()], + hasPrimaryFailure, + ) + } + }) + it.each([`throw`, `reject`] as const)( `releases only transferred upstream ownership after a load %s`, async (mode) => { @@ -2521,6 +9893,270 @@ describe(`persistedCollectionOptions`, () => { }) }) + it(`fail-stops queued and later work when reset reload fails after truncation`, async () => { + const seed = { id: `seed`, title: `durable baseline` } + const adapter = createRecordingAdapter([seed]) + const coordinator = createFailStopCoordinatorHarness( + `terminal-reset-reload-failure`, + ) + const reloadEntered = createEventGate() + const reload = createEventGate() + const reloadError = new Error(`reset reload failed exactly`) + let sourceParams!: TodoSyncParams + const collection = createCollection( + persistedCollectionOptions({ + id: `terminal-reset-reload-failure`, + getKey: (row) => row.id, + sync: { + sync: (params) => { + sourceParams = params + params.markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + const warning = vi.spyOn(console, `warn`).mockImplementation(() => {}) + let queuedReceipt: Promise | undefined + let hasPrimaryFailure = false + + try { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `reset reload collection ready`, + ) + expect(stripVirtualProps(collection.get(seed.id))).toEqual(seed) + adapter.loadSubset = async () => { + reloadEntered.resolve() + await reload.promise + throw reloadError + } + coordinator.emit({ + type: `collection:reset`, + schemaVersion: 1, + resetEpoch: 1, + }) + await atPersistedOracleCheckpoint( + reloadEntered.promise, + `reset reload entered after memory truncate`, + ) + expect(collection.get(seed.id)).toBeUndefined() + + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `queued`, title: `must reject with reset failure` }, + }) + queuedReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + void queuedReceipt.catch(() => undefined) + const queuedSettlement = observeSettlement(queuedReceipt) + expect(queuedSettlement.read()).toEqual({ status: `pending` }) + + reload.reject(reloadError) + await flushAsyncWork() + await flushAsyncWork() + + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `late`, title: `must not continue after reset failure` }, + }) + const lateOutcome = await atPersistedOracleCheckpoint( + Promise.resolve(sourceParams.commit()).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `post-reset-reload-failure receipt`, + ) + const queuedOutcome = queuedSettlement.read() + + expect({ + queuedStatus: queuedOutcome.status, + queuedExact: + queuedOutcome.status === `rejected` && + queuedOutcome.reason === reloadError, + lateStatus: lateOutcome.status, + lateExact: + lateOutcome.status === `rejected` && + lateOutcome.reason === reloadError, + status: collection.status, + exactPublicError: collection._lifecycle.getSyncError() === reloadError, + visibleSeed: collection.get(seed.id), + durableSeed: adapter.rows.get(seed.id), + visibleQueued: collection.get(`queued`), + durableQueued: adapter.rows.get(`queued`), + visibleLate: collection.get(`late`), + durableLate: adapter.rows.get(`late`), + applyCalls: adapter.applyCommittedTxCalls.length, + publishCalls: coordinator.publishCalls.length, + }).toEqual({ + queuedStatus: `rejected`, + queuedExact: true, + lateStatus: `rejected`, + lateExact: true, + status: `error`, + exactPublicError: true, + visibleSeed: undefined, + durableSeed: seed, + visibleQueued: undefined, + durableQueued: undefined, + visibleLate: undefined, + durableLate: undefined, + applyCalls: 0, + publishCalls: 0, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + reload.reject(reloadError) + warning.mockRestore() + await cleanupPersistedOracle( + [ + () => collection.cleanup(), + () => queuedReceipt?.catch(() => undefined), + ], + hasPrimaryFailure, + ) + } + }) + + it(`fences coordinator delivery after terminal failure while allowing release and cleanup`, async () => { + const harness = await createTerminalFailureHarness( + `durability`, + `terminal-coordinator-delivery-fence`, + ) + const { + adapter, + collection, + coordinator, + initialLease, + terminalError, + upstreamLoads, + upstreamUnloads, + } = harness + const adapterLoadsBefore = adapter.loadSubsetCalls.length + const ensureCallsBefore = coordinator.remoteEnsureCalls.length + const upstreamLoadsBefore = upstreamLoads.length + const applyCallsBefore = adapter.applyCommittedTxCalls.length + const publishCallsBefore = coordinator.publishCalls.length + adapter.loadSubset = (collectionId, options, context) => { + adapter.loadSubsetCalls.push({ + collectionId, + options, + requiredIndexSignatures: context?.requiredIndexSignatures ?? [], + }) + return Promise.resolve([ + { + key: `coordinator-late`, + value: { id: `coordinator-late`, title: `must not arrive` }, + }, + ]) + } + let cleanedUp = false + let hasPrimaryFailure = false + + try { + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 1, + txId: `post-terminal-targeted`, + latestRowVersion: 1, + requiresFullReload: false, + changedRows: [ + { + key: `targeted-late`, + value: { id: `targeted-late`, title: `must not arrive` }, + }, + ], + deletedKeys: [], + }) + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 2, + txId: `post-terminal-full-reload`, + latestRowVersion: 2, + requiresFullReload: true, + }) + coordinator.emit({ + type: `collection:reset`, + schemaVersion: 1, + resetEpoch: 2, + }) + const remoteEnsureOutcome = await atPersistedOracleCheckpoint( + Promise.resolve(collection._sync.loadSubset({ limit: 9 })).then( + () => ({ status: `fulfilled` as const }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ), + `post-terminal remote ensure delivery`, + ) + await flushAsyncWork() + await flushAsyncWork() + await flushAsyncWork() + + const beforeCleanup = { + targetedVisible: collection.get(`targeted-late`), + reloadVisible: collection.get(`coordinator-late`), + status: collection.status, + exactPublicError: + collection._lifecycle.getSyncError() === terminalError, + } + + collection._sync.unloadSubset(initialLease) + const unloadCount = upstreamUnloads.length + await collection.cleanup() + cleanedUp = true + + expect({ + remoteEnsureStatus: remoteEnsureOutcome.status, + remoteEnsureExact: + remoteEnsureOutcome.status === `rejected` && + remoteEnsureOutcome.reason === terminalError, + adapterLoads: adapter.loadSubsetCalls.length, + ensureCalls: coordinator.remoteEnsureCalls.length, + upstreamLoads: upstreamLoads.length, + unloadCount, + unsubscribeCalls: coordinator.unsubscribeCalls, + applyCalls: adapter.applyCommittedTxCalls.length, + publishCalls: coordinator.publishCalls.length, + targetedVisible: beforeCleanup.targetedVisible, + reloadVisible: beforeCleanup.reloadVisible, + targetedDurable: adapter.rows.get(`targeted-late`), + reloadDurable: adapter.rows.get(`coordinator-late`), + status: beforeCleanup.status, + exactPublicError: beforeCleanup.exactPublicError, + }).toEqual({ + remoteEnsureStatus: `rejected`, + remoteEnsureExact: true, + adapterLoads: adapterLoadsBefore, + ensureCalls: ensureCallsBefore, + upstreamLoads: upstreamLoadsBefore, + unloadCount: 1, + unsubscribeCalls: 1, + applyCalls: applyCallsBefore, + publishCalls: publishCallsBefore, + targetedVisible: undefined, + reloadVisible: undefined, + targetedDurable: undefined, + reloadDurable: undefined, + status: `error`, + exactPublicError: true, + }) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + await cleanupPersistedOracle( + [() => (cleanedUp ? undefined : collection.cleanup())], + hasPrimaryFailure, + ) + } + }) + it(`keeps a hydrated resume baseline across narrow full reloads`, async () => { const adapter = createRecordingAdapter([ { id: `1`, title: `Narrow` }, @@ -2605,7 +10241,7 @@ describe(`persistedCollectionOptions`, () => { }) }) -describe(`persisted key and identifier helpers`, () => { +describeUnlessOracleReplay(`persisted key and identifier helpers`, () => { it(`encodes and decodes persisted storage keys without collisions`, () => { expect(encodePersistedStorageKey(1)).toBe(`n:1`) expect(encodePersistedStorageKey(`1`)).toBe(`s:1`) diff --git a/packages/db/src/collection/sync.ts b/packages/db/src/collection/sync.ts index a334c7665..aa5e9fbcd 100644 --- a/packages/db/src/collection/sync.ts +++ b/packages/db/src/collection/sync.ts @@ -22,8 +22,10 @@ import type { LoadSubsetOptions, LoadSubsetRequestResult, OptimisticChangeMessage, + SyncAppliedReceipt, SyncConfigRes, SyncMetadataApi, + SyncTransactionHandle, } from '../types' import type { CollectionImpl } from './index.js' import type { CollectionStateManager } from './state' @@ -136,156 +138,12 @@ export class CollectionSyncManager< const syncRes = normalizeSyncFnResult( this.config.sync.sync({ collection: this.collection, - begin: (options?: { immediate?: boolean }) => { - if (!isCurrentSync()) return - const applied = createDeferred() - // A source may ignore a stream receipt. Keep cancellation from - // becoming an unhandled rejection while preserving the original - // promise's rejection for callers that do await it. - void applied.promise.catch(() => undefined) - this.state.pendingSyncedTransactions.push({ - committed: false, - applicationStarted: false, - layoutChanged: false, - operations: [], - deletedKeys: new Set(), - rowMetadataWrites: new Map(), - collectionMetadataWrites: new Map(), - immediate: options?.immediate, - applied, - }) - }, - write: ( - messageWithOptionalKey: ChangeMessageOrDeleteKeyMessage< - TOutput, - TKey - >, - ) => { - if (!isCurrentSync()) return - const pendingTransaction = - this.state.pendingSyncedTransactions[ - this.state.pendingSyncedTransactions.length - 1 - ] - if (!pendingTransaction) { - throw new NoPendingSyncTransactionWriteError() - } - if (pendingTransaction.committed) { - throw new SyncTransactionAlreadyCommittedWriteError() - } - - let key: TKey | undefined = undefined - if (`key` in messageWithOptionalKey) { - key = messageWithOptionalKey.key - } else { - key = this.config.getKey(messageWithOptionalKey.value) - } - - let messageType = messageWithOptionalKey.type - - // Check if an item with this key already exists when inserting - if (messageWithOptionalKey.type === `insert`) { - const insertingIntoExistingSynced = this.state.syncedData.has(key) - const hasPendingDeleteForKey = - pendingTransaction.deletedKeys.has(key) - const isTruncateTransaction = pendingTransaction.truncate === true - // Allow insert after truncate in the same transaction even if it existed in syncedData - if ( - insertingIntoExistingSynced && - !hasPendingDeleteForKey && - !isTruncateTransaction - ) { - const existingValue = this.state.syncedData.get(key) - const valuesEqual = - existingValue !== undefined && - deepEquals(existingValue, messageWithOptionalKey.value) - if (valuesEqual || this.state.hydrationSeedKeys.has(key)) { - // The "insert" is an echo of a value we already have locally. - // Hydration and initialData are also provisional base state, so - // accept the adapter's first authoritative value as an update - // using the configured rowUpdateMode semantics. - messageType = `update` - } else { - const utils = this.config.utils as - | Partial - | undefined - const internal = utils?.[LIVE_QUERY_INTERNAL] - throw new DuplicateKeySyncError(key, this.id, { - hasCustomGetKey: internal?.hasCustomGetKey ?? false, - hasJoins: internal?.hasJoins ?? false, - hasDistinct: internal?.hasDistinct ?? false, - }) - } - } - } - - const message = { - ...messageWithOptionalKey, - type: messageType, - key, - } as OptimisticChangeMessage - pendingTransaction.operations.push(message) - - if (messageType === `delete`) { - pendingTransaction.deletedKeys.add(key) - pendingTransaction.rowMetadataWrites.set(key, { type: `delete` }) - } else if (messageType === `insert`) { - if (message.metadata !== undefined) { - pendingTransaction.rowMetadataWrites.set(key, { - type: `set`, - value: message.metadata, - }) - } else { - pendingTransaction.rowMetadataWrites.set(key, { - type: `delete`, - }) - } - } else if (message.metadata !== undefined) { - pendingTransaction.rowMetadataWrites.set(key, { - type: `set`, - value: message.metadata, - }) - } - }, - commit: (signal?: AbortSignal) => { - if (!isCurrentSync()) return true - const pendingTransaction = - this.state.pendingSyncedTransactions[ - this.state.pendingSyncedTransactions.length - 1 - ] - if (!pendingTransaction) { - throw new NoPendingSyncTransactionCommitError() - } - if (pendingTransaction.committed) { - throw new SyncTransactionAlreadyCommittedError() - } - - if (signal?.aborted) { - this.state.cancelPendingSyncedTransaction(pendingTransaction) - return pendingTransaction.applied.promise - } - - pendingTransaction.committed = true - - const cancel = () => { - this.state.cancelPendingSyncedTransaction(pendingTransaction) - } - signal?.addEventListener(`abort`, cancel, { once: true }) - - this.state.commitPendingTransactions() - if (!pendingTransaction.applied.isPending()) { - signal?.removeEventListener(`abort`, cancel) - return true - } - - const receipt = pendingTransaction.applied.promise - if (signal) { - const removeAbortListener = () => { - signal.removeEventListener(`abort`, cancel) - } - void receipt.then(removeAbortListener, removeAbortListener) - } - return receipt - }, + begin: (options?: { immediate?: boolean }) => + this.beginSyncTransaction(options, isCurrentSync), + write: (message) => + this.writeSyncTransaction(message, undefined, isCurrentSync), + commit: (signal?: AbortSignal) => + this.commitSyncTransaction(undefined, signal, isCurrentSync), markReady: () => { if (!isCurrentSync()) return if (syncEntryActive) { @@ -297,42 +155,8 @@ export class CollectionSyncManager< markError: (error?: unknown) => { if (isCurrentSync()) this.lifecycle.markError(error) }, - truncate: () => { - if (!isCurrentSync()) return - const pendingTransaction = - this.state.pendingSyncedTransactions[ - this.state.pendingSyncedTransactions.length - 1 - ] - if (!pendingTransaction) { - throw new NoPendingSyncTransactionWriteError() - } - if (pendingTransaction.committed) { - throw new SyncTransactionAlreadyCommittedWriteError() - } - - // Clear all operations from the current transaction - pendingTransaction.operations = [] - pendingTransaction.deletedKeys.clear() - pendingTransaction.rowMetadataWrites.clear() - // Intentionally preserve collectionMetadataWrites across truncate. - // Collection-scoped metadata (for example persisted resume/reset - // state) can be staged before truncate and should commit atomically - // with the truncate transaction. - - // Mark the transaction as a truncate operation. During commit, this triggers: - // - Delete events for all previously synced keys (excluding optimistic-deleted keys) - // - Clearing of syncedData/syncedMetadata - // - Subsequent synced ops applied on the fresh base - // - Finally, optimistic mutations re-applied on top (single batch) - pendingTransaction.truncate = true - - // Capture optimistic state NOW to preserve it even if transactions complete - // before this truncate transaction is committed - pendingTransaction.optimisticSnapshot = { - upserts: new Map(this.state.optimisticUpserts), - deletes: new Set(this.state.optimisticDeletes), - } - }, + truncate: () => + this.truncateSyncTransaction(undefined, isCurrentSync), metadata: this.createSyncMetadataApi(isCurrentSync), }), ) @@ -434,13 +258,219 @@ export class CollectionSyncManager< } } - private getActivePendingSyncTransaction() { - const pendingTransaction = - this.state.pendingSyncedTransactions[ - this.state.pendingSyncedTransactions.length - 1 - ] + private beginSyncTransaction( + options: { immediate?: boolean } | undefined, + isCurrentSync: () => boolean, + ): SyncTransactionHandle | undefined { + if (!isCurrentSync()) return undefined + const applied = createDeferred() + // A source may ignore a stream receipt. Keep cancellation from becoming an + // unhandled rejection while preserving the original promise for callers. + void applied.promise.catch(() => undefined) + const pendingTransaction = { + committed: false, + applicationStarted: false, + layoutChanged: false, + operations: [], + deletedKeys: new Set(), + rowMetadataWrites: new Map(), + collectionMetadataWrites: new Map(), + immediate: options?.immediate, + applied, + } + this.state.pendingSyncedTransactions.push(pendingTransaction) + + return { + write: (message, knownKey) => + this.writeSyncTransaction( + message, + pendingTransaction, + isCurrentSync, + knownKey, + ), + commit: (signal) => + this.commitSyncTransaction(pendingTransaction, signal, isCurrentSync), + truncate: () => + this.truncateSyncTransaction(pendingTransaction, isCurrentSync), + metadata: this.createSyncMetadataApi(isCurrentSync, pendingTransaction), + abort: () => { + if (!isCurrentSync()) return + this.state.cancelPendingSyncedTransaction(pendingTransaction) + }, + } + } + private writeSyncTransaction( + messageWithOptionalKey: ChangeMessageOrDeleteKeyMessage, + transaction: + | CollectionStateManager< + TOutput, + TKey, + TSchema, + TInput + >[`pendingSyncedTransactions`][number] + | undefined, + isCurrentSync: () => boolean, + knownKey?: TKey, + ): void { + if (!isCurrentSync()) return + const pendingTransaction = this.getActivePendingSyncTransaction(transaction) + + let key: TKey + if (knownKey !== undefined) { + key = knownKey + } else if (`key` in messageWithOptionalKey) { + key = messageWithOptionalKey.key + } else { + key = this.config.getKey(messageWithOptionalKey.value) + } + + let messageType = messageWithOptionalKey.type + if (messageWithOptionalKey.type === `insert`) { + const insertingIntoExistingSynced = this.state.syncedData.has(key) + const hasPendingDeleteForKey = pendingTransaction.deletedKeys.has(key) + const isTruncateTransaction = pendingTransaction.truncate === true + if ( + insertingIntoExistingSynced && + !hasPendingDeleteForKey && + !isTruncateTransaction + ) { + const existingValue = this.state.syncedData.get(key) + const valuesEqual = + existingValue !== undefined && + deepEquals(existingValue, messageWithOptionalKey.value) + if (valuesEqual || this.state.hydrationSeedKeys.has(key)) { + messageType = `update` + } else { + const utils = this.config.utils as + | Partial + | undefined + const internal = utils?.[LIVE_QUERY_INTERNAL] + throw new DuplicateKeySyncError(key, this.id, { + hasCustomGetKey: internal?.hasCustomGetKey ?? false, + hasJoins: internal?.hasJoins ?? false, + hasDistinct: internal?.hasDistinct ?? false, + }) + } + } + } + + const message = { + ...messageWithOptionalKey, + type: messageType, + key, + } as OptimisticChangeMessage + pendingTransaction.operations.push(message) + + if (messageType === `delete`) { + pendingTransaction.deletedKeys.add(key) + pendingTransaction.rowMetadataWrites.set(key, { type: `delete` }) + } else if (messageType === `insert`) { + pendingTransaction.rowMetadataWrites.set( + key, + message.metadata === undefined + ? { type: `delete` } + : { type: `set`, value: message.metadata }, + ) + } else if (message.metadata !== undefined) { + pendingTransaction.rowMetadataWrites.set(key, { + type: `set`, + value: message.metadata, + }) + } + } + + private commitSyncTransaction( + transaction: + | CollectionStateManager< + TOutput, + TKey, + TSchema, + TInput + >[`pendingSyncedTransactions`][number] + | undefined, + signal: AbortSignal | undefined, + isCurrentSync: () => boolean, + ): SyncAppliedReceipt { + if (!isCurrentSync()) return true + const pendingTransaction = + transaction ?? this.state.pendingSyncedTransactions.at(-1) if (!pendingTransaction) { + throw new NoPendingSyncTransactionCommitError() + } + if (!this.state.pendingSyncedTransactions.includes(pendingTransaction)) { + throw new NoPendingSyncTransactionCommitError() + } + if (pendingTransaction.committed) { + throw new SyncTransactionAlreadyCommittedError() + } + + if (signal?.aborted) { + this.state.cancelPendingSyncedTransaction(pendingTransaction) + return pendingTransaction.applied.promise + } + + pendingTransaction.committed = true + const cancel = () => { + this.state.cancelPendingSyncedTransaction(pendingTransaction) + } + signal?.addEventListener(`abort`, cancel, { once: true }) + + this.state.commitPendingTransactions() + if (!pendingTransaction.applied.isPending()) { + signal?.removeEventListener(`abort`, cancel) + return true + } + + const receipt = pendingTransaction.applied.promise + if (signal) { + const removeAbortListener = () => { + signal.removeEventListener(`abort`, cancel) + } + void receipt.then(removeAbortListener, removeAbortListener) + } + return receipt + } + + private truncateSyncTransaction( + transaction: + | CollectionStateManager< + TOutput, + TKey, + TSchema, + TInput + >[`pendingSyncedTransactions`][number] + | undefined, + isCurrentSync: () => boolean, + ): void { + if (!isCurrentSync()) return + const pendingTransaction = this.getActivePendingSyncTransaction(transaction) + pendingTransaction.operations = [] + pendingTransaction.deletedKeys.clear() + pendingTransaction.rowMetadataWrites.clear() + // Collection-scoped metadata intentionally survives truncate. + pendingTransaction.truncate = true + pendingTransaction.optimisticSnapshot = { + upserts: new Map(this.state.optimisticUpserts), + deletes: new Set(this.state.optimisticDeletes), + } + } + + private getActivePendingSyncTransaction( + transaction?: CollectionStateManager< + TOutput, + TKey, + TSchema, + TInput + >[`pendingSyncedTransactions`][number], + ) { + const pendingTransaction = + transaction ?? this.state.pendingSyncedTransactions.at(-1) + + if ( + !pendingTransaction || + !this.state.pendingSyncedTransactions.includes(pendingTransaction) + ) { throw new NoPendingSyncTransactionWriteError() } if (pendingTransaction.committed) { @@ -452,15 +482,19 @@ export class CollectionSyncManager< private createSyncMetadataApi( isCurrentSync: () => boolean, + transaction?: CollectionStateManager< + TOutput, + TKey, + TSchema, + TInput + >[`pendingSyncedTransactions`][number], ): SyncMetadataApi { return { row: { get: (key) => { if (!isCurrentSync()) return undefined const pendingTransaction = - this.state.pendingSyncedTransactions[ - this.state.pendingSyncedTransactions.length - 1 - ] + transaction ?? this.state.pendingSyncedTransactions.at(-1) const pendingWrite = pendingTransaction?.rowMetadataWrites.get(key) if (pendingWrite) { return pendingWrite.type === `delete` @@ -474,7 +508,8 @@ export class CollectionSyncManager< }, set: (key, metadata) => { if (!isCurrentSync()) return - const pendingTransaction = this.getActivePendingSyncTransaction() + const pendingTransaction = + this.getActivePendingSyncTransaction(transaction) pendingTransaction.rowMetadataWrites.set(key, { type: `set`, value: metadata, @@ -482,7 +517,8 @@ export class CollectionSyncManager< }, delete: (key) => { if (!isCurrentSync()) return - const pendingTransaction = this.getActivePendingSyncTransaction() + const pendingTransaction = + this.getActivePendingSyncTransaction(transaction) pendingTransaction.rowMetadataWrites.set(key, { type: `delete`, }) @@ -492,9 +528,7 @@ export class CollectionSyncManager< get: (key) => { if (!isCurrentSync()) return undefined const pendingTransaction = - this.state.pendingSyncedTransactions[ - this.state.pendingSyncedTransactions.length - 1 - ] + transaction ?? this.state.pendingSyncedTransactions.at(-1) const pendingWrite = pendingTransaction?.collectionMetadataWrites.get(key) if (pendingWrite) { @@ -506,7 +540,8 @@ export class CollectionSyncManager< }, set: (key, value) => { if (!isCurrentSync()) return - const pendingTransaction = this.getActivePendingSyncTransaction() + const pendingTransaction = + this.getActivePendingSyncTransaction(transaction) pendingTransaction.collectionMetadataWrites.set(key, { type: `set`, value, @@ -514,7 +549,8 @@ export class CollectionSyncManager< }, delete: (key) => { if (!isCurrentSync()) return - const pendingTransaction = this.getActivePendingSyncTransaction() + const pendingTransaction = + this.getActivePendingSyncTransaction(transaction) pendingTransaction.collectionMetadataWrites.set(key, { type: `delete`, }) @@ -523,9 +559,7 @@ export class CollectionSyncManager< if (!isCurrentSync()) return [] const merged = new Map(this.state.syncedCollectionMetadata) const pendingTransaction = - this.state.pendingSyncedTransactions[ - this.state.pendingSyncedTransactions.length - 1 - ] + transaction ?? this.state.pendingSyncedTransactions.at(-1) if (pendingTransaction) { for (const [ key, diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index f6300fb08..51d32a95f 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -376,6 +376,25 @@ export type LoadSubsetFn = (options: LoadSubsetOptions) => true | Promise */ export type SyncAppliedReceipt = true | Promise +/** + * Owns one sync transaction independently of the ambient adapter callback + * stack. First-party wrappers use the handle when application may be deferred + * beyond the callback that began the transaction. + */ +export type SyncTransactionHandle< + T extends object = Record, + TKey extends string | number = string | number, +> = { + write: ( + message: ChangeMessageOrDeleteKeyMessage, + knownKey?: TKey, + ) => void + commit: (signal?: AbortSignal) => SyncAppliedReceipt + truncate: () => void + metadata: SyncMetadataApi + abort: () => void +} + /** * Releases the exact acquisition created for `options`. * @@ -404,7 +423,9 @@ export interface SyncConfig< * @param options.immediate - When true, the transaction will be processed immediately * even if there are persisting user transactions. Used by manual write operations. */ - begin: (options?: { immediate?: boolean }) => void + begin: (options?: { + immediate?: boolean + }) => SyncTransactionHandle | undefined write: (message: ChangeMessageOrDeleteKeyMessage) => void /** * Commit the active sync transaction in FIFO order. diff --git a/packages/db/tests/collection-subscription-lifecycle-publication.property.test.ts b/packages/db/tests/collection-subscription-lifecycle-publication.property.test.ts index d0c002f98..3790321ef 100644 --- a/packages/db/tests/collection-subscription-lifecycle-publication.property.test.ts +++ b/packages/db/tests/collection-subscription-lifecycle-publication.property.test.ts @@ -623,7 +623,7 @@ async function runPublicationHistory( const subscription = collection.subscribeChanges( (changes) => { const batch = changes.map((change): PublicationChange => { - const key = change.key + const key: unknown = change.key if (key !== `a` && key !== `b` && key !== `c` && key !== `d`) { throw new Error(`publication used an unknown row key`) } diff --git a/packages/db/tests/collection-sync-reentrancy.test.ts b/packages/db/tests/collection-sync-reentrancy.test.ts index bf0e66d31..5b872b03f 100644 --- a/packages/db/tests/collection-sync-reentrancy.test.ts +++ b/packages/db/tests/collection-sync-reentrancy.test.ts @@ -166,7 +166,7 @@ async function runListenerScenario(scenario: ListenerScenario): Promise { const subscription = collection.subscribeChanges((changes) => { listenerDepth++ maxListenerDepth = Math.max(maxListenerDepth, listenerDepth) - batches.push(changes.map((change) => change.key as number)) + batches.push(changes.map((change) => change.key)) if (!ranActions && changes.some(({ key }) => key === 1)) { ranActions = true @@ -219,6 +219,49 @@ const { multiplier, ...replay } = readOracleRunConfig() const generatedRuns = 30 * multiplier describe(`sync publication reentrancy`, () => { + it(`keeps explicit sync transaction handles bound to their reserved owners`, async () => { + const harness = createSyncHarness(`owned-sync-transactions`) + const { collection, sync } = harness + + try { + const first = sync.begin() + const second = sync.begin() + expect(first).toBeDefined() + expect(second).toBeDefined() + first!.write({ type: `insert`, value: { id: 1, value: `first` } }) + second!.write({ type: `insert`, value: { id: 2, value: `second` } }) + + expect( + collection._state.pendingSyncedTransactions.map((transaction) => + transaction.operations.map((operation) => operation.key), + ), + ).toEqual([[1], [2]]) + + first!.commit() + expect([...collection.keys()]).toEqual([1]) + expect(collection._state.pendingSyncedTransactions).toHaveLength(1) + + second!.truncate() + second!.write({ type: `insert`, value: { id: 2, value: `replacement` } }) + expect(collection._state.pendingSyncedTransactions[0]).toMatchObject({ + truncate: true, + operations: [{ key: 2 }], + }) + second!.commit() + expect([...collection.entries()]).toEqual([ + [2, expect.objectContaining({ id: 2, value: `replacement` })], + ]) + + const abandoned = sync.begin() + abandoned!.write({ type: `insert`, value: { id: 3, value: `abandoned` } }) + abandoned!.abort() + expect(collection._state.pendingSyncedTransactions).toHaveLength(0) + expect(collection.has(3)).toBe(false) + } finally { + await collection.cleanup() + } + }) + it(`publishes nested deferrals as one coherent batch`, async () => { const harness = createSyncHarness(`nested-publication-cycle`) const { collection } = harness @@ -404,7 +447,7 @@ describe(`sync publication reentrancy`, () => { const subscription = collection.subscribeChanges( (changes) => { callbacks.push({ - changes: changes.map(({ key }) => key as number), + changes: changes.map(({ key }) => key), keys: [...collection.keys()], values: collection.toArray.map(({ value }) => value), markedReceiptSettled: false, @@ -481,7 +524,7 @@ describe(`sync publication reentrancy`, () => { const subscription = collection.subscribeChanges( (changes) => { callbacks.push({ - changes: changes.map(({ key }) => key as number), + changes: changes.map(({ key }) => key), keys: [...collection.keys()], values: collection.toArray.map(({ value }) => value), }) @@ -573,7 +616,7 @@ describe(`sync publication reentrancy`, () => { const subscription = collection.subscribeChanges( (changes) => { callbacks.push({ - changes: changes.map(({ key }) => key as number), + changes: changes.map(({ key }) => key), keys: [...collection.keys()], values: collection.toArray.map(({ value }) => value), markedReceiptSettled: parkedReceiptSettled, @@ -656,7 +699,7 @@ describe(`sync publication reentrancy`, () => { const subscription = collection.subscribeChanges( (changes) => { callbacks.push({ - changes: changes.map(({ key }) => key as number), + changes: changes.map(({ key }) => key), keys: [...collection.keys()], values: collection.toArray.map(({ value }) => value), markedReceiptSettled, @@ -750,7 +793,7 @@ describe(`sync publication reentrancy`, () => { const subscription = collection.subscribeChanges( (changes) => { callbacks.push({ - changes: changes.map(({ key }) => key as number), + changes: changes.map(({ key }) => key), keys: [...collection.keys()], values: collection.toArray.map(({ value }) => value), markedReceiptSettled: firstReceiptSettled, @@ -879,7 +922,7 @@ describe(`sync publication reentrancy`, () => { const subscription = collection.subscribeChanges( (changes) => { callbacks.push({ - changes: changes.map(({ key }) => key as number), + changes: changes.map(({ key }) => key), keys: [...collection.keys()], values: collection.toArray.map(({ value }) => value), markedReceiptSettled, @@ -985,7 +1028,7 @@ describe(`sync publication reentrancy`, () => { listenerDepth++ maxListenerDepth = Math.max(maxListenerDepth, listenerDepth) callbacks.push({ - changes: changes.map(({ key }) => key as number), + changes: changes.map(({ key }) => key), keys: [...collection.keys()], values: collection.toArray.map(({ value }) => value), markedReceiptSettled: innerReceiptSettled, @@ -1060,7 +1103,7 @@ describe(`sync publication reentrancy`, () => { const batches: Array> = [] const subscription = collection.subscribeChanges((changes) => { - batches.push(changes.map((change) => change.key as number)) + batches.push(changes.map((change) => change.key)) if (!openedInnerTransaction && changes.some(({ key }) => key === 1)) { openedInnerTransaction = true stageInsert(harness.sync, { id: 2, value: `inner` }) @@ -1104,7 +1147,7 @@ describe(`sync publication reentrancy`, () => { const subscription = collection.subscribeChanges((changes) => { listenerDepth++ maxListenerDepth = Math.max(maxListenerDepth, listenerDepth) - batches.push(changes.map((change) => change.key as number)) + batches.push(changes.map((change) => change.key)) if (!committedInnerTransaction && changes.some(({ key }) => key === 1)) { committedInnerTransaction = true @@ -1135,7 +1178,7 @@ describe(`sync publication reentrancy`, () => { let ranListenerActions = false const subscription = collection.subscribeChanges((changes) => { - batches.push(changes.map((change) => change.key as number)) + batches.push(changes.map((change) => change.key)) if (ranListenerActions || !changes.some(({ key }) => key === 1)) return ranListenerActions = true @@ -1184,7 +1227,7 @@ describe(`sync publication reentrancy`, () => { let stagedInnerTransactions = false const subscription = collection.subscribeChanges((changes) => { - batches.push(changes.map((change) => change.key as number)) + batches.push(changes.map((change) => change.key)) if (stagedInnerTransactions || !changes.some(({ key }) => key === 1)) { return } @@ -1334,7 +1377,7 @@ describe(`sync publication reentrancy`, () => { (changes) => { listenerDepth++ maxListenerDepth = Math.max(maxListenerDepth, listenerDepth) - batches.push(changes.map((change) => change.key as number)) + batches.push(changes.map((change) => change.key)) listenerDepth-- }, { includeInitialState: true }, diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts index 00b8eab32..4f576ad7c 100644 --- a/packages/db/tests/oracle-config.ts +++ b/packages/db/tests/oracle-config.ts @@ -19,6 +19,10 @@ const staticOracleProperties = [ `cursor-pagination.reset`, `cursor-pagination.failure`, `oracle-replay.calibration`, + `sqlite-persistence.immediate-order`, + `sqlite-persistence.abort-graph`, + `sqlite-persistence.owner-isolation`, + `sqlite-persistence.reservation-boundary`, `trailbase.lifecycle`, `electric.bound-descriptor-history`, `electric.persisted-tag-history`, diff --git a/packages/db/tests/oracle-replay-manifest.ts b/packages/db/tests/oracle-replay-manifest.ts index 7b1eeae86..807d1fde5 100644 --- a/packages/db/tests/oracle-replay-manifest.ts +++ b/packages/db/tests/oracle-replay-manifest.ts @@ -11,6 +11,11 @@ const ownerGroups: ReadonlyArray = [ [`db/tests/cleanup-queue.property.test.ts`, `cleanup-queue`, `history`], [`db/tests/SortedMap.test.ts`, `sorted-map`, `key ascending descending`], [`db/tests/oracle-replay.fixture.test.ts`, `oracle-replay`, `calibration`], + [ + `db-sqlite-persistence-core/tests/persisted.test.ts`, + `sqlite-persistence`, + `immediate-order abort-graph owner-isolation reservation-boundary`, + ], [ `db/tests/optimistic-transaction-oracle.property.test.ts`, `collection-state`, diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index dd1d2d584..b4544fdd8 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -1935,6 +1935,12 @@ function createElectricSync>( !isResettingSnapshot const bufferedMessages: Array> = [] // Buffer change messages during initial sync + // Presence belongs to the source transaction, which can span multiple + // ShapeStream callbacks. Rebuild it only when no transaction is open; + // otherwise later callbacks must see writes staged by earlier ones. + const pendingPresence = new Map() + let usesBaseline = true + // Track keys that have been synced to handle overlapping subset queries. // When multiple subset queries return the same row, the server sends `insert` // for each response. We convert subsequent inserts to updates to avoid @@ -2102,6 +2108,8 @@ function createElectricSync>( begin() transactionStarted = true truncate() + pendingPresence.clear() + usesBaseline = false syncedKeys.clear() clearTagTrackingState() isResettingSnapshot = true @@ -2111,23 +2119,25 @@ function createElectricSync>( // Applied rows can also arrive through persistence invalidations. // Overlay only unapplied writes, once per callback rather than once // per message. A queued truncate fences off the previous snapshot. - const pendingPresence = new Map() - let usesBaseline = true - for (const pending of collection._state.pendingSyncedTransactions) { - if (pending.truncate) { - pendingPresence.clear() - usesBaseline = false - } - for (const operation of pending.operations) { - pendingPresence.set(operation.key, operation.type !== `delete`) + if (!transactionStarted) { + pendingPresence.clear() + usesBaseline = true + for (const pending of collection._state.pendingSyncedTransactions) { + if (pending.truncate) { + pendingPresence.clear() + usesBaseline = false + } + for (const operation of pending.operations) { + pendingPresence.set(operation.key, operation.type !== `delete`) + } } - } - for (const message of bufferedMessages) { - if (isChangeMessage(message)) { - pendingPresence.set( - collection.getKeyFromItem(message.value), - message.headers.operation !== `delete`, - ) + for (const message of bufferedMessages) { + if (isChangeMessage(message)) { + pendingPresence.set( + collection.getKeyFromItem(message.value), + message.headers.operation !== `delete`, + ) + } } } @@ -2378,7 +2388,15 @@ function createElectricSync>( void applied.then( () => wrappedMarkReady(wasBufferingInitialSync, readyErrorVersion), - () => undefined, + (error: unknown) => { + if (!isActiveLifecycle() || abortController.signal.aborted) { + return + } + streamErrorVersion++ + unsubscribeStream() + abortController.abort() + if (collection.status !== `error`) markError(error) + }, ) } diff --git a/packages/electric-db-collection/tests/electric-descriptor-isolation.test.ts b/packages/electric-db-collection/tests/electric-descriptor-isolation.test.ts index e2e63cef0..a7cbd5590 100644 --- a/packages/electric-db-collection/tests/electric-descriptor-isolation.test.ts +++ b/packages/electric-db-collection/tests/electric-descriptor-isolation.test.ts @@ -10,9 +10,41 @@ import type { Message } from '@electric-sql/client' import type { PersistenceAdapter } from '../../db-sqlite-persistence-core/src' import type { ElectricCollectionUtils } from '../src/electric' +/** + * # Can reused Electric descriptors keep independent owners and tag state? + * + * One descriptor may create several Collections, but each Collection must own + * its sync run, acknowledgement waiters, persisted tag membership, and cleanup. + * A compatible resume restores selected tags; a fresh snapshot replaces them. + * A `move-out` removes a row only after its modeled tag membership is empty. + * + * Plain Maps and tag sets form the independent model. Generated histories vary + * descriptor form, sync mode, warm and cold restart, interrupted recovery, + * edits, and tag removals. The driver uses the real Collection, Electric + * adapter, persisted wrapper, and a controlled ShapeStream SDK boundary. + * Checkpoints compare coherent public snapshots, durable rows, recovery traces, + * acknowledgement ownership, and unsubscribe calls. + * + * Fixed and random campaigns retain replay inputs through the shared oracle + * configuration. The controlled stream does not establish live Electric HTTP + * framing or service behavior; those have separate owners. + */ + type TestRow = { id: number; name: string; stable: string } type TagExposure = { cut: string; rows: Array } +function expectMoveOutCheckpoint(observation: { + status: string + publicRowPresent: boolean + durableRowPresent: boolean +}) { + expect(observation).toEqual({ + status: `ready`, + publicRowPresent: false, + durableRowPresent: false, + }) +} + function expectWholeTagRecovery( entries: Array, allowed: Array>, @@ -214,8 +246,12 @@ async function runTagHistory(history: { })) const durableRows = () => [...rows.values()].map((entry) => entry.value) const check = async () => { - expect(publicRows(), `cold=${cold}, fresh=${fresh}`).toEqual( - expectedRows(), + await vi.waitFor( + () => + expect(publicRows(), `cold=${cold}, fresh=${fresh}`).toEqual( + expectedRows(), + ), + { interval: 1 }, ) await vi.waitFor(() => expect(durableRows()).toEqual(expectedRows()), { interval: 1, @@ -602,6 +638,21 @@ it(`keeps insert acknowledgements on the owner of a reused persisted descriptor` } }) +it(`rejects a descriptor move-out checkpoint that deletes rows by fail-stopping`, () => { + expectMoveOutCheckpoint({ + status: `ready`, + publicRowPresent: false, + durableRowPresent: false, + }) + expect(() => + expectMoveOutCheckpoint({ + status: `error`, + publicRowPresent: false, + durableRowPresent: false, + }), + ).toThrow() +}) + it.each([`resume`, `fresh`] as const)( `restores compatible tags and discards obsolete tags on persisted $0 restart`, async (restart) => { @@ -646,9 +697,14 @@ it.each([`resume`, `fresh`] as const)( if (restart === `fresh`) streams[1]!.send([insert(1, currentTag), upToDate]) streams[1]!.send([moveOut(currentTag), upToDate]) - await vi.waitFor(() => expect(collection.status).toBe(`ready`)) - expect(collection.has(1)).toBe(false) + await vi.waitFor(() => expect(collection.has(1)).toBe(false)) await vi.waitFor(() => expect(rows.has(1)).toBe(false)) + await vi.waitFor(() => expect(collection.status).toBe(`ready`)) + expectMoveOutCheckpoint({ + status: collection.status, + publicRowPresent: collection.has(1), + durableRowPresent: rows.has(1), + }) } finally { await collection.cleanup() } diff --git a/packages/electric-db-collection/tests/electric-oracle.property.test.ts b/packages/electric-db-collection/tests/electric-oracle.property.test.ts index eb208a5d5..430d5449f 100644 --- a/packages/electric-db-collection/tests/electric-oracle.property.test.ts +++ b/packages/electric-db-collection/tests/electric-oracle.property.test.ts @@ -4,11 +4,14 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { IR, createCollection, createTransaction } from '@tanstack/db' import { ShapeStream } from '@electric-sql/client' import { QueryClient } from '@tanstack/query-core' -import { persistedCollectionOptions } from '../../db-sqlite-persistence-core/src' +import { + PersistenceDurabilityError, + persistedCollectionOptions, +} from '../../db-sqlite-persistence-core/src' import { queryCollectionOptions } from '../../query-db-collection/src/query' import { electricCollectionOptions } from '../src/electric' import { oraclePropertyOptions, oracleRuns } from '../../db/tests/oracle-config' -import { withElectricCleanup } from './electric-oracle-lifecycle' +import { atCheckpoint, withElectricCleanup } from './electric-oracle-lifecycle' import type { Collection, SyncMetadataApi } from '@tanstack/db' import type { ChangeMessage, Message, Offset, Row } from '@electric-sql/client' import type { @@ -23,22 +26,23 @@ import type { ElectricCollectionUtils, ElectricSyncMode } from '../src/electric' * Electric delivers change messages, reset controls, offsets, handles, and * snapshot availability through an SDK stream. The adapter must publish atomic * callback cuts, wait for applied receipts before readiness, persist valid - * resume evidence, reject unseen partial updates, and keep stale callbacks and - * metadata scoped to the lifecycle that created them. + * resume evidence for a valid source prefix, reject unseen partial updates, + * and keep stale callbacks and metadata scoped to the sync run that created + * them. * - * Independent Maps model rows, selected-tag membership, resume evidence, and - * persisted state. A controlled ShapeStream boundary supplies authored message - * partitions and settlement order. The production Collection, Query-backed - * persistence wrapper, metadata APIs, and cleanup all run unchanged. Checks - * cover rows, exact batches, readiness, errors, metadata, waiters, stream - * ownership, and restart. + * Independent Maps model rows, selected-tag membership, source commits, resume + * evidence, and persisted state. A controlled ShapeStream boundary supplies + * authored message partitions and settlement order. The production Collection, + * Query-backed persistence wrapper, metadata APIs, and cleanup run unchanged. + * Checks cover public and durable rows, exact batches, readiness, errors, + * metadata, waiters, stream ownership, and restart at named checkpoints. * - * The suite uses fixed protocol histories, exhaustive contiguous partitions, - * and fixed plus random generated campaigns. The installed-SDK delivery oracle - * separately owns real HTTP framing and pause behavior; PostgreSQL expression - * semantics and service-backed execution remain separate evidence. + * The suite retains fixed protocol histories, exhaustive contiguous + * partitions, fixed campaigns, random campaigns, and explicit seed/path replay + * for durability policies. Sensitivity checks reject missing prefix evidence. + * Installed-SDK HTTP delivery, PostgreSQL semantics, and live-service execution + * remain separate evidence. */ - type OracleRow = Row & { id: number name: string @@ -207,6 +211,194 @@ function createPersistedAdapter( } } +type SourceLedgerEvent = { + type: `source-commit` + offset: string + mutations: Array< + { type: `set`; row: OracleRow } | { type: `delete`; key: string | number } + > +} + +// A persisted resume offset is a claim about an append-only source prefix, not +// about whichever adapter calls happened to succeed. This fold shares neither +// Electric's callback state nor the persistence runtime's queue. The fixture +// runs the real Electric collection adapter with the installed SDK boundary +// mocked at ShapeStream; it does not earn a live Electric service receipt. +function foldSourceLedgerThrough( + events: ReadonlyArray, + resumeOffset: string, +): Map { + const rows = new Map() + for (const event of events) { + for (const mutation of event.mutations) { + if (mutation.type === `delete`) rows.delete(mutation.key) + else rows.set(mutation.row.id, structuredClone(mutation.row)) + } + if (event.offset === resumeOffset) return rows + } + throw new Error(`resume offset ${resumeOffset} is absent from source ledger`) +} + +function durablePrefixPropertyOptions(): { + numRuns: number + seed: number + path?: string +} { + const seedValue = process.env.TANSTACK_DB_ELECTRIC_DURABILITY_SEED + const path = process.env.TANSTACK_DB_ELECTRIC_DURABILITY_PATH + if (path !== undefined && seedValue === undefined) { + throw new Error( + `TANSTACK_DB_ELECTRIC_DURABILITY_PATH requires TANSTACK_DB_ELECTRIC_DURABILITY_SEED`, + ) + } + const seed = seedValue === undefined ? 20260921 : Number(seedValue) + if (!Number.isSafeInteger(seed)) { + throw new Error(`TANSTACK_DB_ELECTRIC_DURABILITY_SEED must be an integer`) + } + return { + numRuns: path === undefined ? 10 : 1, + seed, + ...(path === undefined ? {} : { path }), + } +} + +function persistencePolicyPropertyOptions(): { + numRuns: number + seed: number + path?: string +} { + const seedValue = process.env.TANSTACK_DB_ELECTRIC_PERSISTENCE_POLICY_SEED + const path = process.env.TANSTACK_DB_ELECTRIC_PERSISTENCE_POLICY_PATH + if (path !== undefined && seedValue === undefined) { + throw new Error( + `TANSTACK_DB_ELECTRIC_PERSISTENCE_POLICY_PATH requires TANSTACK_DB_ELECTRIC_PERSISTENCE_POLICY_SEED`, + ) + } + const seed = seedValue === undefined ? 20260923 : Number(seedValue) + if (!Number.isSafeInteger(seed)) { + throw new Error( + `TANSTACK_DB_ELECTRIC_PERSISTENCE_POLICY_SEED must be an integer`, + ) + } + return { + numRuns: path === undefined ? 8 : 1, + seed, + ...(path === undefined ? {} : { path }), + } +} + +async function runRejectedDurablePrefixWitness( + id: string, + names: readonly [string, string], +): Promise { + let subscriber: ((messages: Array>) => void) | undefined + mockSubscribe.mockImplementationOnce((callback) => { + subscriber = callback + return vi.fn() + }) + const firstAttempt = createDeferred() + const persistedMetadata = new Map() + const persistedRows = new Map() + const adapter = createPersistedAdapter(persistedMetadata, persistedRows) + const applyCommittedTx = adapter.applyCommittedTx.bind(adapter) + const persistenceError = new Error(`first persistence attempt rejected`) + const durableAttempts: Array<{ + txId: string + keys: Array + sourceTxids: Array + }> = [] + let attempt = 0 + adapter.applyCommittedTx = async (...args) => { + const tx = args[1] + durableAttempts.push({ + txId: tx.txId, + keys: tx.mutations.map((mutation) => mutation.key), + sourceTxids: (tx.rowMetadataMutations ?? []).flatMap((mutation) => { + if (mutation.type === `delete`) return [] + const metadata = mutation.value as { txids?: Array } + return metadata.txids ?? [] + }), + }) + attempt++ + if (attempt === 1) { + firstAttempt.resolve() + throw persistenceError + } + await applyCommittedTx(...args) + } + + const collection = createCollection( + persistedCollectionOptions< + OracleRow, + string | number, + never, + ElectricCollectionUtils + >({ + ...electricCollectionOptions({ + id, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + syncMode: `progressive`, + getKey: (row) => row.id, + startSync: true, + }), + persistence: { adapter }, + }), + ) + await withElectricCleanup(async () => { + collection.startSyncImmediate() + await vi.waitFor(() => expect(subscriber).toBeTypeOf(`function`), { + interval: 1, + timeout: 250, + }) + + const firstRow: OracleRow = { + id: 1, + name: names[0], + stable: `stable-1`, + } + mockStream.lastOffset = `21_0` + const firstMessage = change(`insert`, firstRow.id, firstRow.name) + firstMessage.headers.txids = [701] + subscriber!([firstMessage, upToDate]) + await atCheckpoint(firstAttempt.promise, `first durability rejection`) + await new Promise((resolve) => setTimeout(resolve, 0)) + + const secondRow: OracleRow = { + id: 2, + name: names[1], + stable: `stable-2`, + } + mockStream.lastOffset = `22_0` + const secondMessage = change(`insert`, secondRow.id, secondRow.name) + secondMessage.headers.txids = [702] + subscriber!([secondMessage, upToDate]) + await new Promise((resolve) => setTimeout(resolve, 0)) + const statusAtTerminalCut = collection.status + await atCheckpoint(collection.cleanup(), `durable-prefix terminal cleanup`) + + const durableResume = observableResume( + persistedMetadata.get(`electric:resume`), + ) + const durableOffset = + durableResume && + typeof durableResume === `object` && + `offset` in durableResume + ? durableResume.offset + : undefined + expect(durableAttempts).toHaveLength(1) + expect(durableAttempts[0]?.keys).toContain(1) + expect(durableAttempts[0]?.sourceTxids).toEqual([701]) + // A rejected durability boundary is terminal: it cannot retry, admit the + // later source commit, or publish a resume claim for an undurable prefix. + expect(statusAtTerminalCut).toBe(`error`) + expect(durableOffset).toBeUndefined() + expect(rowsFromMap(persistedRows)).toEqual([]) + }, [() => collection.cleanup()]) +} + function createOracleCollection( id: string, syncMode: ElectricSyncMode, @@ -285,6 +477,133 @@ function rowsFromMap( ]).sort(([left], [right]) => String(left).localeCompare(String(right))) } +type QueuedPresenceHistory = + | `insert-update` + | `update-update` + | `delete-update` + | `truncate-update` + +let queuedPresenceHistoryId = 0 + +async function runQueuedPresenceHistory( + history: QueuedPresenceHistory, + queuedName: string, + laterName: string, +): Promise { + const historyId = ++queuedPresenceHistoryId + let subscriber: ((messages: Array>) => void) | undefined + mockSubscribe.mockImplementationOnce((callback) => { + subscriber = callback + return vi.fn() + }) + const firstPersistenceEntered = createDeferred() + const releaseFirstPersistence = createDeferred() + const persistedMetadata = new Map() + const persistedRows = new Map() + const adapter = createPersistedAdapter(persistedMetadata, persistedRows) + const applyCommittedTx = adapter.applyCommittedTx.bind(adapter) + let heldGate = false + adapter.applyCommittedTx = async (...args) => { + if (!heldGate && args[1].mutations.some((mutation) => mutation.key === 1)) { + heldGate = true + firstPersistenceEntered.resolve() + await releaseFirstPersistence.promise + } + await applyCommittedTx(...args) + } + const collection = createCollection( + persistedCollectionOptions< + OracleRow, + string | number, + never, + ElectricCollectionUtils + >({ + ...electricCollectionOptions({ + id: `generated-queued-presence-${historyId}`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + syncMode: `progressive`, + getKey: (row) => row.id, + startSync: true, + }), + persistence: { adapter }, + }), + ) + + await withElectricCleanup(async () => { + await vi.waitFor(() => expect(subscriber).toBeTypeOf(`function`), { + interval: 1, + timeout: 250, + }) + mockStream.lastOffset = `100_0` + subscriber!([upToDate]) + await vi.waitFor(() => expect(collection.status).toBe(`ready`), { + interval: 1, + timeout: 250, + }) + + if (history !== `insert-update`) { + const baseline = change(`insert`, 2, `baseline`) + baseline.value = { id: 2, name: `baseline`, stable: `stable-2` } + mockStream.lastOffset = `101_0` + subscriber!([baseline, upToDate]) + await vi.waitFor( + () => expect(persistedRows.get(2)).toEqual(baseline.value), + { interval: 1, timeout: 250 }, + ) + } + + const gate = change(`insert`, 1, `holds persistence`) + gate.value = { id: 1, name: `holds persistence`, stable: `stable-1` } + mockStream.lastOffset = `102_0` + subscriber!([gate, upToDate]) + await atCheckpoint( + firstPersistenceEntered.promise, + `generated Electric persistence hold`, + ) + + mockStream.lastOffset = `103_0` + if (history === `truncate-update`) { + subscriber!([mustRefetch, upToDate]) + } else { + const operation = history.split(`-`)[0] as `insert` | `update` | `delete` + const queued = change(operation, 2, queuedName) + if (operation === `insert`) { + queued.value = { id: 2, name: queuedName, stable: `stable-2` } + } + subscriber!([queued, upToDate]) + } + + const laterUpdate = change(`update`, 2, laterName) + mockStream.lastOffset = `104_0` + subscriber!([laterUpdate, upToDate]) + + const expectedRows = new Map() + if (history !== `truncate-update`) expectedRows.set(1, gate.value) + if (history === `insert-update` || history === `update-update`) { + expectedRows.set(2, { + id: 2, + name: laterName, + stable: `stable-2`, + }) + } + + releaseFirstPersistence.resolve() + await vi.waitFor( + () => + expect(rowsFromMap(persistedRows)).toEqual(rowsFromMap(expectedRows)), + { interval: 1, timeout: 250 }, + ) + await vi.waitFor(() => expect(collection.status).toBe(`ready`), { + interval: 1, + timeout: 250, + }) + expect(rowsFromCollection(collection)).toEqual(rowsFromMap(expectedRows)) + }, [() => releaseFirstPersistence.resolve(), () => collection.cleanup()]) +} + function applyReferenceBatch( state: ReferenceState, batch: ReadonlyArray>, @@ -875,6 +1194,261 @@ function expectPreloadOutcome( } } +async function runPublicationBeforePersistenceWitness( + id: string, + row: OracleRow, + txid: number, +): Promise { + let subscriber: ((messages: Array>) => void) | undefined + mockSubscribe.mockImplementationOnce((callback) => { + subscriber = callback + return vi.fn() + }) + const persistenceEntered = createDeferred() + const persistenceCompleted = createDeferred() + const releasePersistence = createDeferred() + const persistedMetadata = new Map() + const persistedRows = new Map() + const adapter = createPersistedAdapter(persistedMetadata, persistedRows) + const applyCommittedTx = adapter.applyCommittedTx.bind(adapter) + adapter.applyCommittedTx = async (...args) => { + persistenceEntered.resolve() + await releasePersistence.promise + await applyCommittedTx(...args) + persistenceCompleted.resolve() + } + + const collection = createCollection( + persistedCollectionOptions< + OracleRow, + string | number, + never, + ElectricCollectionUtils + >({ + ...electricCollectionOptions({ + id, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + syncMode: `progressive`, + getKey: (value) => value.id, + startSync: true, + }), + persistence: { adapter }, + }), + ) + const publications = observePublications(collection) + + await withElectricCleanup(async () => { + await vi.waitFor(() => expect(subscriber).toBeTypeOf(`function`), { + interval: 1, + timeout: 250, + }) + const txidEvidence = collection.utils.awaitTxId(txid, 250) + mockStream.lastOffset = `${txid}_0` + const message = change(`insert`, row.id, row.name) + message.value = structuredClone(row) + message.headers.txids = [txid] + subscriber!([message, upToDate]) + + await atCheckpoint(persistenceEntered.promise, `${id} persistence entered`) + publications.record(`persistence-pending`) + + expect(rowsFromCollection(collection)).toEqual( + rowsFromMap(new Map([[row.id, row]])), + ) + expect(publications.entries).toContainEqual({ + cut: `event`, + rows: rowsFromMap(new Map([[row.id, row]])), + }) + expect(publications.entries).toContainEqual({ + cut: `persistence-pending`, + rows: rowsFromMap(new Map([[row.id, row]])), + }) + expect(rowsFromMap(persistedRows)).toEqual([]) + await expect(txidEvidence).resolves.toBe(true) + + releasePersistence.resolve() + await atCheckpoint( + persistenceCompleted.promise, + `${id} persistence completed`, + ) + expect(rowsFromMap(persistedRows)).toEqual( + rowsFromMap(new Map([[row.id, row]])), + ) + }, [ + () => { + releasePersistence.resolve() + publications.stop() + }, + () => collection.cleanup(), + ]) +} + +async function runPersistenceFailureCrashOnlyWitness( + id: string, + firstRow: OracleRow, + secondRow: OracleRow, +): Promise { + let subscriber: ((messages: Array>) => void) | undefined + mockSubscribe.mockImplementationOnce((callback) => { + subscriber = callback + return vi.fn() + }) + const persistenceEntered = createDeferred() + let rejectPersistence!: (reason: unknown) => void + const persistence = new Promise((_resolve, reject) => { + rejectPersistence = reject + }) + const persistenceRejected = createDeferred() + const rawPersistenceError = Object.assign( + new Error(`adapter durability rejected exactly`), + { + code: `SQLITE_IOERR`, + path: `electric.adapter.applyCommittedTx`, + }, + ) + const persistedMetadata = new Map() + const persistedRows = new Map() + const adapter = createPersistedAdapter(persistedMetadata, persistedRows) + let persistenceAttempts = 0 + adapter.applyCommittedTx = () => { + persistenceAttempts++ + persistenceEntered.resolve() + return persistence.finally(() => persistenceRejected.resolve()) + } + + const collection = createCollection( + persistedCollectionOptions< + OracleRow, + string | number, + never, + ElectricCollectionUtils + >({ + ...electricCollectionOptions({ + id, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + syncMode: `progressive`, + getKey: (value) => value.id, + startSync: true, + }), + persistence: { adapter }, + }), + ) + const publications = observePublications(collection) + + await withElectricCleanup(async () => { + await vi.waitFor(() => expect(subscriber).toBeTypeOf(`function`), { + interval: 1, + timeout: 250, + }) + mockStream.lastOffset = `31_0` + const firstMessage = change(`insert`, firstRow.id, firstRow.name) + firstMessage.value = structuredClone(firstRow) + firstMessage.headers.txids = [731] + subscriber!([firstMessage, upToDate]) + await atCheckpoint( + persistenceEntered.promise, + `${id} adapter persistence entered`, + ) + + // Visibility is the earlier boundary. This exact pending cut kills an + // implementation that rejects before publishing merely because it knows + // persistence will fail later. + expect({ + status: collection.status, + publicError: collection._lifecycle.getSyncError(), + visibleRows: rowsFromCollection(collection), + durableRows: rowsFromMap(persistedRows), + published: publications.entries.some(({ rows }) => + isDeepStrictEqual( + rows, + rowsFromMap(new Map([[firstRow.id, firstRow]])), + ), + ), + }).toEqual({ + status: `ready`, + publicError: undefined, + visibleRows: rowsFromMap(new Map([[firstRow.id, firstRow]])), + durableRows: [], + published: true, + }) + + rejectPersistence(rawPersistenceError) + await atCheckpoint( + persistenceRejected.promise, + `${id} adapter persistence rejection`, + ) + await new Promise((resolve) => setTimeout(resolve, 0)) + + const statusAtAdmissionCut = collection.status + const publicErrorAtAdmissionCut = collection._lifecycle.getSyncError() + const publicationsBeforeRetiredWork = structuredClone(publications.entries) + mockStream.lastOffset = `32_0` + const secondMessage = change(`insert`, secondRow.id, secondRow.name) + secondMessage.value = structuredClone(secondRow) + secondMessage.headers.txids = [732] + subscriber!([secondMessage, upToDate]) + await new Promise((resolve) => setTimeout(resolve, 0)) + + const publicPersistenceError = collection._lifecycle.getSyncError() + const durabilityError = + publicPersistenceError instanceof Error + ? (publicPersistenceError as Error & { + code?: unknown + path?: unknown + }) + : undefined + expect({ + statusAtAdmissionCut, + publicErrorNameAtAdmissionCut: + publicErrorAtAdmissionCut instanceof Error + ? publicErrorAtAdmissionCut.name + : undefined, + status: collection.status, + publicErrorName: + publicPersistenceError instanceof Error + ? publicPersistenceError.name + : undefined, + hasNamedPersistenceSemantics: + publicPersistenceError instanceof PersistenceDurabilityError, + preservesCause: durabilityError?.cause === rawPersistenceError, + persistenceErrorCode: durabilityError?.code, + persistenceErrorPath: durabilityError?.path, + persistenceAttempts, + visibleRows: rowsFromCollection(collection), + durableRows: rowsFromMap(persistedRows), + publicationsUnchanged: isDeepStrictEqual( + publications.entries, + publicationsBeforeRetiredWork, + ), + }).toEqual({ + statusAtAdmissionCut: `error`, + publicErrorNameAtAdmissionCut: `PersistenceDurabilityError`, + status: `error`, + publicErrorName: `PersistenceDurabilityError`, + hasNamedPersistenceSemantics: true, + preservesCause: true, + persistenceErrorCode: `SQLITE_IOERR`, + persistenceErrorPath: `electric.adapter.applyCommittedTx`, + persistenceAttempts: 1, + visibleRows: rowsFromMap(new Map([[firstRow.id, firstRow]])), + durableRows: [], + publicationsUnchanged: true, + }) + }, [ + () => { + rejectPersistence(rawPersistenceError) + publications.stop() + }, + () => collection.cleanup(), + ]) +} + const processCommandArb: fc.Arbitrary = fc.oneof( fc.record({ kind: fc.constant(`create` as const), @@ -1574,6 +2148,7 @@ async function runSchedulerPermutation( describe(`Electric adapter laws`, () => { let processGrammarRun = 0 + let persistencePolicyRun = 0 beforeEach(() => { vi.clearAllMocks() @@ -1809,6 +2384,182 @@ describe(`Electric adapter laws`, () => { }, ) + it(`publishes an Electric source commit before its adapter persistence settles`, async () => { + await runPublicationBeforePersistenceWitness( + `publication-before-persistence-fixed`, + { id: 41, name: `visible first`, stable: `stable-41` }, + 741, + ) + }) + + it(`keeps queued persisted writes visible to later Electric callbacks`, async () => { + let subscriber: ((messages: Array>) => void) | undefined + mockSubscribe.mockImplementationOnce((callback) => { + subscriber = callback + return vi.fn() + }) + const firstPersistenceEntered = createDeferred() + const releaseFirstPersistence = createDeferred() + const persistedMetadata = new Map() + const persistedRows = new Map() + const adapter = createPersistedAdapter(persistedMetadata, persistedRows) + const applyCommittedTx = adapter.applyCommittedTx.bind(adapter) + let heldFirstRow = false + adapter.applyCommittedTx = async (...args) => { + if ( + !heldFirstRow && + args[1].mutations.some((mutation) => mutation.key === 1) + ) { + heldFirstRow = true + firstPersistenceEntered.resolve() + await releaseFirstPersistence.promise + } + await applyCommittedTx(...args) + } + const collection = createCollection( + persistedCollectionOptions< + OracleRow, + string | number, + never, + ElectricCollectionUtils + >({ + ...electricCollectionOptions({ + id: `queued-persisted-presence`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + syncMode: `progressive`, + getKey: (row) => row.id, + startSync: true, + }), + persistence: { adapter }, + }), + ) + + await withElectricCleanup(async () => { + await vi.waitFor(() => expect(subscriber).toBeTypeOf(`function`), { + interval: 1, + timeout: 250, + }) + mockStream.lastOffset = `90_0` + subscriber!([upToDate]) + await vi.waitFor(() => expect(collection.status).toBe(`ready`), { + interval: 1, + timeout: 250, + }) + + const first = change(`insert`, 1, `holds persistence`) + first.value = { id: 1, name: `holds persistence`, stable: `stable-1` } + mockStream.lastOffset = `91_0` + subscriber!([first, upToDate]) + await atCheckpoint( + firstPersistenceEntered.promise, + `first Electric persistence entered`, + ) + + const queued = change(`insert`, 2, `queued insert`) + queued.value = { id: 2, name: `queued insert`, stable: `stable-2` } + mockStream.lastOffset = `92_0` + subscriber!([queued, upToDate]) + + const laterUpdate = change(`update`, 2, `later update`) + laterUpdate.value = { id: 2, name: `later update`, stable: `stable-2` } + mockStream.lastOffset = `93_0` + subscriber!([laterUpdate, upToDate]) + + releaseFirstPersistence.resolve() + await vi.waitFor( + () => + expect(rowsFromMap(persistedRows)).toEqual( + rowsFromMap( + new Map([ + [1, first.value], + [2, laterUpdate.value], + ]), + ), + ), + { interval: 1, timeout: 250 }, + ) + expect(rowsFromCollection(collection)).toEqual( + rowsFromMap( + new Map([ + [1, first.value], + [2, laterUpdate.value], + ]), + ), + ) + expect(collection.status).toBe(`ready`) + }, [() => releaseFirstPersistence.resolve(), () => collection.cleanup()]) + }) + + fcTest.prop( + [ + fc.uniqueArray( + fc.constantFrom( + `insert-update`, + `update-update`, + `delete-update`, + `truncate-update`, + ) as fc.Arbitrary, + { minLength: 4, maxLength: 4 }, + ), + fc.tuple(fc.string({ maxLength: 12 }), fc.string({ maxLength: 12 })), + ], + persistencePolicyPropertyOptions(), + )( + `generated queued Electric histories preserve staged presence and reset fences`, + async (histories, names) => { + for (const history of histories) { + await runQueuedPresenceHistory(history, names[0], names[1]) + } + }, + ) + + fcTest.prop( + [fc.integer({ min: 1, max: 20 }), fc.string({ maxLength: 12 })], + // Replay one generated publication schedule directly with: + // TANSTACK_DB_ELECTRIC_PERSISTENCE_POLICY_SEED=20260923 + // TANSTACK_DB_ELECTRIC_PERSISTENCE_POLICY_PATH= + persistencePolicyPropertyOptions(), + )( + `generated Electric commits publish before adapter persistence settles`, + async (id, name) => { + persistencePolicyRun++ + await runPublicationBeforePersistenceWitness( + `publication-before-persistence-generated-${persistencePolicyRun}`, + { id, name, stable: `stable-${id}` }, + 800 + id, + ) + }, + ) + + it(`turns adapter rejection into a named terminal Electric persistence error`, async () => { + await runPersistenceFailureCrashOnlyWitness( + `terminal-persistence-error-fixed`, + { id: 51, name: `published before failure`, stable: `stable-51` }, + { id: 52, name: `must not be admitted`, stable: `stable-52` }, + ) + }) + + fcTest.prop( + [ + fc.integer({ min: 1, max: 20 }), + fc.tuple(fc.string({ maxLength: 12 }), fc.string({ maxLength: 12 })), + ], + persistencePolicyPropertyOptions(), + )( + `generated adapter failures reject publicly and admit no later Electric work`, + async (id, names) => { + persistencePolicyRun++ + await runPersistenceFailureCrashOnlyWitness( + `terminal-persistence-error-generated-${persistencePolicyRun}`, + { id, name: names[0], stable: `stable-${id}` }, + { id: id + 100, name: names[1], stable: `stable-${id + 100}` }, + ) + }, + ) + fcTest.prop( [fc.array(designTokenArb, { minLength: 1, maxLength: 7 }), fc.nat()], { numRuns: 24 }, @@ -2192,11 +2943,121 @@ describe(`Electric adapter laws`, () => { }, ) + it(`keeps pending row presence across callbacks until one persisted subset commit`, async () => { + let subscriber!: (messages: Array>) => void + mockSubscribe.mockImplementationOnce((callback) => { + subscriber = callback + return vi.fn() + }) + const persistedRows = new Map() + const persistedMetadata = new Map() + const adapter = createPersistedAdapter(persistedMetadata, persistedRows) + const applyCommittedTx = adapter.applyCommittedTx.bind(adapter) + const durabilityEntered = createDeferred() + const releaseDurability = createDeferred() + const durabilitySettled = createDeferred() + let persistedTx: PersistedTx | undefined + adapter.applyCommittedTx = async (...args) => { + persistedTx = structuredClone(args[1]) + durabilityEntered.resolve() + await releaseDurability.promise + await applyCommittedTx(...args) + durabilitySettled.resolve() + } + const collection = createCollection( + persistedCollectionOptions< + OracleRow, + string | number, + never, + ElectricCollectionUtils + >({ + ...electricCollectionOptions({ + id: `persisted-cross-callback-presence`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + syncMode: `eager`, + getKey: (row) => row.id, + startSync: true, + }), + persistence: { adapter }, + }), + ) + + collection.startSyncImmediate() + await withElectricCleanup(async () => { + await vi.waitFor(() => expect(subscriber).toBeTypeOf(`function`)) + + subscriber([change(`insert`, 1, ``)]) + subscriber([change(`update`, 1, `updated across callbacks`)]) + subscriber([subsetEnd]) + await atCheckpoint( + durabilityEntered.promise, + `cross-callback durability entered`, + ) + + expect({ + publicRows: rowsFromCollection(collection), + durableRows: rowsFromMap(persistedRows), + persistedMutations: persistedTx?.mutations.map((mutation) => ({ + type: mutation.type, + key: mutation.key, + value: mutation.type === `delete` ? undefined : mutation.value, + })), + }).toEqual({ + publicRows: [[1, `updated across callbacks`, `stable-1`]], + durableRows: [], + persistedMutations: [ + { + type: `update`, + key: 1, + value: { + id: 1, + name: ``, + stable: `stable-1`, + }, + }, + { + type: `update`, + key: 1, + value: { + id: 1, + name: `updated across callbacks`, + }, + }, + ], + }) + + releaseDurability.resolve() + await atCheckpoint( + durabilitySettled.promise, + `cross-callback durability settled`, + ) + expect(rowsFromMap(persistedRows)).toEqual([ + [1, `updated across callbacks`, `stable-1`], + ]) + }, [ + async () => { + releaseDurability.resolve() + await collection.cleanup() + }, + ]) + }) + fcTest.prop( [fc.array(designTokenArb, { minLength: 1, maxLength: 7 }), fc.nat()], { numRuns: 20, examples: [ + [ + [ + { operation: `insert`, id: 1, name: `` }, + { operation: `update`, id: 1, name: ` ` }, + { operation: `insert`, id: 2, name: `` }, + ], + 14, + ], [ [ { operation: `insert`, id: 2, name: `` }, @@ -3600,6 +4461,62 @@ describe(`Electric adapter laws`, () => { await collection.cleanup() }) + it(`does not advance durable Electric resume metadata past a rejected row commit`, async () => { + await runRejectedDurablePrefixWitness(`durable-prefix-fixed`, [ + `first`, + `second`, + ]) + }) + + fcTest.prop( + [fc.tuple(fc.string({ maxLength: 12 }), fc.string({ maxLength: 12 }))], + // Replay the retained shrink directly with: + // TANSTACK_DB_ELECTRIC_DURABILITY_SEED=20260921 + // TANSTACK_DB_ELECTRIC_DURABILITY_PATH=0:0:0 + durablePrefixPropertyOptions(), + )( + `durable Electric resume metadata covers every generated source prefix after a rejected commit`, + async (names) => { + await runRejectedDurablePrefixWitness( + `durable-prefix-generated-${JSON.stringify(names)}`, + names, + ) + }, + ) + + fcTest.prop( + [ + fc.uniqueArray(fc.integer({ min: 1, max: 20 }), { + minLength: 2, + maxLength: 6, + }), + ], + { numRuns: 20, seed: 20260922 }, + )(`append-only resume judgment rejects a missing committed row`, (ids) => { + const events: Array = ids.map((id) => ({ + type: `source-commit`, + offset: `${id}_0`, + mutations: [ + { + type: `set`, + row: { id, name: `row-${id}`, stable: `stable-${id}` }, + }, + ], + })) + const expected = rowsFromMap( + foldSourceLedgerThrough(events, events.at(-1)!.offset), + ) + // An intermediate durable checkpoint contains exactly its source prefix. + const prefix = rowsFromMap( + foldSourceLedgerThrough(events, events[0]!.offset), + ) + expect(prefix).toEqual(expected.filter(([key]) => key === ids[0])) + // An offset outside the append-only ledger cannot justify resume state. + expect(() => foldSourceLedgerThrough(events, `absent_0`)).toThrow( + /absent from source ledger/, + ) + }) + it(`rehydrates persisted rows and resume metadata after cleanup and restart`, async () => { const subscribers: Array<(messages: Array>) => void> = [] mockSubscribe.mockImplementation((callback) => { diff --git a/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts b/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts index 8a56b825e..714772291 100644 --- a/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts +++ b/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts @@ -20,15 +20,20 @@ import type { ElectricCollectionUtils, ElectricSyncMode } from '../src/electric' * Hydrated rows and resume metadata provide the last complete public snapshot. * A must-refetch starts a private replacement. Until that replacement is fully * applied, readers may see an earlier permitted snapshot but never a torn mix. - * Failure keeps the old public rows and records repair debt; later success may - * replace them atomically. + * Failure keeps the prior public rows and records repair debt; later success + * may replace them atomically. * - * A plain persisted row Map and metadata Map form the reference snapshots. The - * driver controls hydration, SDK callbacks, applied receipts, cleanup, restart, - * and eager or progressive mode through the real persistence coordinator and - * Electric adapter. It records every exposed cut, not only the final rows. + * Plain persisted row and metadata Maps form the reference snapshots. The + * history grammar varies hydration position, empty state, sync mode, external + * invalidation, failure, cleanup, and restart. The driver controls hydration, + * SDK callbacks, applied receipts, cleanup, restart, and eager or progressive + * mode through the real persistence coordinator and Electric adapter. It + * records every exposed publication cut rather than only final rows. + * + * Trace controls reject torn, reverted, or omitted intermediate publications. + * The controlled SDK boundary does not establish HTTP framing, PostgreSQL + * expression semantics, or live-service recovery; those have separate owners. */ - type Item = Row & { id: number; name: string; stable: string } type Subscriber = (messages: Array>) => void type Exposure = { cut: string; rows: Array } @@ -227,9 +232,9 @@ describe(`persisted Electric recovery laws`, () => { f.record(`before`) subscribers[0]!([change(`update`, { id: 1, name: `wrong` }), upToDate]) subscribers[0]!([change(`update`, { id: 1, name: `correct` }), upToDate]) - f.record(`repaired`) const correct = [{ ...oldRow, name: `correct` }] - expect(f.publicRows()).toEqual(correct) + await vi.waitFor(() => expect(f.publicRows()).toEqual(correct)) + f.record(`repaired`) const entries = f.exposures.slice(cut) expect(entries[0]!.rows).toEqual([oldRow]) expect( diff --git a/packages/offline-transactions/tests/leadership-replay.property.test.ts b/packages/offline-transactions/tests/leadership-replay.property.test.ts index 700ea438c..6fc33f64a 100644 --- a/packages/offline-transactions/tests/leadership-replay.property.test.ts +++ b/packages/offline-transactions/tests/leadership-replay.property.test.ts @@ -17,13 +17,18 @@ import type { OfflineTransaction, OnlineDetector } from '../src/types' * Contract and source: OfflineExecutor leadership, TransactionExecutor serial * scheduling, and OutboxManager durability define this boundary. Only the * current leader may admit stored work. Issued work may settle after leadership - * loss, but loss or disposal forbids new provider work. A durable row remains - * owned until its acknowledgement or permanent rejection is durably removed. + * loss, but loss or disposal fences startup, stale reads, retry hooks, and new + * provider work. A durable row remains owned until its acknowledgement or + * permanent rejection is durably removed, and an ID already pending, running, + * completed, permanently rejected, or durably removed must not execute twice. * * Model: this file is a partial relational oracle, not a second executor. Each * history relates three independent projections: durable outbox IDs, scheduler * lifecycle, and caller settlement with optimistic rows. Expected relations * come from those contracts rather than production queues or classifiers. + * `foldReplayLedger` models stored and durably removed IDs without copying + * production revision counters, so stale captured reads cannot restore removed + * work in the model. * * History grammar: leadership may be acquired, lost, regained, or disposed at * construction, outbox read, retry hook, provider, acknowledgement, or retry @@ -46,7 +51,6 @@ import type { OfflineTransaction, OnlineDetector } from '../src/types' * storage engine. Exactly-once network execution across independent leaders is * not promised. Provider side effects after an issued call remain provider-owned. */ - function gate() { let resolve!: () => void const promise = new Promise((done) => { @@ -69,6 +73,36 @@ const storedTransaction = (id: string): OfflineTransaction => ({ version: 1, }) +type ReplayLedgerEvent = + | { type: `stored`; id: string } + | { type: `read-captured`; ids: Array } + | { type: `durably-removed`; id: string } + | { type: `read-delivered` } + +// The model records durable facts instead of copying OutboxManager's version +// fences. A stale captured value cannot undo a later durable removal. These +// fixtures use FakeStorageAdapter and therefore establish executor/outbox +// behavior, not IndexedDB, LocalStorage, or native transaction completion. +function foldReplayLedger(events: ReadonlyArray): { + unfinished: Array + staleCapture: Array +} { + const stored = new Set() + const removed = new Set() + let staleCapture: Array = [] + + for (const event of events) { + if (event.type === `stored`) stored.add(event.id) + else if (event.type === `durably-removed`) removed.add(event.id) + else if (event.type === `read-captured`) staleCapture = [...event.ids] + } + + return { + unfinished: Array.from(stored).filter((id) => !removed.has(id)), + staleCapture, + } +} + // These campaigns share the package replay variables. Target this file and one // test name when replaying a shrink path, because paths are property-specific. const serialWorkOracle = readOfflineOracleConfig({ @@ -1378,6 +1412,105 @@ it(`does not readmit a permanently rejected row from a stale outbox read`, async } }) +it.each([20260923, undefined])( + `filters every durably removed transaction from a generated stale replay read (seed %s)`, + async (seed) => { + await fc.assert( + fc.asyncProperty( + fc.record({ + count: fc.integer({ min: 2, max: 6 }), + removedPrefix: fc.integer({ min: 1, max: 5 }), + }), + async ({ count, removedPrefix }) => { + const captured = gate() + const delivery = gate() + let hold = true + let capturedSnapshot: Array = [] + class Storage extends FakeStorageAdapter { + override async keys() { + const keys = await super.keys() + capturedSnapshot = [...keys] + return keys + } + override async get(key: string) { + const value = await super.get(key) + if (hold && key === `tx:stored-0`) { + captured.resolve() + await delivery.promise + } + return value + } + } + const storage = new Storage() + const outbox = new OutboxManager(storage, {}) + const transactions = Array.from({ length: count }, (_, index) => + storedTransaction(`stored-${index}`), + ) + const ledger: Array = [] + for (const transaction of transactions) { + await outbox.add(transaction) + ledger.push({ type: `stored`, id: transaction.id }) + } + const reading = outbox.getAll() + let hasPrimaryFailure = false + try { + await atOracleCheckpoint(captured.promise, `stale replay captured`) + ledger.push({ + type: `read-captured`, + ids: capturedSnapshot.map((key) => key.slice(`tx:`.length)), + }) + const removed = transactions.slice( + 0, + Math.min(count - 1, removedPrefix), + ) + await outbox.removeMany(removed.map(({ id }) => id)) + for (const transaction of removed) { + ledger.push({ type: `durably-removed`, id: transaction.id }) + } + hold = false + delivery.resolve() + ledger.push({ type: `read-delivered` }) + + const expected = foldReplayLedger(ledger) + const delivered = await atOracleCheckpoint( + reading, + `stale replay delivered`, + ) + expect(delivered.map(({ id }) => id)).toEqual(expected.unfinished) + expect(expected.staleCapture).not.toEqual(expected.unfinished) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + hold = false + delivery.resolve() + await cleanupOfflineOracle([() => reading], hasPrimaryFailure) + } + }, + ), + { + seed, + numRuns: 20, + examples: [[{ count: 3, removedPrefix: 1 }]], + }, + ) + }, +) + +it(`append-only replay judgment rejects stale admission after durable removal`, () => { + const ledger: Array = [ + { type: `stored`, id: `removed` }, + { type: `stored`, id: `peer` }, + { type: `read-captured`, ids: [`removed`, `peer`] }, + { type: `durably-removed`, id: `removed` }, + { type: `read-delivered` }, + ] + const expected = foldReplayLedger(ledger) + + expect(expected.unfinished).toEqual([`peer`]) + expect(expected.staleCapture).not.toEqual(expected.unfinished) +}) + it.each([`remove`, `removeMany`, `clear`] as const)( `filters %s from an outbox read already in flight`, async (operation) => { diff --git a/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts b/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts index 8d8dcc350..eb97c2467 100644 --- a/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts +++ b/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts @@ -1630,9 +1630,9 @@ describe(`query collection ownership lifecycle`, () => { return originalSync.sync({ ...params, metadata: recordMetadata(params.metadata!, metadata), - begin: () => { + begin: (options) => { writes = [] - params.begin() + return params.begin(options) }, write: (change) => { writes.push(change.type)