diff --git a/.changeset/fix-shared-sqlite-hydration-fairness.md b/.changeset/fix-shared-sqlite-hydration-fairness.md new file mode 100644 index 0000000000..8cce3ba892 --- /dev/null +++ b/.changeset/fix-shared-sqlite-hydration-fairness.md @@ -0,0 +1,7 @@ +--- +'@tanstack/db-sqlite-persistence-core': patch +'@tanstack/browser-db-sqlite-persistence': patch +'@tanstack/electron-db-sqlite-persistence': patch +--- + +Schedule complete SQLite hydration units fairly without holding coordinator work inside the local hydration scope. Fence stale startup rows after a coordinator reset. Preserve per-Collection leader adapter routing, mutation results across transport retries, terminal coordinator disposal, real-adapter restart order, and promise-discovered shared scheduling. diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index a340502b46..6b3756ff74 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -74,6 +74,11 @@ jobs: cd examples/react/start-ssr-e2e pnpm exec playwright install --with-deps chromium + - name: Run Browser SQLite OPFS fairness E2E tests + run: | + cd packages/browser-db-sqlite-persistence + pnpm test:opfs-fairness + - name: Run React Start SSR E2E tests run: | cd examples/react/start-ssr-e2e diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index d108f9466b..0fae4080fd 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -111,7 +111,7 @@ comment and the current API/architecture contract before extending its model. | Opaque backend pagination | [window oracle](https://github.com/TanStack/db/blob/main/packages/query-db-collection/tests/cursor-pagination.oracle.test.ts), [cache histories](https://github.com/TanStack/db/blob/main/packages/query-db-collection/tests/cursor-pagination.cache-oracle.test.ts), [cache publication](https://github.com/TanStack/db/blob/main/packages/query-db-collection/tests/cursor-pagination.publication-oracle.test.ts), [browser acquisition boundaries](https://github.com/TanStack/db/blob/main/packages/query-db-collection/tests/cursor-pagination.boundary-oracle.test.ts), [QueryCollection integration](https://github.com/TanStack/db/blob/main/packages/query-db-collection/tests/cursor-pagination.integration.test.ts) | Full filter/sort/slice reference, opaque token transport, actual Query cache expiry/invalidation/GC, forced refresh during growth, protocol failure publication/recovery, bounded slice work, nested cancellation/replacement, reader abort, browser retry defaults, manual-write cache isolation, and production window publications. Stable backend sequences; not snapshot guarantees for changing endpoints. Peek-ahead remains enabled. | | Electric and TrailBase | [Electric histories](https://github.com/TanStack/db/blob/main/packages/electric-db-collection/tests/electric-oracle.property.test.ts), [recovery histories](https://github.com/TanStack/db/blob/main/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts), [held resume snapshots](https://github.com/TanStack/db/blob/main/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts), [PostgreSQL semantics](https://github.com/TanStack/db/blob/main/packages/electric-db-collection/e2e/sql-predicate-semantics.e2e.test.ts), [TrailBase contract](https://github.com/TanStack/db/blob/main/packages/trailbase-db-collection/tests/ORACLE.md) | Installed SDK delivery/framing, independent predicates, exact subscription arguments, restart/reset lineage, held certification and durability races, source-order publication before durability, and late errors. The queued-presence property runs identical fixed/random generators plus isolated seed-and-path replay across insert, update, delete, and truncate callbacks. The recovery fixtures use a mocked ShapeStream; they do not establish live Electric-service framing or native persistence-host behavior. | | PowerSync | [tests](https://github.com/TanStack/db/tree/main/packages/powersync-db-collection/tests), `tests/correctness-oracle.test.ts` | Applied receipt positions crossed with held peers, native SQLite/SDK and cleanup evidence. Run the focused owner with the package's `test:oracles` command. A timeout mutant proves a progress failure, not every value assertion. | -| SQLite persistence and native hosts | [persisted histories](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/persisted.test.ts), [reset/resume histories](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts), [dual-adapter resume snapshots](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts), [Browser composed-owner histories](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts), [Browser coordinator RPC](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts), [driver contracts](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/contracts/sqlite-driver-contract.ts), [Node shared-handle scheduling](https://github.com/TanStack/db/blob/main/packages/node-db-sqlite-persistence/tests/node-driver.test.ts), [OP-SQLite shared-handle scheduling](https://github.com/TanStack/db/blob/main/packages/react-native-db-sqlite-persistence/tests/op-sqlite-driver.test.ts), [browser OPFS lifecycle](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/opfs-page-lifecycle-oracle.test.ts), [worker diagnostics](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/opfs-worker-diagnostics-oracle.test.ts), [Electron IPC and composed owner](https://github.com/TanStack/db/blob/main/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts), [113-law manifest](https://github.com/TanStack/db/blob/main/packages/db-collection-e2e/src/fixtures/persisted-conformance-manifest.ts) | Core cache/remote rejection/peer/reopen histories, atomic reset/resume lineage, key-set evidence, dual-adapter races, and exact driver results. Browser composes public source commits with per-collection elected-owner routing and covers the complete committed-transaction wire partition through deterministic Node transport seams. Remote-subset histories distinguish logical demand, physical acquisitions, exact acquisition leases, and released replay tombstones. Electron composes source commits with a per-collection renderer owner, IPC adapter, real SQLite, and reopen checks. Same-handle Node and OP-SQLite tests cover transaction admission. Controlled OPFS page/worker histories cover ownership and diagnostic-cause retention. The reset/resume owners use sqlite3 CLI and in-memory node:sqlite seams; they do not prove multi-process WAL, mobile/Tauri, or other native-device execution. Distinct database handles rely on SQLite lock admission rather than one in-process queue. React Native hosts without async-context propagation must use the transaction driver supplied to the callback for nested work. Fake workers and synthetic page events do not prove native handle release or real bfcache admission. The Browser composed seams are not real multi-context/OPFS-worker execution; the Electron harness is not an actual Electron process unless its explicit runtime-bridge mode runs. An ownerless elected node suppresses core routing, while a follower may route demand to the elected owner; host coordinators retry only classified transport or admission failures while demand remains retained. The manifest excludes progressive and move suites; registration and shim runs are not device execution. | +| SQLite persistence and native hosts | [persisted histories](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/persisted.test.ts), [reset/resume histories](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts), [dual-adapter resume snapshots](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts), [Browser composed-owner histories](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts), [Browser coordinator RPC](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts), [shared-driver fairness](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.test.ts), [driver contracts](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/contracts/sqlite-driver-contract.ts), [Node shared-handle scheduling](https://github.com/TanStack/db/blob/main/packages/node-db-sqlite-persistence/tests/node-driver.test.ts), [OP-SQLite shared-handle scheduling](https://github.com/TanStack/db/blob/main/packages/react-native-db-sqlite-persistence/tests/op-sqlite-driver.test.ts), [browser OPFS lifecycle](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/opfs-page-lifecycle-oracle.test.ts), [worker diagnostics](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/opfs-worker-diagnostics-oracle.test.ts), [Electron IPC and composed owner](https://github.com/TanStack/db/blob/main/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts), [113-law manifest](https://github.com/TanStack/db/blob/main/packages/db-collection-e2e/src/fixtures/persisted-conformance-manifest.ts) | Core cache/remote rejection/peer/reopen histories, atomic reset/resume lineage, key-set evidence, dual-adapter races, queued coordinator reload/gap notifications during startup, unscheduled startup/reset overlap, and exact driver results. Browser composes public source commits with per-collection elected-owner routing and covers the complete committed-transaction wire partition through deterministic Node transport seams. Remote-subset histories distinguish logical demand, physical acquisitions, exact acquisition leases, and released replay tombstones. Electron composes source commits with a per-collection renderer owner, IPC adapter, real SQLite, and reopen checks. The shared-driver fairness owner checks K=1 complete-logical-hydrate scheduling with identical fixed, random, and seed-plus-path campaigns. It records public rows, raw dequeue reach, and a persist-first FIFO hostile control. Its Chromium OPFS fixture refines the provider boundary but does not establish a browser matrix, elapsed-time latency, unbounded eventuality, or multi-process coordination. Same-handle Node and OP-SQLite tests cover transaction admission. Controlled OPFS page/worker histories cover ownership and diagnostic-cause retention. The reset/resume owners use sqlite3 CLI and in-memory node:sqlite seams; they do not prove multi-process WAL, mobile/Tauri, or other native-device execution. Distinct database handles rely on SQLite lock admission rather than one in-process queue. React Native hosts without async-context propagation must use the transaction driver supplied to the callback for nested work. Fake workers and synthetic page events do not prove native handle release or real bfcache admission. The Browser composed seams are not real multi-context/OPFS-worker execution; the Electron harness is not an actual Electron process unless its explicit runtime-bridge mode runs. An ownerless elected node suppresses core routing, while a follower may route demand to the elected owner; host coordinators retry only classified transport or admission failures while demand remains retained. The manifest excludes progressive and move suites; registration and shim runs are not device execution. | | SQLite expression-index planning | [Node expression-index oracle](https://github.com/TanStack/db/blob/main/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts) | RFC #1659 invariant 8 owns identical persisted-index and runtime-expression shapes. Independent expected keys are checked against direct captured SQL, adapter results, and named-index plans. Generated BigInts use SQLite's signed range; one fixed case checks legacy oversized-value reads. Other limits: bounded unqualified JSON paths/scalars, Node BetterSQLite, and no null, arbitrary raw SQL, or native-host planning. Run the package's `test:oracles` campaign. | | Offline execution | [scheduler](https://github.com/TanStack/db/blob/main/packages/offline-transactions/tests/KeyScheduler.property.test.ts), [leadership](https://github.com/TanStack/db/blob/main/packages/offline-transactions/tests/leadership-replay.property.test.ts), [settlement](https://github.com/TanStack/db/blob/main/packages/offline-transactions/tests/transaction-settlement.property.test.ts), [serialization](https://github.com/TanStack/db/blob/main/packages/offline-transactions/tests/transaction-serializer.property.test.ts) | Declarative FIFO eligibility, per-transaction outcomes, durable state and typed wire trees. Issued work may finish after ownership loss, but new work must not start. Exactly-once network execution is not promised. | | Frameworks | [React conformance](https://github.com/TanStack/db/blob/main/packages/react-db/tests/conformance.test.tsx), [React pagination](https://github.com/TanStack/db/blob/main/packages/react-db/tests/infinite-query-conformance.test.tsx), [shared suites](https://github.com/TanStack/db/tree/main/packages/db-collection-e2e/src/suites) | Exact exposed rows/pages and each framework's own lifecycle cuts. A React witness does not prove Vue/Solid/Angular/Svelte scheduling. Preserve their receiving registrations. | diff --git a/packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.html b/packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.html new file mode 100644 index 0000000000..215878cfd3 --- /dev/null +++ b/packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.html @@ -0,0 +1,13 @@ + + + + + + + Shared driver OPFS fairness oracle + + + running + + + diff --git a/packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.spec.ts b/packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.spec.ts new file mode 100644 index 0000000000..849de99ccc --- /dev/null +++ b/packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.spec.ts @@ -0,0 +1,102 @@ +/** + * Browser checkpoint assertions for the OPFS refinement. Expected public rows + * are rebuilt from the scenario IDs rather than from production output. The + * neutral case proves cold-query reach; the storm case requires no K=1 + * violation. Failure-before-checkpoint, semantic mismatch, driver cleanup, and + * OPFS cleanup remain distinct outcomes so setup or teardown cannot satisfy the + * scheduling law. + */ +import { expect, test } from '@playwright/test' +import type { Page } from '@playwright/test' +import type { OPFSOracleResult } from './shared-driver-fairness.opfs' + +async function readOracleResult( + page: Page, + mode: `neutral` | `storm`, +): Promise { + let rejectPageError!: (error: Error) => void + const pageError = new Promise((_resolve, reject) => { + rejectPageError = reject + }) + void pageError.catch(() => undefined) + const onPageError = (error: Error) => { + rejectPageError( + new Error( + `OPFS fairness page failed before publishing a result: ${error.message}`, + ), + ) + } + page.on(`pageerror`, onPageError) + + try { + await page.goto(`/e2e/shared-driver-fairness.opfs.html?mode=${mode}`) + await Promise.race([ + page.waitForFunction( + () => window.__tanstackDriverFairnessOracle !== undefined, + ), + pageError, + ]) + return page.evaluate(() => window.__tanstackDriverFairnessOracle!) + } finally { + page.off(`pageerror`, onPageError) + } +} + +function expectedHydratedCollections(scenarioId: string, count: number) { + return Array.from({ length: count }, (_, index) => ({ + collectionId: `${scenarioId}-hydrate-${index}`, + rows: [ + { id: `row-${index}-0`, value: index * 10 }, + { id: `row-${index}-1`, value: index * 10 + 1 }, + ], + })) +} + +test(`real Chromium OPFS fixture reaches and cleans up cold hydration`, async ({ + page, +}) => { + const result = await readOracleResult(page, `neutral`) + + if (result.status !== `complete`) throw new Error(result.primaryFailure) + expect(result.provider).toBe(`Chromium OPFSCoopSyncVFS worker`) + expect(result.observation.admittedHydrateIds).toHaveLength(2) + // These are actual public Collection rows captured after preload, compared + // with seed values built independently by this browser assertion. + expect(result.observation.hydratedCollections).toEqual( + expectedHydratedCollections(`opfs-neutral-reach`, 2), + ) + expect( + result.observation.rawDequeues.some((entry) => + entry.sql.startsWith(`SELECT key, value, metadata, row_version FROM`), + ), + ).toBe(true) + expect(result.observation.cleanupFailures).toEqual([]) + expect(result.opfsCleanupFailures).toEqual([]) +}) + +test(`real Chromium OPFS fixture bounds pending cold hydration behind persists`, async ({ + page, +}) => { + const result = await readOracleResult(page, `storm`) + + if (result.status !== `complete`) throw new Error(result.primaryFailure) + expect(result.provider).toBe(`Chromium OPFSCoopSyncVFS worker`) + expect(result.observation.admittedHydrateIds).toHaveLength(4) + expect(result.observation.hydratedCollections).toEqual( + expectedHydratedCollections(`opfs-fixed-persist-storm`, 4), + ) + // This is the semantic RED checkpoint. Setup, wall time, and cleanup are + // reported independently and cannot satisfy this assertion. + if (result.violation !== undefined) { + throw new Error( + `real OPFS fairness mismatch: ${JSON.stringify(result.violation)}; ` + + `logical completion order: ${JSON.stringify(result.observation.logicalCompletionOrder)}; ` + + `driver admissions: ${result.observation.driverAdmissions.length}; ` + + `raw dequeues: ${result.observation.rawDequeues.length}; ` + + `driver cleanup diagnostics: ${JSON.stringify(result.observation.cleanupFailures)}; ` + + `OPFS cleanup diagnostics: ${JSON.stringify(result.opfsCleanupFailures)}`, + ) + } + expect(result.observation.cleanupFailures).toEqual([]) + expect(result.opfsCleanupFailures).toEqual([]) +}) diff --git a/packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.ts b/packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.ts new file mode 100644 index 0000000000..a443f680f2 --- /dev/null +++ b/packages/browser-db-sqlite-persistence/e2e/shared-driver-fairness.opfs.ts @@ -0,0 +1,160 @@ +/** + * Real-provider refinement of the shared-driver fairness oracle. The page runs + * the same legal neutral and fixed-storm histories through public `preload()`, + * the browser/core adapter boundary, and Chromium's OPFSCoopSyncVFS worker. It + * freezes logical completions, public rows, raw dequeue reach, and the K=1 + * violation result before cleanup, then reports provider cleanup separately. + * This fixture adds real OPFS/worker evidence; it does not claim multi-tab, + * multi-process, non-Chromium, latency, or unbounded-eventuality coverage. + */ +import { openBrowserWASQLiteOPFSDatabase } from '../src/index' +import { + findSharedDriverFairnessViolation, + observeSharedDriverFairness, +} from '../tests/shared-driver-fairness-oracle' +import type { + SharedDriverFairnessObservation, + SharedDriverFairnessScenario, + SharedDriverFairnessViolation, +} from '../tests/shared-driver-fairness-oracle' + +export type OPFSOracleResult = + | { + status: `complete` + provider: `Chromium OPFSCoopSyncVFS worker` + observation: SharedDriverFairnessObservation + violation: SharedDriverFairnessViolation | undefined + opfsCleanupFailures: ReadonlyArray + } + | { + status: `failed-before-checkpoint` + provider: `Chromium OPFSCoopSyncVFS worker` + primaryFailure: string + opfsCleanupFailures: ReadonlyArray + } + +declare global { + interface Window { + __tanstackDriverFairnessOracle?: OPFSOracleResult + } +} + +async function removeOPFSArtifacts( + databaseName: string, +): Promise> { + const failures: Array = [] + const root = await navigator.storage.getDirectory() + for (const suffix of [``, `-journal`, `-wal`]) { + try { + await root.removeEntry(`${databaseName}${suffix}`) + } catch (error) { + if (!(error instanceof DOMException && error.name === `NotFoundError`)) { + failures.push( + `${databaseName}${suffix}: ${error instanceof Error ? error.message : String(error)}`, + ) + } + } + } + + // OPFSCoopSyncVFS creates one private temporary access-handle directory per + // worker. This page owns its isolated origin for the fixture run. + const iterableRoot = root as FileSystemDirectoryHandle & { + entries: () => AsyncIterableIterator<[string, FileSystemHandle]> + } + for await (const [name, handle] of iterableRoot.entries()) { + if (handle.kind !== `directory` || !name.startsWith(`.ahp-`)) continue + try { + await root.removeEntry(name, { recursive: true }) + } catch (error) { + failures.push( + `${name}: ${error instanceof Error ? error.message : String(error)}`, + ) + } + } + return failures +} + +function scenarioFromLocation(): SharedDriverFairnessScenario { + const mode = new URL(location.href).searchParams.get(`mode`) + if (mode === `neutral`) { + return { + id: `opfs-neutral-reach`, + work: [0, 1].map((index) => ({ + kind: `hydrate` as const, + id: `hydrate-${index}`, + seededRows: [ + { id: `row-${index}-0`, value: index * 10 }, + { id: `row-${index}-1`, value: index * 10 + 1 }, + ], + })), + } + } + return { + id: `opfs-fixed-persist-storm`, + work: [ + ...Array.from({ length: 5 }, (_, index) => ({ + kind: `persist` as const, + id: `persist-${index}`, + mutationsPerPersist: 2, + })), + ...Array.from({ length: 4 }, (_, index) => ({ + kind: `hydrate` as const, + id: `hydrate-${index}`, + seededRows: [ + { id: `row-${index}-0`, value: index * 10 }, + { id: `row-${index}-1`, value: index * 10 + 1 }, + ], + })), + ], + } +} + +async function run(): Promise { + const status = document.querySelector(`#oracle-status`) + const databaseName = `ws5b-${crypto.randomUUID()}.sqlite` + let observation: SharedDriverFairnessObservation | undefined + let violation: SharedDriverFairnessViolation | undefined + let primaryFailure: string | undefined + let opfsCleanupFailures: ReadonlyArray = [] + try { + observation = await observeSharedDriverFairness( + () => openBrowserWASQLiteOPFSDatabase({ databaseName }), + scenarioFromLocation(), + ) + // Freeze the reached semantic checkpoint before attempting OPFS cleanup. + violation = findSharedDriverFairnessViolation(observation) + } catch (error) { + primaryFailure = error instanceof Error ? error.message : String(error) + } + + try { + opfsCleanupFailures = await removeOPFSArtifacts(databaseName) + } catch (cleanupError) { + opfsCleanupFailures = [ + cleanupError instanceof Error + ? cleanupError.message + : String(cleanupError), + ] + } + + if (observation) { + window.__tanstackDriverFairnessOracle = { + status: `complete`, + provider: `Chromium OPFSCoopSyncVFS worker`, + observation, + violation, + opfsCleanupFailures, + } + if (status) status.value = `complete` + } else { + window.__tanstackDriverFairnessOracle = { + status: `failed-before-checkpoint`, + provider: `Chromium OPFSCoopSyncVFS worker`, + primaryFailure: primaryFailure ?? `unknown failure before checkpoint`, + opfsCleanupFailures, + } + if (status) status.value = `failed-before-checkpoint` + } +} + +void run() diff --git a/packages/browser-db-sqlite-persistence/package.json b/packages/browser-db-sqlite-persistence/package.json index 9bae7bb83c..26b2b70fef 100644 --- a/packages/browser-db-sqlite-persistence/package.json +++ b/packages/browser-db-sqlite-persistence/package.json @@ -23,6 +23,8 @@ "dev": "vite build --watch", "lint": "eslint . --fix", "test": "vitest --run", + "test:oracles": "vitest --run tests/shared-driver-fairness-oracle.test.ts", + "test:opfs-fairness": "playwright test --config playwright.opfs.config.ts", "test:e2e": "pnpm --filter @tanstack/db-ivm build && pnpm --filter @tanstack/db build && pnpm --filter @tanstack/db-sqlite-persistence-core build && pnpm --filter @tanstack/browser-db-sqlite-persistence build && vitest --config vitest.e2e.config.ts --run" }, "type": "module", @@ -56,6 +58,7 @@ }, "devDependencies": { "@journeyapps/wa-sqlite": "^1.4.1", + "@playwright/test": "^1.60.0", "@types/better-sqlite3": "^7.6.13", "@vitest/coverage-istanbul": "^3.2.4", "better-sqlite3": "^12.6.2" diff --git a/packages/browser-db-sqlite-persistence/playwright.opfs.config.ts b/packages/browser-db-sqlite-persistence/playwright.opfs.config.ts new file mode 100644 index 0000000000..ecfa8af0a6 --- /dev/null +++ b/packages/browser-db-sqlite-persistence/playwright.opfs.config.ts @@ -0,0 +1,25 @@ +import { defineConfig } from '@playwright/test' + +const baseURL = `http://127.0.0.1:4185` +const browserChannel = + process.env.PLAYWRIGHT_CHANNEL ?? (process.env.CI ? undefined : `chrome`) + +export default defineConfig({ + testDir: `./e2e`, + testMatch: `shared-driver-fairness.opfs.spec.ts`, + timeout: 60_000, + fullyParallel: false, + workers: 1, + use: { + baseURL, + ...(browserChannel ? { channel: browserChannel } : {}), + headless: true, + trace: `retain-on-failure`, + }, + webServer: { + command: `vite --config vite.opfs.config.ts --host 127.0.0.1 --port 4185`, + reuseExistingServer: false, + timeout: 120_000, + url: `${baseURL}/e2e/shared-driver-fairness.opfs.html`, + }, +}) diff --git a/packages/browser-db-sqlite-persistence/src/browser-persistence.ts b/packages/browser-db-sqlite-persistence/src/browser-persistence.ts index 0b8fa8931b..3f1647c512 100644 --- a/packages/browser-db-sqlite-persistence/src/browser-persistence.ts +++ b/packages/browser-db-sqlite-persistence/src/browser-persistence.ts @@ -157,6 +157,9 @@ export function createBrowserWASQLitePersistence( `sync-absent`, undefined, ) + if (resolvedCoordinator instanceof BrowserCollectionCoordinator) { + resolvedCoordinator.setAdapter(defaultPersistence.adapter) + } return { ...defaultPersistence, diff --git a/packages/browser-db-sqlite-persistence/src/wa-sqlite-driver.ts b/packages/browser-db-sqlite-persistence/src/wa-sqlite-driver.ts index c61136cfe5..1fe5dafb36 100644 --- a/packages/browser-db-sqlite-persistence/src/wa-sqlite-driver.ts +++ b/packages/browser-db-sqlite-persistence/src/wa-sqlite-driver.ts @@ -1,4 +1,7 @@ -import { InvalidPersistedCollectionConfigError } from '@tanstack/db-sqlite-persistence-core' +import { + InvalidPersistedCollectionConfigError, + SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY, +} from '@tanstack/db-sqlite-persistence-core' import type { SQLiteDriver } from '@tanstack/db-sqlite-persistence-core' export type BrowserWASQLiteDatabase = { @@ -62,6 +65,7 @@ function assertDatabaseShape( } export class BrowserWASQLiteDriver implements SQLiteDriver { + readonly [SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY] = {} private readonly database: BrowserWASQLiteDatabase private readonly executionState: DatabaseExecutionState private closed = false @@ -72,29 +76,33 @@ export class BrowserWASQLiteDriver implements SQLiteDriver { this.executionState = getDatabaseExecutionState(this.database) } - async exec(sql: string): Promise { - await this.enqueue(async () => { + exec(sql: string): Promise { + return this.enqueue(async () => { await this.database.execute(sql) }) } - async query( + query( sql: string, params: ReadonlyArray = [], ): Promise> { return this.enqueue(() => this.database.execute(sql, params)) } - async run(sql: string, params: ReadonlyArray = []): Promise { - await this.enqueue(async () => { + run(sql: string, params: ReadonlyArray = []): Promise { + return this.enqueue(async () => { await this.database.execute(sql, params) }) } - async transaction( + transaction( fn: (transactionDriver: SQLiteDriver) => Promise, ): Promise { - assertTransactionCallbackHasDriverArg(fn) + try { + assertTransactionCallbackHasDriverArg(fn) + } catch (error) { + return Promise.reject(error) + } return this.enqueue(async () => { await this.database.execute(`BEGIN IMMEDIATE`) @@ -113,7 +121,7 @@ export class BrowserWASQLiteDriver implements SQLiteDriver { }) } - async transactionWithDriver( + transactionWithDriver( fn: (transactionDriver: SQLiteDriver) => Promise, ): Promise { return this.transaction(fn) @@ -170,6 +178,14 @@ export class BrowserWASQLiteDriver implements SQLiteDriver { private enqueue(operation: () => Promise | T): Promise { const queuedOperation = this.executionState.queue.then(operation, operation) + // Brand the exact Promise returned to callers. Transparent wrappers may + // preserve this identity for late discovery; wrappers that create a new + // Promise must forward the driver key before adapter construction. + Object.defineProperty( + queuedOperation, + SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY, + { value: this[SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY] }, + ) this.executionState.queue = queuedOperation.then( () => undefined, () => undefined, diff --git a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts index fd43e941d2..d3150f7aad 100644 --- a/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts @@ -1,5 +1,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { IR } from '@tanstack/db' +import fc from 'fast-check' +import { BasicIndex, IR, createCollection } from '@tanstack/db' import { RetryableRemoteSubsetAcquisitionError } from '@tanstack/db-sqlite-persistence-core' import { BrowserCollectionCoordinator } from '../src/browser-coordinator' import { @@ -64,6 +65,8 @@ const channels: Map< string, Set<{ onmessage: MessageHandler | null }> > = new Map() +let dropNextMessageWhen: ((data: unknown) => boolean) | undefined +let observePostedMessage: ((data: unknown) => void) | undefined let dropNextBroadcastMessage: ((data: unknown) => boolean) | undefined let duplicateNextBroadcastMessage: | ((data: unknown) => unknown | undefined) @@ -83,6 +86,11 @@ class MockBroadcastChannel { } postMessage(data: unknown): void { + observePostedMessage?.(data) + if (dropNextMessageWhen?.(data)) { + dropNextMessageWhen = undefined + return + } observeBroadcastMessage?.(data) if (dropNextBroadcastMessage?.(data)) { dropNextBroadcastMessage = undefined @@ -245,6 +253,8 @@ function installGlobals(): void { } function cleanupGlobals(): void { + dropNextMessageWhen = undefined + observePostedMessage = undefined dropNextBroadcastMessage = undefined duplicateNextBroadcastMessage = undefined observeBroadcastMessage = undefined @@ -563,6 +573,31 @@ async function flush(ms: number = 10): Promise { await new Promise((resolve) => setTimeout(resolve, ms)) } +type Deferred = { + promise: Promise + resolve: () => void +} + +function createDeferred(): Deferred { + let resolve!: () => void + const promise = new Promise((settle) => { + resolve = settle + }) + return { promise, resolve } +} + +type CoordinatorInspection = { + collectionAdapters: Map + collections: Map + appliedEnvelopes: Map +} + +function inspectCoordinator( + coordinator: BrowserCollectionCoordinator, +): CoordinatorInspection { + return coordinator as unknown as CoordinatorInspection +} + // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- @@ -1600,6 +1635,319 @@ describe(`BrowserCollectionCoordinator`, () => { coordinator.dispose() } }) + + it(`scopes envelope deduplication by collection`, async () => { + const alphaAdapter = createStubAdapter() + const betaAdapter = createStubAdapter() + const coordinator = createCoordinator(alphaAdapter) + coordinator.setAdapterForCollection(`alpha`, alphaAdapter) + coordinator.setAdapterForCollection(`beta`, betaAdapter) + coordinator.subscribe(`alpha`, () => {}) + coordinator.subscribe(`beta`, () => {}) + await flush(50) + + const internals = coordinator as unknown as { + handleApplyLocalMutations: ( + collectionId: string, + request: { + type: `rpc:applyLocalMutations:req` + rpcId: string + envelopeId: string + mutations: Array<{ + mutationId: string + type: `insert` + key: string + value: { id: string } + }> + }, + ) => Promise<{ ok: boolean; rpcId: string }> + } + + try { + const alpha = await internals.handleApplyLocalMutations(`alpha`, { + type: `rpc:applyLocalMutations:req`, + rpcId: `alpha-rpc`, + envelopeId: `shared-envelope`, + mutations: [ + { + mutationId: `alpha-mutation`, + type: `insert`, + key: `alpha`, + value: { id: `alpha` }, + }, + ], + }) + const beta = await internals.handleApplyLocalMutations(`beta`, { + type: `rpc:applyLocalMutations:req`, + rpcId: `beta-rpc`, + envelopeId: `shared-envelope`, + mutations: [ + { + mutationId: `beta-mutation`, + type: `insert`, + key: `beta`, + value: { id: `beta` }, + }, + ], + }) + + expect({ + alpha, + alphaApplies: alphaAdapter.appliedTxs, + beta, + betaApplies: betaAdapter.appliedTxs, + }).toMatchObject({ + alpha: { ok: true, rpcId: `alpha-rpc` }, + alphaApplies: [{ collectionId: `alpha` }], + beta: { ok: true, rpcId: `beta-rpc` }, + betaApplies: [{ collectionId: `beta` }], + }) + } finally { + coordinator.dispose() + } + }) + + it(`coalesces an envelope retry while its first write is in flight`, async () => { + const adapter = createStubAdapter() + const firstApplyEntered = createDeferred() + const releaseFirstApply = createDeferred() + let applyCalls = 0 + adapter.applyCommittedTx = async (collectionId, tx) => { + applyCalls++ + if (applyCalls === 1) { + firstApplyEntered.resolve() + await releaseFirstApply.promise + } + adapter.appliedTxs.push({ collectionId, tx }) + } + const coordinator = createCoordinator(adapter) + coordinator.subscribe(`todos`, () => {}) + await flush(50) + + const internals = coordinator as unknown as { + handleApplyLocalMutations: ( + collectionId: string, + request: { + type: `rpc:applyLocalMutations:req` + rpcId: string + envelopeId: string + mutations: Array<{ + mutationId: string + type: `insert` + key: string + value: { id: string } + }> + }, + ) => Promise<{ + ok: boolean + rpcId: string + term?: number + seq?: number + latestRowVersion?: number + }> + } + const request = { + type: `rpc:applyLocalMutations:req` as const, + envelopeId: `in-flight-envelope`, + mutations: [ + { + mutationId: `mutation`, + type: `insert` as const, + key: `row`, + value: { id: `row` }, + }, + ], + } + + try { + const first = internals.handleApplyLocalMutations(`todos`, { + ...request, + rpcId: `first-rpc`, + }) + await firstApplyEntered.promise + const retry = internals.handleApplyLocalMutations(`todos`, { + ...request, + rpcId: `retry-rpc`, + }) + await Promise.resolve() + releaseFirstApply.resolve() + + const [firstResponse, retryResponse] = await Promise.all([first, retry]) + expect({ + applyCalls, + first: { ...firstResponse, rpcId: undefined }, + retry: { ...retryResponse, rpcId: undefined }, + }).toEqual({ + applyCalls: 1, + first: { ...retryResponse, rpcId: undefined }, + retry: { ...firstResponse, rpcId: undefined }, + }) + } finally { + releaseFirstApply.resolve() + coordinator.dispose() + } + }) + + it(`replays the successful mutation result when its first response is lost`, async () => { + vi.useFakeTimers() + const adapter = createStubAdapter() + let leadershipRead!: () => void + const leadershipReadPromise = new Promise((resolve) => { + leadershipRead = resolve + }) + let firstSuccessDropped!: () => void + const firstSuccessDroppedPromise = new Promise((resolve) => { + firstSuccessDropped = resolve + }) + adapter.getStreamPosition = async () => { + leadershipRead() + return { latestTerm: 0, latestSeq: 0, latestRowVersion: 0 } + } + + const leader = createCoordinator(adapter) + const follower = createCoordinator(adapter) + + try { + leader.subscribe(`todos`, () => {}) + await leadershipReadPromise + await Promise.resolve() + follower.subscribe(`todos`, () => {}) + + dropNextMessageWhen = (data) => { + const payload = ( + data as { payload?: { type?: string; ok?: boolean } } + ).payload + if ( + payload?.type === `rpc:applyLocalMutations:res` && + payload.ok === true + ) { + firstSuccessDropped() + return true + } + return false + } + + const responsePromise = follower.requestApplyLocalMutations(`todos`, [ + { + mutationId: `mut-lost-response`, + type: `insert`, + key: `lost-response`, + value: { id: `lost-response`, title: `Persisted once` }, + }, + ]) + + await firstSuccessDroppedPromise + expect(adapter.appliedTxs).toHaveLength(1) + + await vi.advanceTimersByTimeAsync(10_200) + const response = await responsePromise + expect(response).toMatchObject({ + ok: true, + acceptedMutationIds: [`mut-lost-response`], + }) + expect(adapter.appliedTxs).toHaveLength(1) + } finally { + follower.dispose() + leader.dispose() + vi.useRealTimers() + } + }) + + it(`preserves one durable mutation result across generated response-delivery histories`, async () => { + vi.useFakeTimers() + let run = 0 + try { + await fc.assert( + fc.asyncProperty( + fc.record({ + mutationCount: fc.integer({ min: 1, max: 4 }), + dropFirstSuccess: fc.boolean(), + }), + async ({ mutationCount, dropFirstSuccess }) => { + run++ + const collectionId = `delivery-history-${run}` + const adapter = createStubAdapter() + let leadershipRead!: () => void + const leadershipReadPromise = new Promise((resolve) => { + leadershipRead = resolve + }) + let firstSuccessDropped!: () => void + const firstSuccessDroppedPromise = new Promise( + (resolve) => { + firstSuccessDropped = resolve + }, + ) + adapter.getStreamPosition = async () => { + leadershipRead() + return { latestTerm: 0, latestSeq: 0, latestRowVersion: 0 } + } + + const leader = createCoordinator(adapter) + const follower = createCoordinator(adapter) + try { + leader.subscribe(collectionId, () => {}) + await leadershipReadPromise + await Promise.resolve() + follower.subscribe(collectionId, () => {}) + + if (dropFirstSuccess) { + dropNextMessageWhen = (data) => { + const payload = ( + data as { payload?: { type?: string; ok?: boolean } } + ).payload + if ( + payload?.type === `rpc:applyLocalMutations:res` && + payload.ok === true + ) { + firstSuccessDropped() + return true + } + return false + } + } + + const mutations = Array.from( + { length: mutationCount }, + (_, index) => ({ + mutationId: `mut-${run}-${index}`, + type: `insert` as const, + key: `${index}`, + value: { id: `${index}`, title: `row ${index}` }, + }), + ) + const responsePromise = follower.requestApplyLocalMutations( + collectionId, + mutations, + ) + if (dropFirstSuccess) { + await firstSuccessDroppedPromise + await vi.advanceTimersByTimeAsync(10_200) + } + + const response = await responsePromise + expect(response).toMatchObject({ + ok: true, + acceptedMutationIds: mutations.map( + (mutation) => mutation.mutationId, + ), + }) + expect(adapter.appliedTxs).toHaveLength(1) + } finally { + dropNextMessageWhen = undefined + follower.dispose() + leader.dispose() + for (let microtask = 0; microtask < 4; microtask++) { + await Promise.resolve() + } + } + }, + ), + { seed: 1868, numRuns: 12, endOnFailure: true }, + ) + } finally { + vi.useRealTimers() + } + }) }) describe(`RPC - applyCommittedTx`, () => { @@ -5102,6 +5450,479 @@ describe(`BrowserCollectionCoordinator`, () => { leader.dispose() follower.dispose() }) + + it(`does not repeat successful leader-local index creation`, async () => { + const adapter = createStubAdapter() + let leadershipRead!: () => void + const leadershipReadPromise = new Promise((resolve) => { + leadershipRead = resolve + }) + adapter.getStreamPosition = async () => { + leadershipRead() + return { latestTerm: 0, latestSeq: 0, latestRowVersion: 0 } + } + adapter.ensureIndex = vi.fn().mockResolvedValue(undefined) + + const coord = createCoordinator(adapter) + try { + coord.subscribe(`todos`, () => {}) + await leadershipReadPromise + await Promise.resolve() + + const spec = { expressionSql: [`title`] } + await adapter.ensureIndex(`todos`, `idx-once`, spec) + await coord.requestEnsurePersistedIndex( + `todos`, + `idx-once`, + spec, + adapter, + true, + ) + + expect(adapter.ensureIndex).toHaveBeenCalledOnce() + } finally { + coord.dispose() + } + }) + + it(`keeps production collection bootstrap index work exact-once on the leader`, async () => { + const adapter = createStubAdapter() + adapter.ensureIndex = vi.fn().mockResolvedValue(undefined) + const coordinator = createCoordinator(adapter) + const leaderReady = createDeferred() + observePostedMessage = (data) => { + const envelope = data as { + collectionId?: string + payload?: { type?: string } + } + if ( + envelope.collectionId === `leader-bootstrap-index` && + envelope.payload?.type === `leader:heartbeat` + ) { + leaderReady.resolve() + } + } + const releaseLeader = coordinator.subscribe( + `leader-bootstrap-index`, + () => {}, + ) + await leaderReady.promise + observePostedMessage = undefined + + const collection = createCollection( + persistedCollectionOptions<{ id: string; title: string }, string>({ + id: `leader-bootstrap-index`, + getKey: (row) => row.id, + defaultIndexType: BasicIndex, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + collection.createIndex((row) => row.title, { name: `title` }) + + try { + await collection.preload() + expect(adapter.ensureIndex).toHaveBeenCalledOnce() + } finally { + await collection.cleanup() + releaseLeader() + coordinator.dispose() + } + }) + + it(`reaches crossed production RPC leadership only after both hydration scopes exit`, async () => { + const bothLocalIndexesEntered = createDeferred() + const bothLeaderRPCsEntered = createDeferred() + const releaseSchedulerCycle = createDeferred() + let localIndexEntries = 0 + let leaderRPCEntries = 0 + + const createTab = () => { + const adapter = createStubAdapter() + const baseEnsureIndex = adapter.ensureIndex.bind(adapter) + let hydrationScopeActive = false + const leaderRPCScopeObservations: Array = [] + const scopedAdapter: PersistenceAdapter = { + ...adapter, + ensureIndex: async (...args) => { + localIndexEntries++ + if (localIndexEntries === 2) bothLocalIndexesEntered.resolve() + await bothLocalIndexesEntered.promise + await baseEnsureIndex(...args) + }, + } + adapter.runInHydrationScope = async (task) => { + hydrationScopeActive = true + try { + return await task(scopedAdapter) + } finally { + hydrationScopeActive = false + } + } + adapter.ensureIndex = async (...args) => { + leaderRPCScopeObservations.push(hydrationScopeActive) + leaderRPCEntries++ + if (leaderRPCEntries === 2) bothLeaderRPCsEntered.resolve() + if (hydrationScopeActive) await releaseSchedulerCycle.promise + await baseEnsureIndex(...args) + } + return { adapter, leaderRPCScopeObservations } + } + + const tab1 = createTab() + const tab2 = createTab() + const coordinator1 = createCoordinator(tab1.adapter) + const coordinator2 = createCoordinator(tab2.adapter) + const leaderCollections = new Set() + const bothLeadersReady = createDeferred() + observePostedMessage = (data) => { + const envelope = data as { + collectionId?: string + payload?: { type?: string } + } + if ( + envelope.payload?.type === `leader:heartbeat` && + (envelope.collectionId === `crossed-a` || + envelope.collectionId === `crossed-b`) + ) { + leaderCollections.add(envelope.collectionId) + if (leaderCollections.size === 2) bothLeadersReady.resolve() + } + } + coordinator1.setAdapterForCollection(`crossed-b`, tab1.adapter) + coordinator2.setAdapterForCollection(`crossed-a`, tab2.adapter) + const releaseLeaderB = coordinator1.subscribe(`crossed-b`, () => {}) + const releaseLeaderA = coordinator2.subscribe(`crossed-a`, () => {}) + await bothLeadersReady.promise + observePostedMessage = undefined + + const createFollowerCollection = ( + id: string, + adapter: PersistenceAdapter, + coordinator: BrowserCollectionCoordinator, + ) => { + const collection = createCollection( + persistedCollectionOptions<{ id: string; title: string }, string>({ + id, + getKey: (row) => row.id, + defaultIndexType: BasicIndex, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + collection.createIndex((row) => row.title, { name: `${id}-title` }) + return collection + } + const followerA = createFollowerCollection( + `crossed-a`, + tab1.adapter, + coordinator1, + ) + const followerB = createFollowerCollection( + `crossed-b`, + tab2.adapter, + coordinator2, + ) + const preloadA = Promise.resolve(followerA.preload()) + const preloadB = Promise.resolve(followerB.preload()) + void preloadA.catch(() => undefined) + void preloadB.catch(() => undefined) + + try { + await bothLeaderRPCsEntered.promise + expect({ + tab1: tab1.leaderRPCScopeObservations, + tab2: tab2.leaderRPCScopeObservations, + }).toEqual({ tab1: [false], tab2: [false] }) + await Promise.all([preloadA, preloadB]) + } finally { + releaseSchedulerCycle.resolve() + await Promise.all([preloadA, preloadB]).catch(() => undefined) + await Promise.all([followerA.cleanup(), followerB.cleanup()]) + releaseLeaderA() + releaseLeaderB() + coordinator2.dispose() + coordinator1.dispose() + } + }) + + it(`uses a supplied leader-local adapter when local work is not complete`, async () => { + const registeredAdapter = createStubAdapter() + const scopedAdapter = createStubAdapter() + let leadershipRead!: () => void + const leadershipReadPromise = new Promise((resolve) => { + leadershipRead = resolve + }) + registeredAdapter.getStreamPosition = async () => { + leadershipRead() + return { latestTerm: 0, latestSeq: 0, latestRowVersion: 0 } + } + registeredAdapter.ensureIndex = vi.fn().mockResolvedValue(undefined) + scopedAdapter.ensureIndex = vi.fn().mockResolvedValue(undefined) + + const coord = createCoordinator(registeredAdapter) + try { + coord.subscribe(`todos`, () => {}) + await leadershipReadPromise + await Promise.resolve() + + await coord.requestEnsurePersistedIndex( + `todos`, + `idx-scoped`, + { expressionSql: [`title`] }, + scopedAdapter, + ) + + expect(scopedAdapter.ensureIndex).toHaveBeenCalledOnce() + expect(registeredAdapter.ensureIndex).not.toHaveBeenCalled() + } finally { + coord.dispose() + } + }) + + it(`keeps leader RPC work on the adapter registered for its collection`, async () => { + const todosAdapter = createStubAdapter() + const notesAdapter = createStubAdapter() + let leadershipRead!: () => void + const leadershipReadPromise = new Promise((resolve) => { + leadershipRead = resolve + }) + todosAdapter.ensureIndex = vi.fn().mockResolvedValue(undefined) + notesAdapter.ensureIndex = vi.fn().mockResolvedValue(undefined) + todosAdapter.getStreamPosition = async () => { + leadershipRead() + return { latestTerm: 0, latestSeq: 0, latestRowVersion: 0 } + } + + const leader = createCoordinator(todosAdapter) + const follower = createCoordinator(notesAdapter) + + try { + leader.setAdapterForCollection(`todos`, todosAdapter) + leader.subscribe(`todos`, () => {}) + await leadershipReadPromise + await Promise.resolve() + expect(leader.isLeader(`todos`)).toBe(true) + follower.subscribe(`todos`, () => {}) + + // Resolving a later collection variant must not replace the adapter + // already owning leader-side work for `todos`. + leader.setAdapter(notesAdapter) + await follower.requestEnsurePersistedIndex(`todos`, `idx-todos`, { + expressionSql: [`title`], + }) + + expect(todosAdapter.ensureIndex).toHaveBeenCalledOnce() + expect(notesAdapter.ensureIndex).not.toHaveBeenCalled() + } finally { + follower.dispose() + leader.dispose() + } + }) + + it(`re-registers the collection adapter after cleanup and restart`, async () => { + const defaultAdapter = createStubAdapter() + const collectionAdapter = createStubAdapter() + const defaultEnsure = vi.fn().mockResolvedValue(undefined) + const collectionEnsure = vi.fn().mockResolvedValue(undefined) + defaultAdapter.ensureIndex = defaultEnsure + collectionAdapter.ensureIndex = collectionEnsure + const coordinator = createCoordinator(defaultAdapter) + const collection = createCollection( + persistedCollectionOptions<{ id: string; title: string }, string>({ + id: `todos`, + getKey: (row) => row.id, + sync: { sync: ({ markReady }) => markReady() }, + persistence: { adapter: collectionAdapter, coordinator }, + }), + ) + + try { + await collection.stateWhenReady() + await vi.waitFor(() => expect(coordinator.isLeader(`todos`)).toBe(true)) + await coordinator.requestEnsurePersistedIndex(`todos`, `before`, { + expressionSql: [`title`], + }) + + await collection.cleanup() + expect( + inspectCoordinator(coordinator).collectionAdapters.has(`todos`), + ).toBe(false) + await collection.stateWhenReady() + await vi.waitFor(() => expect(coordinator.isLeader(`todos`)).toBe(true)) + await coordinator.requestEnsurePersistedIndex(`todos`, `after`, { + expressionSql: [`title`], + }) + + expect(collectionEnsure.mock.calls.map((call) => call[1])).toEqual([ + `before`, + `after`, + ]) + expect(defaultEnsure).not.toHaveBeenCalled() + } finally { + await collection.cleanup() + coordinator.dispose() + } + }) + + it(`replaces a cached default adapter with its collection registration`, async () => { + const defaultAdapter = createStubAdapter() + const replacementAdapter = createStubAdapter() + let leadershipRead!: () => void + const leadershipReadPromise = new Promise((resolve) => { + leadershipRead = resolve + }) + defaultAdapter.getStreamPosition = async () => { + leadershipRead() + return { latestTerm: 0, latestSeq: 0, latestRowVersion: 0 } + } + defaultAdapter.ensureIndex = vi.fn().mockResolvedValue(undefined) + replacementAdapter.ensureIndex = vi.fn().mockResolvedValue(undefined) + + const coordinator = createCoordinator(defaultAdapter) + try { + coordinator.subscribe(`todos`, () => {}) + await leadershipReadPromise + await Promise.resolve() + + await coordinator.requestEnsurePersistedIndex(`todos`, `idx-default`, { + expressionSql: [`title`], + }) + coordinator.setAdapterForCollection(`todos`, replacementAdapter) + await coordinator.requestEnsurePersistedIndex( + `todos`, + `idx-replacement`, + { expressionSql: [`title`] }, + ) + + expect(defaultAdapter.ensureIndex).toHaveBeenCalledOnce() + expect(replacementAdapter.ensureIndex).toHaveBeenCalledOnce() + expect(replacementAdapter.ensureIndex).toHaveBeenCalledWith( + `todos`, + `idx-replacement`, + { expressionSql: [`title`] }, + ) + } finally { + coordinator.dispose() + } + }) + + it(`routes generated collection and RPC histories through their owning adapter`, async () => { + let run = 0 + await fc.assert( + fc.asyncProperty( + fc.integer({ min: 2, max: 4 }).chain((collectionCount) => + fc.record({ + collectionCount: fc.constant(collectionCount), + registrationOrder: fc.shuffledSubarray( + Array.from({ length: collectionCount }, (_, index) => index), + { minLength: collectionCount, maxLength: collectionCount }, + ), + targetIndex: fc.integer({ min: 0, max: collectionCount - 1 }), + rpcKind: fc.constantFrom(`index`, `pull`, `mutation`), + }), + ), + async ({ + collectionCount, + registrationOrder, + targetIndex, + rpcKind, + }) => { + run++ + const collectionIds = Array.from( + { length: collectionCount }, + (_, index) => `routing-${run}-${index}`, + ) + const adapters = collectionIds.map(() => createStubAdapter()) + const ensureCalls = adapters.map(() => vi.fn()) + const pullCalls = adapters.map(() => vi.fn()) + adapters.forEach((adapter, index) => { + adapter.ensureIndex = + ensureCalls[index]!.mockResolvedValue(undefined) + adapter.pullSince = pullCalls[index]!.mockResolvedValue({ + latestRowVersion: index, + requiresFullReload: false, + changedKeys: [], + deletedKeys: [], + }) + }) + + let leadershipRead!: () => void + const leadershipReadPromise = new Promise((resolve) => { + leadershipRead = resolve + }) + adapters[targetIndex]!.getStreamPosition = async () => { + leadershipRead() + return { latestTerm: 0, latestSeq: 0, latestRowVersion: 0 } + } + + const leader = createCoordinator(adapters[0]) + const follower = createCoordinator(adapters.at(-1)) + const targetCollectionId = collectionIds[targetIndex]! + try { + for (const index of registrationOrder) { + leader.setAdapterForCollection( + collectionIds[index]!, + adapters[index]!, + ) + } + leader.subscribe(targetCollectionId, () => {}) + await leadershipReadPromise + await Promise.resolve() + follower.subscribe(targetCollectionId, () => {}) + + if (rpcKind === `index`) { + await follower.requestEnsurePersistedIndex( + targetCollectionId, + `idx-${run}`, + { expressionSql: [`title`] }, + ) + } else if (rpcKind === `pull`) { + await follower.pullSince(targetCollectionId, 0) + } else { + const response = await follower.requestApplyLocalMutations( + targetCollectionId, + [ + { + mutationId: `mut-${run}`, + type: `insert`, + key: `${run}`, + value: { id: `${run}`, title: `row ${run}` }, + }, + ], + ) + expect(response.ok).toBe(true) + } + + adapters.forEach((adapter, index) => { + const callCount = + rpcKind === `index` + ? ensureCalls[index]!.mock.calls.length + : rpcKind === `pull` + ? pullCalls[index]!.mock.calls.length + : adapter.appliedTxs.length + expect(callCount).toBe(index === targetIndex ? 1 : 0) + }) + } finally { + follower.dispose() + leader.dispose() + for (let microtask = 0; microtask < 4; microtask++) { + await Promise.resolve() + } + } + }, + ), + { seed: 1868, numRuns: 18, endOnFailure: true }, + ) + }) }) describe(`dispose`, () => { @@ -5173,5 +5994,347 @@ describe(`BrowserCollectionCoordinator`, () => { // Should not throw after disposal expect(coord.isLeader(`todos`)).toBe(false) }) + + it(`settles an in-flight RPC at dispose without posting retries`, async () => { + vi.useFakeTimers() + const adapter = createStubAdapter() + let leadershipRead!: () => void + const leadershipReadPromise = new Promise((resolve) => { + leadershipRead = resolve + }) + let firstRequestPosted!: () => void + const firstRequestPostedPromise = new Promise((resolve) => { + firstRequestPosted = resolve + }) + adapter.getStreamPosition = async () => { + leadershipRead() + return { latestTerm: 0, latestSeq: 0, latestRowVersion: 0 } + } + + const leader = createCoordinator(adapter) + const follower = createCoordinator(adapter) + let requestPosts = 0 + let settled = false + + try { + leader.subscribe(`todos`, () => {}) + await leadershipReadPromise + await Promise.resolve() + follower.subscribe(`todos`, () => {}) + + observePostedMessage = (data) => { + const type = (data as { payload?: { type?: string } }).payload?.type + if (type === `rpc:ensurePersistedIndex:req`) { + requestPosts++ + if (requestPosts === 1) firstRequestPosted() + } + } + dropNextMessageWhen = (data) => + (data as { payload?: { type?: string } }).payload?.type === + `rpc:ensurePersistedIndex:req` + + const request = follower + .requestEnsurePersistedIndex(`todos`, `idx-dispose`, { + expressionSql: [`title`], + }) + .then( + () => { + settled = true + }, + () => { + settled = true + }, + ) + + await firstRequestPostedPromise + follower.dispose() + for (let microtask = 0; microtask < 8; microtask++) { + await Promise.resolve() + } + const settledAtDispose = settled + + await vi.advanceTimersByTimeAsync(21_000) + await request + + expect({ settledAtDispose, requestPosts }).toEqual({ + settledAtDispose: true, + requestPosts: 1, + }) + } finally { + follower.dispose() + leader.dispose() + vi.useRealTimers() + } + }) + + it(`makes disposal terminal across generated RPC kinds and lifecycle phases`, async () => { + vi.useFakeTimers() + let run = 0 + try { + await fc.assert( + fc.asyncProperty( + fc.record({ + rpcKind: fc.constantFrom(`index`, `pull`, `mutation`), + disposePhase: fc.constantFrom( + `before-request`, + `pending`, + `retry-delay`, + ), + }), + async ({ rpcKind, disposePhase }) => { + run++ + const collectionId = `dispose-history-${run}` + const adapter = createStubAdapter() + let leadershipRead!: () => void + const leadershipReadPromise = new Promise((resolve) => { + leadershipRead = resolve + }) + let firstRequestPosted!: () => void + const firstRequestPostedPromise = new Promise((resolve) => { + firstRequestPosted = resolve + }) + adapter.getStreamPosition = async () => { + leadershipRead() + return { latestTerm: 0, latestSeq: 0, latestRowVersion: 0 } + } + + const leader = createCoordinator(adapter) + const follower = createCoordinator(adapter) + let requestPosts = 0 + const requestType = + rpcKind === `index` + ? `rpc:ensurePersistedIndex:req` + : rpcKind === `pull` + ? `rpc:pullSince:req` + : `rpc:applyLocalMutations:req` + + try { + leader.subscribe(collectionId, () => {}) + await leadershipReadPromise + await Promise.resolve() + follower.subscribe(collectionId, () => {}) + + observePostedMessage = (data) => { + if ( + (data as { payload?: { type?: string } }).payload?.type === + requestType + ) { + requestPosts++ + if (requestPosts === 1) firstRequestPosted() + } + } + dropNextMessageWhen = (data) => + (data as { payload?: { type?: string } }).payload?.type === + requestType + + if (disposePhase === `before-request`) { + follower.dispose() + } + const request = + rpcKind === `index` + ? follower.requestEnsurePersistedIndex( + collectionId, + `idx-${run}`, + { expressionSql: [`title`] }, + ) + : rpcKind === `pull` + ? follower.pullSince(collectionId, 0) + : follower.requestApplyLocalMutations(collectionId, [ + { + mutationId: `mut-${run}`, + type: `insert`, + key: `${run}`, + value: { id: `${run}`, title: `row ${run}` }, + }, + ]) + const settled = request.then( + () => `resolved` as const, + () => `rejected` as const, + ) + + if (disposePhase !== `before-request`) { + await firstRequestPostedPromise + if (disposePhase === `retry-delay`) { + await vi.advanceTimersByTimeAsync(10_000) + } + follower.dispose() + } + + await expect(settled).resolves.toBe(`rejected`) + expect(requestPosts).toBe( + disposePhase === `before-request` ? 0 : 1, + ) + } finally { + observePostedMessage = undefined + dropNextMessageWhen = undefined + follower.dispose() + leader.dispose() + for (let microtask = 0; microtask < 4; microtask++) { + await Promise.resolve() + } + } + }, + ), + { seed: 1868, numRuns: 18, endOnFailure: true }, + ) + } finally { + vi.useRealTimers() + } + }) + }) + + describe(`collection and retry-result retention`, () => { + it(`releases generated collection-owned state after the last subscriber`, async () => { + let run = 0 + await fc.assert( + fc.asyncProperty( + fc.integer({ min: 1, max: 4 }).chain((collectionCount) => + fc.record({ + collectionCount: fc.constant(collectionCount), + releaseOrder: fc.shuffledSubarray( + Array.from({ length: collectionCount }, (_, index) => index), + { minLength: collectionCount, maxLength: collectionCount }, + ), + }), + ), + async ({ collectionCount, releaseOrder }) => { + run++ + const coordinator = createCoordinator() + const collectionIds = Array.from( + { length: collectionCount }, + (_, index) => `lifecycle-${run}-${index}`, + ) + const readyCollections = new Set() + const allReady = createDeferred() + observePostedMessage = (data) => { + const envelope = data as { + collectionId?: string + payload?: { type?: string } + } + if ( + envelope.payload?.type === `leader:heartbeat` && + envelope.collectionId && + collectionIds.includes(envelope.collectionId) + ) { + readyCollections.add(envelope.collectionId) + if (readyCollections.size === collectionCount) { + allReady.resolve() + } + } + } + for (const collectionId of collectionIds) { + coordinator.setAdapterForCollection( + collectionId, + createStubAdapter(), + ) + } + const releases = collectionIds.map((collectionId) => + coordinator.subscribe(collectionId, () => {}), + ) + + try { + await allReady.promise + observePostedMessage = undefined + expect( + inspectCoordinator(coordinator).collectionAdapters.size, + ).toBe(collectionCount) + + let remaining = collectionCount + for (const releasedIndex of releaseOrder) { + releases[releasedIndex]!() + remaining-- + expect({ + adapters: + inspectCoordinator(coordinator).collectionAdapters.size, + collections: inspectCoordinator(coordinator).collections.size, + }).toEqual({ + adapters: remaining, + collections: remaining, + }) + } + } finally { + observePostedMessage = undefined + for (const release of releases) release() + coordinator.dispose() + } + }, + ), + { seed: 1868, numRuns: 12, endOnFailure: true }, + ) + }) + + it(`bounds generated retry-result histories by retention and collection lifecycle`, async () => { + vi.useFakeTimers() + let run = 0 + try { + await fc.assert( + fc.asyncProperty( + fc.record({ + mutationCount: fc.integer({ min: 1, max: 4 }), + terminal: fc.constantFrom(`retention`, `unsubscribe`, `dispose`), + }), + async ({ mutationCount, terminal }) => { + run++ + vi.setSystemTime(0) + const adapter = createStubAdapter() + const coordinator = createCoordinator(adapter) + const collectionId = `envelope-retention-${run}` + const ready = createDeferred() + observePostedMessage = (data) => { + const envelope = data as { + collectionId?: string + payload?: { type?: string } + } + if ( + envelope.collectionId === collectionId && + envelope.payload?.type === `leader:heartbeat` + ) { + ready.resolve() + } + } + const release = coordinator.subscribe(collectionId, () => {}) + + try { + await ready.promise + observePostedMessage = undefined + for (let index = 0; index < mutationCount; index++) { + await coordinator.requestApplyLocalMutations(collectionId, [ + { + mutationId: `mutation-${run}-${index}`, + type: `insert`, + key: `${index}`, + value: { id: `${index}`, title: `row ${index}` }, + }, + ]) + } + expect( + inspectCoordinator(coordinator).appliedEnvelopes.size, + ).toBe(mutationCount) + + if (terminal === `retention`) { + await vi.advanceTimersByTimeAsync(60_000) + } else if (terminal === `unsubscribe`) { + release() + } else { + coordinator.dispose() + } + + expect( + inspectCoordinator(coordinator).appliedEnvelopes.size, + ).toBe(0) + } finally { + observePostedMessage = undefined + release() + coordinator.dispose() + vi.clearAllTimers() + } + }, + ), + { seed: 1868, numRuns: 12, endOnFailure: true }, + ) + } finally { + vi.useRealTimers() + } + }) }) }) diff --git a/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.test.ts b/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.test.ts new file mode 100644 index 0000000000..297c59e576 --- /dev/null +++ b/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.test.ts @@ -0,0 +1,555 @@ +/** + * Node campaign for the shared-driver K=1 contract documented in + * `shared-driver-fairness-oracle.ts`. Fixed histories prove neutral reach and a + * persist storm; generated legal histories vary both lane sizes, mutation + * width, and tail order. Public rows and logical completion checkpoints are + * checked independently of production scheduling, and an executable + * named persist-first FIFO wrong answer proves the checker rejects the original + * fault. Grammar controls reconstruct that witness, exercise the bounded + * marginals, reject a nearby invalid storm, and ablate tail-order variation. + * The identical generated property runs in retained fixed-seed and seedless + * random lanes. Supplying both TANSTACK_DB_DRIVER_FAIRNESS_SEED and + * TANSTACK_DB_DRIVER_FAIRNESS_PATH replaces those lanes with one checked + * seed+shrink-path replay. An axis-removal calibration proves that removing + * tail-order variation loses hydrate-before-later-persist histories. + */ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import fc from 'fast-check' +import { describe, expect, it } from 'vitest' +import { + SHARED_DRIVER_FAIRNESS_BOUND, + createPersistFirstFaultObservation, + findSharedDriverFairnessViolation, + observeSharedDriverFairness, +} from './shared-driver-fairness-oracle' +import { createWASQLiteTestDatabase } from './helpers/wa-sqlite-test-db' +import type { + SharedDriverFairnessObservation, + SharedDriverFairnessOptions, + SharedDriverFairnessScenario, + SharedDriverFairnessWork, +} from './shared-driver-fairness-oracle' + +const FIXED_SEED = 165_905 +const GENERATED_RUNS = 12 +const HYDRATE_COUNT_RANGE = { min: 2, max: 7 } as const +const PERSIST_COUNT_RANGE = { min: 3, max: 7 } as const +const MUTATIONS_PER_PERSIST_RANGE = { min: 1, max: 3 } as const + +type FairnessReplayEnvironment = Record + +type FairnessReplayConfig = { + seed: number + path: string +} + +type FairnessPropertyMode = { + label: `fixed-seed` | `seedless-random` | `checked-replay` + seed?: number + path?: string +} + +function readFairnessReplayConfig( + environment: FairnessReplayEnvironment = process.env, +): FairnessReplayConfig | undefined { + const seedText = environment.TANSTACK_DB_DRIVER_FAIRNESS_SEED + const path = environment.TANSTACK_DB_DRIVER_FAIRNESS_PATH + if (seedText === undefined && path === undefined) return undefined + if (seedText === undefined || path === undefined) { + throw new Error( + `TANSTACK_DB_DRIVER_FAIRNESS_SEED and TANSTACK_DB_DRIVER_FAIRNESS_PATH must be supplied together`, + ) + } + + const seed = Number(seedText) + if (seedText.trim() === `` || !Number.isSafeInteger(seed)) { + throw new Error(`TANSTACK_DB_DRIVER_FAIRNESS_SEED must be an integer`) + } + if (!/^\d+(?::\d+)*$/.test(path)) { + throw new Error( + `TANSTACK_DB_DRIVER_FAIRNESS_PATH must contain colon-separated nonnegative integers`, + ) + } + return { seed, path } +} + +function createFairnessPropertyModes( + replay: FairnessReplayConfig | undefined, +): ReadonlyArray { + if (replay) { + return [{ label: `checked-replay`, ...replay }] + } + return [ + { label: `fixed-seed`, seed: FIXED_SEED }, + { label: `seedless-random` }, + ] +} + +function readFairnessCalibration( + environment: FairnessReplayEnvironment = process.env, +): SharedDriverFairnessOptions { + const fault = environment.TANSTACK_DB_DRIVER_FAIRNESS_CALIBRATION + if (fault === undefined) return {} + if (fault !== `persist-first-fifo`) { + throw new Error( + `TANSTACK_DB_DRIVER_FAIRNESS_CALIBRATION must be persist-first-fifo`, + ) + } + return { schedulingFault: fault } +} + +const fairnessPropertyModes = createFairnessPropertyModes( + readFairnessReplayConfig(), +) +const fairnessCalibration = readFairnessCalibration() + +type WorkKind = SharedDriverFairnessWork[`kind`] + +type GeneratedFairnessHistory = { + hydrateCount: number + persistCount: number + mutationsPerPersist: number + orderedKinds: ReadonlyArray + replayHistory: string +} + +type GeneratedTailEntry = { + kind: WorkKind + token: string +} + +function createScenario( + id: string, + orderedKinds: ReadonlyArray, + mutationsPerPersist = 1, +): SharedDriverFairnessScenario { + let hydrateIndex = 0 + let persistIndex = 0 + return { + id, + work: orderedKinds.map((kind) => { + if (kind === `persist`) { + const index = persistIndex++ + return { + kind, + id: `persist-${index}`, + mutationsPerPersist, + } + } + const index = hydrateIndex++ + return { + kind, + id: `hydrate-${index}`, + seededRows: [ + { id: `row-${index}-0`, value: index * 10 }, + { id: `row-${index}-1`, value: index * 10 + 1 }, + ], + } + }), + } +} + +function expectedHydratedCollections(scenario: SharedDriverFairnessScenario) { + return scenario.work.flatMap((work) => + work.kind === `hydrate` + ? [ + { + collectionId: `${scenario.id}-${work.id}`, + rows: work.seededRows.map((row) => ({ ...row })), + }, + ] + : [], + ) +} + +async function withNodeScenario( + scenario: SharedDriverFairnessScenario, + assertion: ( + observation: SharedDriverFairnessObservation, + ) => void | Promise, + options: SharedDriverFairnessOptions = {}, +): Promise { + const directory = mkdtempSync(join(tmpdir(), `db-driver-fairness-`)) + let primaryFailure: unknown + try { + const observation = await observeSharedDriverFairness( + () => + createWASQLiteTestDatabase({ + filename: join(directory, `state.sqlite`), + }), + scenario, + options, + ) + await assertion(observation) + } catch (error) { + primaryFailure = error + } + + let nodeCleanupFailure: unknown + try { + rmSync(directory, { recursive: true, force: true }) + } catch (error) { + nodeCleanupFailure = error + } + + if (primaryFailure !== undefined) { + if (nodeCleanupFailure !== undefined) { + const primaryMessage = + primaryFailure instanceof Error + ? primaryFailure.message + : String(primaryFailure) + const cleanupMessage = + nodeCleanupFailure instanceof Error + ? nodeCleanupFailure.message + : String(nodeCleanupFailure) + throw new Error( + `${primaryMessage}; node cleanup diagnostics: ${cleanupMessage}`, + ) + } + throw primaryFailure + } + if (nodeCleanupFailure !== undefined) throw nodeCleanupFailure +} + +function expectObservationReach( + observation: SharedDriverFairnessObservation, +): void { + const expectedHydrates = expectedHydratedCollections(observation.scenario) + expect(observation.admittedHydrateIds).toEqual( + expectedHydrates.map(({ collectionId }) => collectionId), + ) + expect(observation.hydrationCompletions).toHaveLength(expectedHydrates.length) + // Actual rows come from the public Collection after preload. The expected + // rows above are derived independently from the generated seed history. + expect(observation.hydratedCollections).toEqual(expectedHydrates) +} + +function expectFairObservation( + observation: SharedDriverFairnessObservation, +): void { + expectObservationReach(observation) + const violation = findSharedDriverFairnessViolation( + observation, + SHARED_DRIVER_FAIRNESS_BOUND, + ) + if (violation) { + throw new Error( + `shared-driver fairness mismatch at ${violation.checkpoint.collectionId}: ` + + `${violation.checkpoint.completedPersistIds.length} persists completed ` + + `(maximum ${violation.expectedMaximumCompletedPersists}), ` + + `${violation.checkpoint.pendingPersistCount} remained pending ` + + `(minimum ${violation.expectedMinimumPendingPersists}); ` + + `permitted completed ids: ${JSON.stringify(violation.permittedCompletedPersistIds)}; ` + + `cleanup diagnostics: ${JSON.stringify(observation.cleanupFailures)}`, + ) + } + expect(observation.cleanupFailures).toEqual([]) +} + +function buildGeneratedHistory( + hydrateCount: number, + persistCount: number, + mutationsPerPersist: number, + generatedTail: ReadonlyArray, +): GeneratedFairnessHistory { + return { + hydrateCount, + persistCount, + mutationsPerPersist, + orderedKinds: [ + `persist` as const, + ...generatedTail.map(({ kind }) => kind), + ], + replayHistory: `p0,${generatedTail.map(({ token }) => token).join(`,`)}`, + } +} + +function createGeneratedHistoryArbitrary( + tailOrderAxis: `varied` | `removed`, +): fc.Arbitrary { + return fc + .record({ + hydrateCount: fc.integer(HYDRATE_COUNT_RANGE), + persistCount: fc.integer(PERSIST_COUNT_RANGE), + mutationsPerPersist: fc.integer(MUTATIONS_PER_PERSIST_RANGE), + }) + .chain(({ hydrateCount, persistCount, mutationsPerPersist }) => { + const tail = [ + ...Array.from({ length: persistCount - 1 }, (_, index) => ({ + kind: `persist` as const, + token: `p${index + 1}`, + })), + ...Array.from({ length: hydrateCount }, (_, index) => ({ + kind: `hydrate` as const, + token: `h${index}`, + })), + ] + const orderedTail = + tailOrderAxis === `varied` + ? fc.shuffledSubarray(tail, { + minLength: tail.length, + maxLength: tail.length, + }) + : fc.constant(tail) + return orderedTail.map((generatedTail) => + buildGeneratedHistory( + hydrateCount, + persistCount, + mutationsPerPersist, + generatedTail, + ), + ) + }) +} + +function reachesHydrateBeforeLaterPersist( + history: GeneratedFairnessHistory, +): boolean { + let hydrateSeen = false + for (const kind of history.orderedKinds.slice(1)) { + if (kind === `hydrate`) hydrateSeen = true + if (kind === `persist` && hydrateSeen) return true + } + return false +} + +const generatedHistoryArbitrary = createGeneratedHistoryArbitrary(`varied`) +const tailOrderAblatedArbitrary = createGeneratedHistoryArbitrary(`removed`) +let generatedPropertyExecutions = 0 + +const generatedFairnessProperty = fc.asyncProperty( + generatedHistoryArbitrary, + async ({ + hydrateCount, + persistCount, + mutationsPerPersist, + orderedKinds, + replayHistory, + }) => { + generatedPropertyExecutions++ + const observationId = + `generated-h${hydrateCount}-p${persistCount}-m${mutationsPerPersist}-` + + replayHistory.replaceAll(`,`, `-`) + await withNodeScenario( + createScenario(observationId, orderedKinds, mutationsPerPersist), + expectFairObservation, + fairnessCalibration, + ) + }, +) + +describe(`shared BrowserWASQLiteDriver fairness oracle`, () => { + it(`selects fixed and seedless lanes by default and only replay when requested`, () => { + expect(createFairnessPropertyModes(readFairnessReplayConfig({}))).toEqual([ + { label: `fixed-seed`, seed: FIXED_SEED }, + { label: `seedless-random` }, + ]) + expect( + createFairnessPropertyModes( + readFairnessReplayConfig({ + TANSTACK_DB_DRIVER_FAIRNESS_SEED: `42`, + TANSTACK_DB_DRIVER_FAIRNESS_PATH: `0:1`, + }), + ), + ).toEqual([{ label: `checked-replay`, seed: 42, path: `0:1` }]) + expect(readFairnessCalibration({})).toEqual({}) + expect( + readFairnessCalibration({ + TANSTACK_DB_DRIVER_FAIRNESS_CALIBRATION: `persist-first-fifo`, + }), + ).toEqual({ schedulingFault: `persist-first-fifo` }) + }) + + it.each([ + { + name: `seed without path`, + environment: { TANSTACK_DB_DRIVER_FAIRNESS_SEED: `42` }, + message: `must be supplied together`, + }, + { + name: `path without seed`, + environment: { TANSTACK_DB_DRIVER_FAIRNESS_PATH: `0` }, + message: `must be supplied together`, + }, + { + name: `non-integer seed`, + environment: { + TANSTACK_DB_DRIVER_FAIRNESS_SEED: `4.2`, + TANSTACK_DB_DRIVER_FAIRNESS_PATH: `0`, + }, + message: `must be an integer`, + }, + { + name: `invalid shrink path`, + environment: { + TANSTACK_DB_DRIVER_FAIRNESS_SEED: `42`, + TANSTACK_DB_DRIVER_FAIRNESS_PATH: `0:-1`, + }, + message: `colon-separated nonnegative integers`, + }, + ])(`rejects incomplete or invalid replay coordinates: $name`, (probe) => { + expect(() => readFairnessReplayConfig(probe.environment)).toThrow( + probe.message, + ) + }) + + it(`reconstructs the known persist-first witness, covers range marginals, and rejects a nearby invalid storm`, async () => { + const knownWitness = buildGeneratedHistory(2, 4, 1, [ + { kind: `persist`, token: `p1` }, + { kind: `persist`, token: `p2` }, + { kind: `persist`, token: `p3` }, + { kind: `hydrate`, token: `h0` }, + { kind: `hydrate`, token: `h1` }, + ]) + expect(knownWitness).toEqual({ + hydrateCount: 2, + persistCount: 4, + mutationsPerPersist: 1, + orderedKinds: [ + `persist`, + `persist`, + `persist`, + `persist`, + `hydrate`, + `hydrate`, + ], + replayHistory: `p0,p1,p2,p3,h0,h1`, + }) + + const marginalHistories = [ + buildGeneratedHistory(2, 3, 1, [ + { kind: `persist`, token: `p1` }, + { kind: `persist`, token: `p2` }, + { kind: `hydrate`, token: `h0` }, + { kind: `hydrate`, token: `h1` }, + ]), + buildGeneratedHistory(7, 7, 3, [ + ...Array.from({ length: 6 }, (_, index) => ({ + kind: `persist` as const, + token: `p${index + 1}`, + })), + ...Array.from({ length: 7 }, (_, index) => ({ + kind: `hydrate` as const, + token: `h${index}`, + })), + ]), + ] + expect( + marginalHistories.map((history) => [ + history.hydrateCount, + history.persistCount, + history.mutationsPerPersist, + ]), + ).toEqual([ + [2, 3, 1], + [7, 7, 3], + ]) + + await expect( + withNodeScenario( + createScenario(`invalid-storm-boundary`, [ + `hydrate`, + `persist`, + `hydrate`, + ]), + expectFairObservation, + ), + ).rejects.toThrow(`must begin with its already-running persist`) + }) + + it(`reaches cold hydration and preserves independently seeded public rows without queued persists`, async () => { + await withNodeScenario( + createScenario(`neutral-reach`, [`hydrate`, `hydrate`, `hydrate`]), + (observation) => { + expectFairObservation(observation) + expect( + observation.rawDequeues.some((entry) => + entry.sql.startsWith( + `SELECT key, value, metadata, row_version FROM`, + ), + ), + ).toBe(true) + }, + ) + }) + + it(`completes a pending cold hydrate before an unrelated persist backlog drains`, async () => { + await withNodeScenario( + createScenario( + `fixed-persist-storm`, + [`persist`, `persist`, `persist`, `hydrate`, `hydrate`], + 2, + ), + expectFairObservation, + ) + }) + + it.each(fairnessPropertyModes)( + `bounds persist completions for generated ordered cold-hydrate/persist histories in $label mode`, + async (mode) => { + const executionsBefore = generatedPropertyExecutions + await fc.assert(generatedFairnessProperty, { + numRuns: GENERATED_RUNS, + verbose: 2, + ...(mode.seed === undefined ? {} : { seed: mode.seed }), + ...(mode.path === undefined ? {} : { path: mode.path }), + }) + expect(generatedPropertyExecutions).toBeGreaterThan(executionsBefore) + }, + ) + + it(`proves tail-order ablation removes a hydrate-before-later-persist history`, () => { + const sampleOptions = { seed: FIXED_SEED, numRuns: GENERATED_RUNS } + const completeGrammar = fc.sample(generatedHistoryArbitrary, sampleOptions) + const tailOrderAblatedGrammar = fc.sample( + tailOrderAblatedArbitrary, + sampleOptions, + ) + + expect(completeGrammar.some(reachesHydrateBeforeLaterPersist)).toBe(true) + expect(tailOrderAblatedGrammar.some(reachesHydrateBeforeLaterPersist)).toBe( + false, + ) + }) + + it(`rejects the named persist-first FIFO wrong answer through the real package path`, async () => { + await withNodeScenario( + createScenario( + `persist-first-fixture-fault`, + [`persist`, `persist`, `persist`, `persist`, `hydrate`, `hydrate`], + 1, + ), + (observation) => { + expectObservationReach(observation) + const violation = findSharedDriverFairnessViolation(observation) + if (!violation) { + throw new Error( + `persist-first scheduling mutant escaped the fairness checker; ` + + `cleanup diagnostics: ${JSON.stringify(observation.cleanupFailures)}`, + ) + } + expect(violation.checkpoint.completedPersistIds).toHaveLength(4) + expect(violation.checkpoint.pendingPersistCount).toBe(0) + expect(observation.cleanupFailures).toEqual([]) + }, + { schedulingFault: `persist-first-fifo` }, + ) + }) + + it(`calibrates the checker against a synthetic persist-first observation`, () => { + const scenario = createScenario( + `persist-first-checker-calibration`, + [`persist`, `persist`, `persist`, `persist`, `hydrate`, `hydrate`], + 1, + ) + const fault = createPersistFirstFaultObservation(scenario) + + expect(findSharedDriverFairnessViolation(fault)).toMatchObject({ + checkpoint: fault.hydrationCompletions[0], + expectedMaximumCompletedPersists: 1, + expectedMinimumPendingPersists: 3, + }) + }) +}) diff --git a/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.ts b/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.ts new file mode 100644 index 0000000000..01666bb94d --- /dev/null +++ b/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.ts @@ -0,0 +1,753 @@ +/** + * # When does a cold hydrate get a turn on a shared SQLite driver? + * + * Contract and source: RFC #1659 accepts K=1 complete-logical-cold-hydrate + * scheduling. The persist already executing when the storm begins is + * non-preemptible. After that, at most one additional persist may complete + * between consecutive hydrate completions, with FIFO identity preserved + * inside the hydrate and persist lanes. + * + * History grammar and domain: a legal storm has unique work IDs, starts with + * that already-running persist when any persist exists, and then permutes + * complete hydrate and persist requests. Hydrates contain nonempty unique + * seeded rows; persists contain one or more mutations. The generated campaign + * varies 2..7 hydrates, 3..7 persists, 1..3 mutations per persist, and sampled + * tail permutations. The minimum grammar still has two hydrate checkpoints and + * a persist backlog beyond K=1. Hydrate count supplies repeated checkpoints; + * persist count varies the backlog; mutation width proves K counts logical + * persists rather than their rows; tail order creates overlapping lane work. + * Neutral histories contain only hydrates. + * + * Independent model and production boundary: `createFairnessReference` + * computes permitted completed persist IDs from the ordered history and K; it + * does not import or simulate the production scheduler. The driver exercises + * public `Collection.preload()` through persisted collection options, the core + * adapter, and one real `BrowserWASQLiteDriver`. Each preload completion is a + * checkpoint after the complete logical hydrate, not after an individual SQL + * statement. + * + * Observed public facts: admitted and completed logical IDs, completed and + * pending persists at every hydrate checkpoint, independently seeded public + * collection rows, raw SQL dequeue reach, and cleanup diagnostics. Known + * omissions: the oracle does not establish elapsed-time latency, unbounded + * eventuality, multi-process coordination, or a browser matrix; the Chromium + * OPFS fixture separately refines the provider boundary. + * + * Challenge and replay: the named persist-first FIFO wrong answer must violate + * the same K=1 checker while using the public/core/driver path. The generated + * property has fixed-seed and seedless-random lanes. Supplying both + * TANSTACK_DB_DRIVER_FAIRNESS_SEED and TANSTACK_DB_DRIVER_FAIRNESS_PATH runs + * only their checked oracle replay. Tail-order ablation must lose the legal + * hydrate-before-later-persist cell. The grammar also reconstructs that known + * wrong-answer witness, covers its bounded marginals, and rejects a storm that + * omits the already-running persist boundary. Cleanup preserves the primary + * failure and reports secondary resource-release diagnostics separately. + */ +import { createCollection } from '../../db/src/index' +import { persistedCollectionOptions } from '../src/index' +import { BrowserWASQLiteDriver } from '../src/wa-sqlite-driver' +import { + SingleProcessCoordinator, + createSQLiteCorePersistenceAdapter, +} from '../../db-sqlite-persistence-core/src/index' +import type { Collection } from '../../db/src/index' +import type { + PersistedCollectionPersistence, + PersistedTx, + SQLiteDriver, +} from '../../db-sqlite-persistence-core/src/index' +import type { BrowserWASQLiteDatabase } from '../src/index' + +export const SHARED_DRIVER_FAIRNESS_BOUND = 1 + +export type FairnessRow = { + id: string + value: number +} + +export type SharedDriverFairnessWork = + | { + kind: `hydrate` + id: string + seededRows: ReadonlyArray + } + | { + kind: `persist` + id: string + mutationsPerPersist: number + } + +export type SharedDriverFairnessScenario = { + id: string + /** + * Logical admission history. A storm begins with the one persist that is + * already non-preemptibly running; the remaining hydrate and persist work + * may be interleaved in any order. + */ + work: ReadonlyArray +} + +export type RawDriverDequeue = { + ordinal: number + sql: string + params: ReadonlyArray +} + +export type RawDriverAdmission = RawDriverDequeue & { + kind: `exec` | `query` | `run` | `transaction` +} + +export type HydrationCompletionCheckpoint = { + collectionId: string + completionOrdinal: number + completedPersistIds: ReadonlyArray + pendingPersistCount: number + rawDequeueCount: number +} + +export type HydratedCollectionRows = { + collectionId: string + rows: ReadonlyArray +} + +export type SharedDriverFairnessObservation = { + scenario: SharedDriverFairnessScenario + admittedHydrateIds: ReadonlyArray + logicalCompletionOrder: ReadonlyArray + hydrationCompletions: ReadonlyArray + driverAdmissions: ReadonlyArray + rawDequeues: ReadonlyArray + hydratedCollections: ReadonlyArray + cleanupFailures: ReadonlyArray +} + +export type SharedDriverFairnessViolation = { + checkpoint: HydrationCompletionCheckpoint + expectedCollectionId: string + expectedMaximumCompletedPersists: number + expectedMinimumPendingPersists: number + permittedCompletedPersistIds: ReadonlyArray + unexpectedCompletedPersistIds: ReadonlyArray +} + +export type SharedDriverFairnessOptions = { + /** Test-only hostile control that preserves the current global FIFO fault. */ + schedulingFault?: `persist-first-fifo` +} + +type Deferred = { + promise: Promise + resolve: () => void +} + +type CloseableSQLiteDriver = SQLiteDriver & { + transactionWithDriver: ( + fn: (transactionDriver: SQLiteDriver) => Promise, + ) => Promise + close: () => Promise +} + +type FairnessReferenceCheckpoint = { + collectionId: string + expectedMaximumCompletedPersists: number + expectedMinimumPendingPersists: number + permittedCompletedPersistIds: ReadonlyArray +} + +export type BrowserWASQLiteDatabaseFactory = () => + | BrowserWASQLiteDatabase + | Promise + +function createDeferred(): Deferred { + let resolve!: () => void + const promise = new Promise((settle) => { + resolve = settle + }) + return { promise, resolve } +} + +function normalizeSql(sql: string): string { + return sql.replace(/\s+/g, ` `).trim() +} + +function failureMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} + +function collectionIdFor( + scenario: SharedDriverFairnessScenario, + work: SharedDriverFairnessWork, +): string { + return `${scenario.id}-${work.id}` +} + +function validateScenario(scenario: SharedDriverFairnessScenario): void { + const hydrateWork = scenario.work.filter((work) => work.kind === `hydrate`) + const persistWork = scenario.work.filter((work) => work.kind === `persist`) + if (hydrateWork.length < 1) { + throw new Error(`work must contain at least one hydrate`) + } + if (persistWork.length > 0 && scenario.work[0]?.kind !== `persist`) { + throw new Error( + `a storm history must begin with its already-running persist`, + ) + } + if ( + new Set(scenario.work.map((work) => work.id)).size !== scenario.work.length + ) { + throw new Error(`work ids must be unique`) + } + for (const work of scenario.work) { + if (work.kind === `persist` && work.mutationsPerPersist < 1) { + throw new Error(`mutationsPerPersist must be at least one`) + } + if (work.kind === `hydrate`) { + if (work.seededRows.length < 1) { + throw new Error(`each hydrate must contain at least one seeded row`) + } + if ( + new Set(work.seededRows.map((row) => row.id)).size !== + work.seededRows.length + ) { + throw new Error(`seeded row ids must be unique within each hydrate`) + } + } + } +} + +class ObservedDatabase implements BrowserWASQLiteDatabase { + readonly rawDequeues: Array = [] + private heldBegin: Deferred | undefined + private beginEntered: Deferred | undefined + + constructor(private readonly database: BrowserWASQLiteDatabase) {} + + holdNextTransactionBegin(): { entered: Promise; release: () => void } { + if (this.heldBegin) { + throw new Error(`a transaction begin is already held`) + } + this.heldBegin = createDeferred() + this.beginEntered = createDeferred() + return { + entered: this.beginEntered.promise, + release: () => this.releaseHeldBegin(), + } + } + + clearTrace(): void { + this.rawDequeues.length = 0 + } + + async execute( + sql: string, + params: ReadonlyArray = [], + ): Promise> { + const normalizedSql = normalizeSql(sql) + this.rawDequeues.push({ + ordinal: this.rawDequeues.length, + sql: normalizedSql, + params: [...params], + }) + + if (normalizedSql === `BEGIN IMMEDIATE` && this.heldBegin) { + const heldBegin = this.heldBegin + this.beginEntered?.resolve() + await heldBegin.promise + } + + return this.database.execute(sql, params) + } + + async close(): Promise { + this.releaseHeldBegin() + await this.database.close?.() + } + + private releaseHeldBegin(): void { + this.heldBegin?.resolve() + this.heldBegin = undefined + this.beginEntered = undefined + } +} + +/** + * A test-only hostile scheduler. It admits calls through the same public/core/ + * BrowserWASQLiteDriver path but serializes them in global admission order, so + * a future fair production driver cannot accidentally make this control pass. + */ +class PersistFirstFIFOFaultDriver implements SQLiteDriver { + private tail = Promise.resolve() + + constructor(private readonly driver: CloseableSQLiteDriver) {} + + exec(sql: string): Promise { + return this.enqueue(() => this.driver.exec(sql)) + } + + query( + sql: string, + params: ReadonlyArray = [], + ): Promise> { + return this.enqueue(() => this.driver.query(sql, params)) + } + + run(sql: string, params: ReadonlyArray = []): Promise { + return this.enqueue(() => this.driver.run(sql, params)) + } + + transaction( + fn: (transactionDriver: SQLiteDriver) => Promise, + ): Promise { + return this.enqueue(() => this.driver.transaction(fn)) + } + + transactionWithDriver( + fn: (transactionDriver: SQLiteDriver) => Promise, + ): Promise { + return this.enqueue(() => this.driver.transactionWithDriver(fn)) + } + + close(): Promise { + return this.enqueue(() => this.driver.close()) + } + + private enqueue(operation: () => Promise): Promise { + const result = this.tail.then(operation) + this.tail = result.then( + () => undefined, + () => undefined, + ) + return result + } +} + +class AdmissionObservedDriver implements SQLiteDriver { + readonly admissions: Array = [] + + constructor( + private readonly driver: CloseableSQLiteDriver, + private readonly onAdmission?: (entry: RawDriverAdmission) => void, + ) {} + + exec(sql: string): Promise { + this.record(`exec`, sql, []) + return this.driver.exec(sql) + } + + query( + sql: string, + params: ReadonlyArray = [], + ): Promise> { + this.record(`query`, sql, params) + return this.driver.query(sql, params) + } + + run(sql: string, params: ReadonlyArray = []): Promise { + this.record(`run`, sql, params) + return this.driver.run(sql, params) + } + + transaction( + fn: (transactionDriver: SQLiteDriver) => Promise, + ): Promise { + this.record(`transaction`, `BEGIN IMMEDIATE`, []) + return this.driver.transaction(fn) + } + + transactionWithDriver( + fn: (transactionDriver: SQLiteDriver) => Promise, + ): Promise { + this.record(`transaction`, `BEGIN IMMEDIATE`, []) + return this.driver.transactionWithDriver(fn) + } + + close(): Promise { + return this.driver.close() + } + + clearAdmissions(): void { + this.admissions.length = 0 + } + + private record( + kind: RawDriverAdmission[`kind`], + sql: string, + params: ReadonlyArray, + ): void { + const entry = { + ordinal: this.admissions.length, + kind, + sql: normalizeSql(sql), + params: [...params], + } + this.admissions.push(entry) + this.onAdmission?.(entry) + } +} + +function createSharedPersistence( + database: BrowserWASQLiteDatabase, + onAdmission?: (entry: RawDriverAdmission) => void, + schedulingFault?: SharedDriverFairnessOptions[`schedulingFault`], +): { + driver: AdmissionObservedDriver + persistence: PersistedCollectionPersistence +} { + const productionDriver = new BrowserWASQLiteDriver({ database }) + const scheduledDriver = + schedulingFault === `persist-first-fifo` + ? new PersistFirstFIFOFaultDriver(productionDriver) + : productionDriver + const driver = new AdmissionObservedDriver(scheduledDriver, onAdmission) + const adapter = createSQLiteCorePersistenceAdapter({ + driver, + schemaMismatchPolicy: `sync-absent-error`, + appliedTxPruneMaxRows: 0, + appliedTxPruneMaxAgeSeconds: 0, + }) + return { + driver, + persistence: { + adapter, + coordinator: new SingleProcessCoordinator(), + }, + } +} + +function createPersistedTx( + collectionId: string, + sequence: number, + mutationsPerPersist: number, +): PersistedTx { + return { + txId: `${collectionId}-tx-${sequence}`, + term: 1, + seq: sequence, + rowVersion: sequence, + mutations: Array.from({ length: mutationsPerPersist }, (_, index) => ({ + type: `insert` as const, + key: `${sequence}-${index}`, + value: { + id: `${sequence}-${index}`, + value: sequence * 100 + index, + }, + })), + } +} + +function createFairnessReference( + scenario: SharedDriverFairnessScenario, + fairnessBound: number, +): ReadonlyArray { + const persistIds = scenario.work + .filter((work) => work.kind === `persist`) + .map((work) => collectionIdFor(scenario, work)) + const hydrateIds = scenario.work + .filter((work) => work.kind === `hydrate`) + .map((work) => collectionIdFor(scenario, work)) + + return hydrateIds.map((collectionId, hydrateIndex) => { + // The first persist is the one already executing when hydrate work reaches + // the queue. The ordered reference then permits at most K additional + // persists between successive hydrate completions, preserving FIFO order + // within each logical lane. + const expectedMaximumCompletedPersists = Math.min( + persistIds.length, + persistIds.length === 0 ? 0 : 1 + hydrateIndex * fairnessBound, + ) + return { + collectionId, + expectedMaximumCompletedPersists, + expectedMinimumPendingPersists: + persistIds.length - expectedMaximumCompletedPersists, + permittedCompletedPersistIds: persistIds.slice( + 0, + expectedMaximumCompletedPersists, + ), + } + }) +} + +export function findSharedDriverFairnessViolation( + observation: SharedDriverFairnessObservation, + fairnessBound = SHARED_DRIVER_FAIRNESS_BOUND, +): SharedDriverFairnessViolation | undefined { + const reference = createFairnessReference(observation.scenario, fairnessBound) + for (const [ + hydrateIndex, + checkpoint, + ] of observation.hydrationCompletions.entries()) { + const expected = reference[hydrateIndex] + if (!expected) continue + const permitted = new Set(expected.permittedCompletedPersistIds) + const unexpectedCompletedPersistIds = checkpoint.completedPersistIds.filter( + (id) => !permitted.has(id), + ) + if ( + checkpoint.collectionId !== expected.collectionId || + checkpoint.completedPersistIds.length > + expected.expectedMaximumCompletedPersists || + checkpoint.pendingPersistCount < + expected.expectedMinimumPendingPersists || + unexpectedCompletedPersistIds.length > 0 + ) { + return { + checkpoint, + expectedCollectionId: expected.collectionId, + expectedMaximumCompletedPersists: + expected.expectedMaximumCompletedPersists, + expectedMinimumPendingPersists: expected.expectedMinimumPendingPersists, + permittedCompletedPersistIds: expected.permittedCompletedPersistIds, + unexpectedCompletedPersistIds, + } + } + } + return undefined +} + +/** Checker calibration only; the executable hostile control uses the fixture. */ +export function createPersistFirstFaultObservation( + scenario: SharedDriverFairnessScenario, +): SharedDriverFairnessObservation { + const persistIds = scenario.work + .filter((work) => work.kind === `persist`) + .map((work) => collectionIdFor(scenario, work)) + const firstHydrate = scenario.work.find((work) => work.kind === `hydrate`) + if (!firstHydrate) throw new Error(`fault observation requires a hydrate`) + const firstHydrateId = collectionIdFor(scenario, firstHydrate) + return { + scenario, + admittedHydrateIds: [firstHydrateId], + logicalCompletionOrder: [ + ...persistIds.map((id) => `persist:${id}`), + `hydrate:${firstHydrateId}`, + ], + hydrationCompletions: [ + { + collectionId: firstHydrateId, + completionOrdinal: persistIds.length, + completedPersistIds: persistIds, + pendingPersistCount: 0, + rawDequeueCount: persistIds.length + 1, + }, + ], + rawDequeues: [], + driverAdmissions: [], + hydratedCollections: [], + cleanupFailures: [], + } +} + +/** + * Runs the public persisted-collection startup path over one shared + * BrowserWASQLiteDriver. The expected scheduler is deliberately not imported: + * the oracle observes only logical admissions/completions and raw SQL dequeue. + */ +export async function observeSharedDriverFairness( + openDatabase: BrowserWASQLiteDatabaseFactory, + scenario: SharedDriverFairnessScenario, + options: SharedDriverFairnessOptions = {}, +): Promise { + validateScenario(scenario) + + const hydrateWork = scenario.work.filter((work) => work.kind === `hydrate`) + const persistWork = scenario.work.filter((work) => work.kind === `persist`) + const hydrateIds = hydrateWork.map((work) => collectionIdFor(scenario, work)) + const persistIds = persistWork.map((work) => collectionIdFor(scenario, work)) + const admittedHydrateIds: Array = [] + const allHydratesRequested = createDeferred() + + // A separate connection and adapter own seed/setup. Closing and reopening + // makes every measured hydration cold at the adapter and driver layers. + const seedDatabase = await openDatabase() + const seed = createSharedPersistence(seedDatabase) + let seedPrimaryFailure: unknown + try { + for (const work of hydrateWork) { + const collectionId = collectionIdFor(scenario, work) + await seed.persistence.adapter.applyCommittedTx(collectionId, { + txId: `${collectionId}-seed`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: work.seededRows.map((row) => ({ + type: `insert` as const, + key: row.id, + value: { ...row }, + })), + }) + } + for (const collectionId of persistIds) { + await seed.persistence.adapter.loadSubset(collectionId, {}) + } + } catch (error) { + seedPrimaryFailure = error + } + let seedCleanupFailure: unknown + try { + await seed.driver.close() + } catch (error) { + seedCleanupFailure = error + } + if (seedPrimaryFailure !== undefined) { + if (seedCleanupFailure !== undefined) { + throw new Error( + `${failureMessage(seedPrimaryFailure)}; seed cleanup diagnostics: ${failureMessage(seedCleanupFailure)}`, + ) + } + throw seedPrimaryFailure + } + if (seedCleanupFailure !== undefined) throw seedCleanupFailure + + const observedDatabase = new ObservedDatabase(await openDatabase()) + const { driver, persistence } = createSharedPersistence( + observedDatabase, + undefined, + options.schedulingFault, + ) + + const collections: Array> = [] + const persistPromises: Array> = [] + const preloadPromises: Array> = [] + const logicalCompletionOrder: Array = [] + const completedPersistIds: Array = [] + const hydrationCompletions: Array = [] + const observedRows = new Map>() + const cleanupFailures: Array = [] + let releaseHeldBegin: (() => void) | undefined + let beginEntered: Promise | undefined + let persistSequence = 0 + let primaryFailure: unknown + let observation: SharedDriverFairnessObservation | undefined + + try { + // Cache only the unrelated persist tables. Hydrate tables remain cold. + for (const collectionId of persistIds) { + await persistence.adapter.loadSubset(collectionId, {}) + } + + observedDatabase.clearTrace() + driver.clearAdmissions() + + if (persistIds.length > 0) { + const heldBegin = observedDatabase.holdNextTransactionBegin() + releaseHeldBegin = heldBegin.release + beginEntered = heldBegin.entered + } + + // Admit the explicit legal history while the first persist is held. The + // first item is the non-preemptible transaction; every tail permutation + // is therefore observable at the same deterministic release checkpoint. + for (const work of scenario.work) { + const collectionId = collectionIdFor(scenario, work) + if (work.kind === `persist`) { + persistSequence += 1 + const sequence = persistSequence + const persist = persistence.adapter + .applyCommittedTx( + collectionId, + createPersistedTx(collectionId, sequence, work.mutationsPerPersist), + ) + .then(() => { + completedPersistIds.push(collectionId) + logicalCompletionOrder.push(`persist:${collectionId}`) + }) + persistPromises.push(persist) + continue + } + + const collection = createCollection( + persistedCollectionOptions({ + id: collectionId, + getKey: (row) => row.id, + persistence, + }), + ) + collections.push(collection) + const preloadRequest = collection.preload() + admittedHydrateIds.push(collectionId) + if (admittedHydrateIds.length === hydrateIds.length) { + allHydratesRequested.resolve() + } + const preload = preloadRequest.then(() => { + logicalCompletionOrder.push(`hydrate:${collectionId}`) + observedRows.set( + collectionId, + collection.toArray.map((row) => ({ id: row.id, value: row.value })), + ) + hydrationCompletions.push({ + collectionId, + completionOrdinal: logicalCompletionOrder.length - 1, + completedPersistIds: [...completedPersistIds], + pendingPersistCount: persistIds.length - completedPersistIds.length, + rawDequeueCount: observedDatabase.rawDequeues.length, + }) + }) + preloadPromises.push(preload) + } + + await beginEntered + await allHydratesRequested.promise + // Every public preload request is now pending. Releasing the one + // non-preemptible persist here leaves production responsible for admitting + // and completing each full logical hydrate under the approved scheduler. + releaseHeldBegin?.() + releaseHeldBegin = undefined + + await Promise.all([...persistPromises, ...preloadPromises]) + + observation = { + scenario, + admittedHydrateIds, + logicalCompletionOrder, + hydrationCompletions, + driverAdmissions: driver.admissions.map((entry) => ({ + ...entry, + params: [...entry.params], + })), + rawDequeues: observedDatabase.rawDequeues.map((entry) => ({ + ...entry, + params: [...entry.params], + })), + hydratedCollections: hydrateWork.map((work) => { + const collectionId = collectionIdFor(scenario, work) + return { + collectionId, + rows: + observedRows.get(collectionId)?.map((row) => ({ ...row })) ?? [], + } + }), + cleanupFailures, + } + } catch (error) { + primaryFailure = error + } finally { + releaseHeldBegin?.() + await Promise.allSettled([...persistPromises, ...preloadPromises]) + for (const collection of collections) { + try { + await collection.cleanup() + } catch (error) { + cleanupFailures.push(failureMessage(error)) + } + } + try { + await driver.close() + } catch (error) { + cleanupFailures.push(failureMessage(error)) + } + } + + if (primaryFailure !== undefined) { + if (cleanupFailures.length > 0) { + throw new Error( + `${failureMessage(primaryFailure)}; active cleanup diagnostics: ${JSON.stringify(cleanupFailures)}`, + ) + } + throw primaryFailure + } + if (!observation) { + throw new Error(`shared-driver observation ended without a result`) + } + return observation +} diff --git a/packages/browser-db-sqlite-persistence/tsconfig.json b/packages/browser-db-sqlite-persistence/tsconfig.json index 67f70f2efe..67dc24da8a 100644 --- a/packages/browser-db-sqlite-persistence/tsconfig.json +++ b/packages/browser-db-sqlite-persistence/tsconfig.json @@ -22,6 +22,14 @@ ] } }, - "include": ["src", "tests", "e2e", "vite.config.ts", "vitest.e2e.config.ts"], + "include": [ + "src", + "tests", + "e2e", + "vite.config.ts", + "vite.opfs.config.ts", + "vitest.e2e.config.ts", + "playwright.opfs.config.ts" + ], "exclude": ["node_modules", "dist"] } diff --git a/packages/browser-db-sqlite-persistence/vite.opfs.config.ts b/packages/browser-db-sqlite-persistence/vite.opfs.config.ts new file mode 100644 index 0000000000..c66d394740 --- /dev/null +++ b/packages/browser-db-sqlite-persistence/vite.opfs.config.ts @@ -0,0 +1,30 @@ +import { dirname, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { defineConfig } from 'vite' + +const packageDirectory = dirname(fileURLToPath(import.meta.url)) + +export default defineConfig({ + base: `./`, + // wa-sqlite locates its sibling WASM file through import.meta.url. Keeping + // the module out of Vite's dependency prebundle preserves that relationship + // for this real-browser fixture. + optimizeDeps: { + exclude: [`@journeyapps/wa-sqlite`], + }, + resolve: { + alias: { + '@tanstack/db': resolve(packageDirectory, `../db/src`), + '@tanstack/db-ivm': resolve(packageDirectory, `../db-ivm/src`), + '@tanstack/db-sqlite-persistence-core': resolve( + packageDirectory, + `../db-sqlite-persistence-core/src`, + ), + }, + }, + server: { + fs: { + allow: [resolve(packageDirectory, `../..`)], + }, + }, +}) diff --git a/packages/db-sqlite-persistence-core/src/broadcast-coordinator.ts b/packages/db-sqlite-persistence-core/src/broadcast-coordinator.ts index df8602599f..6d39cf10ec 100644 --- a/packages/db-sqlite-persistence-core/src/broadcast-coordinator.ts +++ b/packages/db-sqlite-persistence-core/src/broadcast-coordinator.ts @@ -20,6 +20,7 @@ import type { ApplyLocalMutationsResponse, EnsureRemoteSubsetRequest, EnsureRemoteSubsetResponse, + HydrationPersistenceAdapter, PersistedCollectionCoordinator, PersistedIndexSpec, PersistedMutationEnvelope, @@ -251,6 +252,9 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi private readonly pendingRPCs = new Map() private readonly appliedEnvelopes = new Map() private readonly inFlightEnvelopes = new Map() + private appliedEnvelopePruneTimer: ReturnType | null = null + private rejectDisposed: ((error: Error) => void) | null = null + private readonly disposedPromise: Promise private disposed = false /** Method indirection to prevent TypeScript from narrowing `disposed` across awaits */ @@ -264,8 +268,11 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi } private requireAdapter(collectionId: string): CoordinatorAdapter { - const adapter = - this.collectionAdapters.get(collectionId) ?? this.defaultAdapter + let adapter = this.collectionAdapters.get(collectionId) + if (!adapter && this.defaultAdapter) { + adapter = this.defaultAdapter + this.collectionAdapters.set(collectionId, adapter) + } if (!adapter) { throw new Error( `${this.coordinatorName}: adapter not set for collection "${collectionId}". Call setAdapterForCollection() before using leader-side operations.`, @@ -282,6 +289,10 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi this.channel.onmessage = (event: MessageEvent) => { this.onChannelMessage(event.data) } + this.disposedPromise = new Promise((_resolve, reject) => { + this.rejectDisposed = reject + }) + void this.disposedPromise.catch(() => undefined) } /** @@ -338,9 +349,9 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi state.subscribers.add(onMessage) return () => { state.subscribers.delete(onMessage) - if (state.subscribers.size === 0 && state.participatesInLeadership) { + if (state.subscribers.size === 0) { state.participatesInLeadership = false - this.releaseLeadership(collectionId, state) + this.releaseCollectionIfUnused(collectionId, state) } } } @@ -591,9 +602,12 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi collectionId: string, signature: string, spec: PersistedIndexSpec, + scopedAdapter?: HydrationPersistenceAdapter, + localEnsureCompleted = false, ): Promise { if (this.isLeader(collectionId)) { - await this.requireAdapter(collectionId).ensureIndex( + if (localEnsureCompleted) return + await (scopedAdapter ?? this.requireAdapter(collectionId)).ensureIndex( collectionId, signature, spec, @@ -644,6 +658,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi async requestApplyCommittedTx( collectionId: string, tx: PersistedTx, + scopedAdapter?: HydrationPersistenceAdapter, ): Promise { const request: Extract = { type: `rpc:applyCommittedTx:req`, @@ -652,7 +667,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi tx, } if (this.isLeader(collectionId)) { - return this.handleApplyCommittedTx(collectionId, request) + return this.handleApplyCommittedTx(collectionId, request, scopedAdapter) } return this.sendRPC(collectionId, request) @@ -661,13 +676,18 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi async pullSince( collectionId: string, fromRowVersion: number, + scopedAdapter?: HydrationPersistenceAdapter, ): Promise { if (this.isLeader(collectionId)) { - return this.handlePullSince(collectionId, { - type: `rpc:pullSince:req`, - rpcId: safeRandomUUID(), - fromRowVersion, - }) + return this.handlePullSince( + collectionId, + { + type: `rpc:pullSince:req`, + rpcId: safeRandomUUID(), + fromRowVersion, + }, + scopedAdapter, + ) } return this.sendRPC(collectionId, { @@ -682,7 +702,11 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi // ----------------------------------------------------------------------- dispose(): void { + if (this.disposed) return this.disposed = true + const disposedError = new Error(`coordinator disposed`) + this.rejectDisposed?.(disposedError) + this.rejectDisposed = null for (const acquisition of this.outboundRemoteSubsetAcquisitions.values()) { this.cancelRemoteSubsetReplayRetry(acquisition) @@ -697,9 +721,13 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi for (const [, pending] of this.pendingRPCs) { clearTimeout(pending.timer) - pending.reject(new Error(`coordinator disposed`)) + pending.reject(disposedError) } this.pendingRPCs.clear() + if (this.appliedEnvelopePruneTimer !== null) { + clearTimeout(this.appliedEnvelopePruneTimer) + this.appliedEnvelopePruneTimer = null + } this.channel.close() this.collections.clear() @@ -765,7 +793,13 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi lockName, { signal: abortController.signal }, async () => { - if (this.isDisposed()) return + if ( + this.isDisposed() || + !this.participatesInLeadership(state) || + this.collections.get(collectionId) !== state + ) { + return + } try { // Restore stream position from DB before claiming leadership @@ -780,6 +814,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi if ( this.isDisposed() || !this.participatesInLeadership(state) || + this.collections.get(collectionId) !== state || abortController.signal.aborted || state.lockAbortController !== abortController ) { @@ -825,7 +860,11 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi return } console.warn(`Failed to acquire leadership for ${collectionId}:`, error) - if (!this.isDisposed() && this.participatesInLeadership(state)) { + if ( + !this.isDisposed() && + this.participatesInLeadership(state) && + this.collections.get(collectionId) === state + ) { await sleep(LEADERSHIP_RETRY_DELAY_MS) } } finally { @@ -838,6 +877,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi if ( !this.isDisposed() && this.participatesInLeadership(state) && + this.collections.get(collectionId) === state && state.lockAbortController === null ) { void this.acquireLeadership(collectionId, state) @@ -910,6 +950,26 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi this.notifyRouteWaiters(state) } + private releaseCollectionIfUnused( + collectionId: string, + state: CollectionState, + ): void { + if ( + state.subscribers.size > 0 || + this.collections.get(collectionId) !== state + ) { + return + } + this.releaseLeadership(collectionId, state) + this.collections.delete(collectionId) + this.collectionAdapters.delete(collectionId) + const prefix = `${JSON.stringify([collectionId]).slice(0, -1)},` + for (const key of this.appliedEnvelopes.keys()) { + if (key.startsWith(prefix)) this.appliedEnvelopes.delete(key) + } + this.pruneAppliedEnvelopes() + } + private postRemoteSubsetRelease( acquisition: OutboundRemoteSubsetAcquisition, ): void { @@ -1122,6 +1182,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi collectionId: string, request: RPCRequest, ): Promise { + if (this.isDisposed()) throw new Error(`coordinator disposed`) let lastError: Error | undefined let firstTransportCause: unknown const mutationRequestType = isMutatingRPCRequest(request) @@ -1133,8 +1194,12 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi for (let attempt = 0; attempt <= RPC_RETRY_ATTEMPTS; attempt++) { if (attempt > 0) { - await sleep(RPC_RETRY_DELAY_MS * attempt) + await Promise.race([ + sleep(RPC_RETRY_DELAY_MS * attempt), + this.disposedPromise, + ]) } + if (this.isDisposed()) throw new Error(`coordinator disposed`) if ( mutationRoute && @@ -1763,7 +1828,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi latestRowVersion: tx.rowVersion, acceptedMutationIds: request.mutations.map((m) => m.mutationId), } - if (this.isDisposed()) { + if (this.isDisposed() || this.collections.get(collectionId) !== state) { return response } this.appliedEnvelopes.set( @@ -1815,6 +1880,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi private async handleApplyCommittedTx( collectionId: string, request: Extract, + scopedAdapter?: HydrationPersistenceAdapter, ): Promise { const envelopeKey = appliedEnvelopeKey(collectionId, request.envelopeId) return this.runDeduplicatedEnvelope({ @@ -1835,7 +1901,8 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi code: `CONFLICT`, error: `envelope ${request.envelopeId} ${phase}`, }), - apply: () => this.applyCommittedTxOnce(collectionId, request), + apply: () => + this.applyCommittedTxOnce(collectionId, request, scopedAdapter), createPending: (response) => ({ requestType: request.type, response, @@ -1887,6 +1954,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi private async applyCommittedTxOnce( collectionId: string, request: Extract, + scopedAdapter?: HydrationPersistenceAdapter, ): Promise { const state = this.collections.get(collectionId) if (!state || !state.isLeader) { @@ -1903,6 +1971,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi collectionId, state, request.tx, + scopedAdapter, ) const response: ApplyCommittedTxResponse = { type: `rpc:applyCommittedTx:res`, @@ -1912,7 +1981,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi seq: tx.seq, latestRowVersion: tx.rowVersion, } - if (this.isDisposed()) { + if (this.isDisposed() || this.collections.get(collectionId) !== state) { return response } this.appliedEnvelopes.set( @@ -1972,6 +2041,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi collectionId: string, state: CollectionState, pendingTx: Omit, + scopedAdapter?: HydrationPersistenceAdapter, ): Promise { return this.withWriterLock(async () => { const tx: PersistedTx = { @@ -1980,7 +2050,7 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi seq: state.latestSeq + 1, rowVersion: state.latestRowVersion + 1, } - const adapter = this.requireAdapter(collectionId) + const adapter = scopedAdapter ?? this.requireAdapter(collectionId) try { await adapter.applyCommittedTx(collectionId, tx) } catch (error) { @@ -1999,10 +2069,11 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi rpcId: string fromRowVersion: number }, + scopedAdapter?: HydrationPersistenceAdapter, ): Promise { const state = this.collections.get(collectionId) - const adapter = this.requireAdapter(collectionId) + const adapter = scopedAdapter ?? this.requireAdapter(collectionId) if (!adapter.pullSince) { return { type: `rpc:pullSince:res`, @@ -2082,13 +2153,30 @@ export class BroadcastCollectionCoordinator implements PersistedCollectionCoordi // ----------------------------------------------------------------------- private pruneAppliedEnvelopes(): void { - // Keep envelopes for 60 seconds for dedup + if (this.appliedEnvelopePruneTimer !== null) { + clearTimeout(this.appliedEnvelopePruneTimer) + this.appliedEnvelopePruneTimer = null + } const cutoff = Date.now() - RPC_DEDUPE_RETENTION_MS for (const [key, envelope] of this.appliedEnvelopes) { - if (envelope.appliedAt < cutoff) { + if (envelope.appliedAt <= cutoff) { this.appliedEnvelopes.delete(key) } } + if (this.disposed || this.appliedEnvelopes.size === 0) return + + let earliestAppliedAt = Number.POSITIVE_INFINITY + for (const envelope of this.appliedEnvelopes.values()) { + earliestAppliedAt = Math.min(earliestAppliedAt, envelope.appliedAt) + } + const delay = Math.max( + 0, + earliestAppliedAt + RPC_DEDUPE_RETENTION_MS - Date.now(), + ) + this.appliedEnvelopePruneTimer = setTimeout(() => { + this.appliedEnvelopePruneTimer = null + this.pruneAppliedEnvelopes() + }, delay) } private setReleasedRemoteSubsetAcquisition( diff --git a/packages/db-sqlite-persistence-core/src/persisted.ts b/packages/db-sqlite-persistence-core/src/persisted.ts index 07a4ab17d8..3e00c491ce 100644 --- a/packages/db-sqlite-persistence-core/src/persisted.ts +++ b/packages/db-sqlite-persistence-core/src/persisted.ts @@ -320,6 +320,21 @@ type PersistedResumeGeneration = { resetEpoch: number } +export type PersistencePullSinceResult = + | { + latestRowVersion: number + requiresFullReload: true + } + | { + latestRowVersion: number + requiresFullReload: false + changedKeys: Array + deletedKeys: Array + deltas?: Array< + ReplayableTxDelta, string | number> + > + } + export type PersistedTx< T extends object = Record, TKey extends string | number = string | number, @@ -350,6 +365,21 @@ export type PersistedTx< collectionMetadataMutations?: Array } +/** + * Opaque identity shared by every adapter over the same physical SQLite driver. + * The core adapter uses it to serialize non-preemptible logical operations + * while alternating one regular operation between queued hydrations. + * + * Delegating drivers must forward this property before they are passed to a + * SQLite persistence adapter. A driver may also brand each returned Promise + * with the same key for late capability discovery, but a wrapper must return + * that exact Promise: discovering the key after a call cannot retroactively + * schedule the wrapper's first logical operation. + */ +export const SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY = Symbol.for( + `tanstack-db.sqlite-driver-supports-shared-logical-scheduling`, +) + export interface PersistenceAdapter { loadSubset: ( collectionId: string, @@ -400,6 +430,23 @@ export interface PersistenceAdapter { latestSeq: number latestRowVersion: number }> + /** + * Runs one complete logical hydrate as a non-preemptible scheduler unit. + * The callback must use the supplied unscheduled adapter for all nested + * persistence work and must not retain it after the callback settles. + */ + runInHydrationScope?: ( + task: (adapter: HydrationPersistenceAdapter) => Promise, + ) => Promise + /** Whether hydration scopes currently enter a shared driver scheduler. */ + isHydrationScopeScheduled?: () => boolean +} + +export type HydrationPersistenceAdapter = PersistenceAdapter & { + pullSince?: ( + collectionId: string, + fromRowVersion: number, + ) => Promise } export type { RemoteSubsetOwner } from './remote-subset-owner' @@ -411,6 +458,7 @@ type SingleProcessRemoteSubsetAcquisition = { } export interface SQLiteDriver { + readonly [SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY]?: object exec: (sql: string) => Promise query: ( sql: string, @@ -425,8 +473,30 @@ export interface SQLiteDriver { ) => Promise } +/** + * Forwards a driver's shared logical scheduling identity to a transparent + * delegating driver before the wrapper is used by a persistence adapter. + */ +export function forwardSQLiteDriverSharedLogicalScheduling< + TDriver extends SQLiteDriver, +>(source: SQLiteDriver, wrapper: TDriver): TDriver { + const key = source[SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY] + if (key) { + Object.defineProperty( + wrapper, + SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY, + { value: key }, + ) + } + return wrapper +} + export interface PersistedCollectionCoordinator { getNodeId: () => string + setAdapterForCollection?: ( + collectionId: string, + adapter: PersistenceAdapter, + ) => void subscribe: ( collectionId: string, onMessage: (message: ProtocolEnvelope) => void, @@ -446,10 +516,17 @@ export interface PersistedCollectionCoordinator { collectionId: string, owner: RemoteSubsetOwner, ) => () => void + /** + * Requests leader-side index creation. The scoped adapter is leader-local + * and is never serialized to a follower. `localEnsureCompleted` lets the + * built-in leader avoid repeating successful local work. + */ requestEnsurePersistedIndex: ( collectionId: string, signature: string, spec: PersistedIndexSpec, + scopedAdapter?: HydrationPersistenceAdapter, + localEnsureCompleted?: boolean, ) => Promise requestApplyLocalMutations?: ( collectionId: string, @@ -458,10 +535,13 @@ export interface PersistedCollectionCoordinator { requestApplyCommittedTx: ( collectionId: string, tx: PersistedTx, + scopedAdapter?: HydrationPersistenceAdapter, ) => Promise + /** The scoped adapter is leader-local and is never serialized to a follower. */ pullSince?: ( collectionId: string, fromRowVersion: number, + scopedAdapter?: HydrationPersistenceAdapter, ) => Promise } @@ -1128,6 +1208,8 @@ class PersistedCollectionRuntime< private readonly activeSubsets = new Map() private activeHydrationContext: { suppliedRowKeys: Set } | undefined private hydrationSequence = 0 + // An unscheduled startup read may finish after a coordinator reset publishes. + private resetSequence = 0 private readonly pendingRemoteSubsetEnsures = new Map< string, LoadSubsetOptions @@ -1151,9 +1233,9 @@ class PersistedCollectionRuntime< metadata: null, markError: null, } - private started = false private startupMetadataPromise: Promise | null = null private startPromise: Promise | null = null + private hasAttemptedStartup = false private resumeBaselinePromise: Promise | null = null private resumeCertificationPromise: Promise | null = null private persistedKeySetEvidence: PersistedKeySetEvidence | undefined @@ -1341,6 +1423,16 @@ class PersistedCollectionRuntime< } } + private runInHydrationScope( + task: (adapter: HydrationPersistenceAdapter) => Promise, + adapter: HydrationPersistenceAdapter = this.persistence.adapter, + ): Promise { + if (adapter.runInHydrationScope) { + return adapter.runInHydrationScope(task) + } + return Promise.resolve().then(() => task(adapter)) + } + async ensureStarted(): Promise { this.throwIfTerminal() if (this.startPromise) { @@ -1348,11 +1440,110 @@ class PersistedCollectionRuntime< } const lifecycleGeneration = this.lifecycleGeneration - this.startPromise = this.startInternal(lifecycleGeneration).catch( - (error) => { - throw this.markTerminalFailure(error, lifecycleGeneration) - }, - ) + const isRestart = this.hasAttemptedStartup + this.hasAttemptedStartup = true + let resolveStartupMetadata!: () => void + let rejectStartupMetadata!: (error: unknown) => void + this.startupMetadataPromise = new Promise((resolve, reject) => { + resolveStartupMetadata = resolve + rejectStartupMetadata = reject + }) + void this.startupMetadataPromise.catch(() => undefined) + + this.startPromise = (async () => { + const loadStartupMetadata = async ( + adapter: HydrationPersistenceAdapter, + ) => { + if (lifecycleGeneration !== this.lifecycleGeneration) { + resolveStartupMetadata() + return false + } + + try { + await this.loadStartupMetadataInternal(lifecycleGeneration, adapter) + return lifecycleGeneration === this.lifecycleGeneration + } catch (error) { + rejectStartupMetadata(error) + throw error + } + } + + let startup: + | { + appliedCursor: number | undefined + indexBootstrapSnapshot: Array + completedLocalIndexSignatures: Set + } + | undefined + const scheduleStartupAsOneHydrate = + this.persistence.adapter.runInHydrationScope !== undefined && + (!isRestart || + (this.persistence.adapter.isHydrationScopeScheduled?.() ?? true)) + if (scheduleStartupAsOneHydrate) { + startup = await this.applyMutex.run(async () => { + const result = await this.runInHydrationScope(async (adapter) => { + if (!(await loadStartupMetadata(adapter))) return undefined + return this.startInternal( + lifecycleGeneration, + adapter, + resolveStartupMetadata, + ) + }) + if (lifecycleGeneration === this.lifecycleGeneration) { + await this.flushQueuedTxCommittedUnsafe() + } + return result + }) + } else { + // Preserve the existing unscheduled-adapter lifecycle contract: a + // replacement upstream may start while stale hydration is settling. + if (await loadStartupMetadata(this.persistence.adapter)) { + if (this.persistence.adapter.isHydrationScopeScheduled?.()) { + startup = await this.applyMutex.run(async () => { + const result = await this.runInHydrationScope((adapter) => + this.startInternal( + lifecycleGeneration, + adapter, + resolveStartupMetadata, + ), + ) + if (lifecycleGeneration === this.lifecycleGeneration) { + await this.flushQueuedTxCommittedUnsafe() + } + return result + }) + } else { + startup = await this.startInternal( + lifecycleGeneration, + this.persistence.adapter, + resolveStartupMetadata, + ) + if (lifecycleGeneration === this.lifecycleGeneration) { + await this.flushQueuedTxCommittedUnsafe() + } + } + } + } + if ( + startup !== undefined && + lifecycleGeneration === this.lifecycleGeneration + ) { + await this.requestCoordinatorPersistedIndexes( + startup.indexBootstrapSnapshot, + startup.completedLocalIndexSignatures, + ) + if ( + startup.appliedCursor !== undefined && + lifecycleGeneration === this.lifecycleGeneration + ) { + await this.waitForAppliedReceiptsAfter(startup.appliedCursor) + } + } + resolveStartupMetadata() + })().catch((error) => { + rejectStartupMetadata(error) + throw this.markTerminalFailure(error, lifecycleGeneration) + }) void this.startPromise.catch(() => undefined) return this.startPromise } @@ -1369,7 +1560,21 @@ class PersistedCollectionRuntime< if (lifecycleGeneration !== this.lifecycleGeneration) return if (this.syncMode !== `on-demand`) return - await this.hydrateBaseline(lifecycleGeneration) + const appliedCursor = await this.applyMutex.run(async () => { + const result = await this.runInHydrationScope((adapter) => + this.hydrateBaseline(lifecycleGeneration, adapter), + ) + if (lifecycleGeneration === this.lifecycleGeneration) { + await this.flushQueuedTxCommittedUnsafe() + } + return result + }) + if ( + appliedCursor !== undefined && + lifecycleGeneration === this.lifecycleGeneration + ) { + await this.waitForAppliedReceiptsAfter(appliedCursor) + } })().catch((error) => { throw this.markTerminalFailure(error, lifecycleGeneration) }) @@ -1407,22 +1612,27 @@ class PersistedCollectionRuntime< return this.resumeGenerationOwner } - private async hydrateBaseline(lifecycleGeneration: number): Promise { - if (lifecycleGeneration !== this.lifecycleGeneration) return + private async hydrateBaseline( + lifecycleGeneration: number, + adapter: HydrationPersistenceAdapter, + ): Promise { + if (lifecycleGeneration !== this.lifecycleGeneration) return undefined const baseline = {} this.activeSubsets.set(this.getSubsetKey(baseline), baseline) const appliedCursor = this.appliedReceiptSequence - await this.applyMutex.run(async () => { - if (lifecycleGeneration !== this.lifecycleGeneration) return - await this.hydrateSubsetUnsafe(baseline, { + await this.hydrateSubsetUnsafe( + baseline, + { requestRemoteEnsure: false, lifecycleGeneration, bindKeySetEvidence: true, - }) - }) - if (lifecycleGeneration !== this.lifecycleGeneration) return - await this.waitForAppliedReceiptsAfter(appliedCursor) + }, + adapter, + ) + return lifecycleGeneration === this.lifecycleGeneration + ? appliedCursor + : undefined } async ensureStartupMetadataLoaded(): Promise { @@ -1431,42 +1641,56 @@ class PersistedCollectionRuntime< return this.startupMetadataPromise } - const lifecycleGeneration = this.lifecycleGeneration - this.startupMetadataPromise = this.loadStartupMetadataInternal( - lifecycleGeneration, - ).catch((error) => { - throw this.markTerminalFailure(error, lifecycleGeneration) - }) - return this.startupMetadataPromise + void this.ensureStarted() + return this.startupMetadataPromise! } - private async startInternal(lifecycleGeneration: number): Promise { - if (this.started) { - return - } - - this.started = true - - await this.ensureStartupMetadataLoaded() - if (lifecycleGeneration !== this.lifecycleGeneration) return + private async startInternal( + lifecycleGeneration: number, + adapter: HydrationPersistenceAdapter, + onStartupMetadataLoaded: () => void, + ): Promise< + | { + appliedCursor: number | undefined + indexBootstrapSnapshot: Array + completedLocalIndexSignatures: Set + } + | undefined + > { + if (lifecycleGeneration !== this.lifecycleGeneration) return undefined const indexBootstrapSnapshot = this.collection?.getIndexMetadata() ?? [] this.attachIndexLifecycleListeners() - await this.bootstrapPersistedIndexes(indexBootstrapSnapshot) - if (lifecycleGeneration !== this.lifecycleGeneration) return - - if (this.syncMode !== `on-demand`) { - await this.hydrateBaseline(lifecycleGeneration) - } + const completedLocalIndexSignatures = await this.bootstrapPersistedIndexes( + indexBootstrapSnapshot, + adapter, + ) + if (lifecycleGeneration !== this.lifecycleGeneration) return undefined + + // Let the source run only once the startup hydrate is about to begin. + // Its first transaction must bind to that hydrate's sequence, not the + // sequence before index bootstrap yielded. + onStartupMetadataLoaded() + const appliedCursor = + this.syncMode !== `on-demand` + ? await this.hydrateBaseline(lifecycleGeneration, adapter) + : undefined + return lifecycleGeneration === this.lifecycleGeneration + ? { + appliedCursor, + indexBootstrapSnapshot, + completedLocalIndexSignatures, + } + : undefined } private async loadStartupMetadataInternal( lifecycleGeneration: number, + adapter: HydrationPersistenceAdapter, ): Promise { - const snapshot = await this.persistence.adapter.loadResumeSnapshot( - this.collectionId, - { includeRows: false }, - ) + const snapshot = await adapter.loadResumeSnapshot(this.collectionId, { + includeRows: false, + }) if (lifecycleGeneration !== this.lifecycleGeneration) return this.persistedResumeGeneration = this.getResumeSnapshotGeneration(snapshot) this.persistedKeySetEvidence = snapshot.keySet @@ -1481,14 +1705,14 @@ class PersistedCollectionRuntime< if (applied !== true) await applied } - private async loadCollectionMetadataSnapshot(): Promise< - Array<{ key: string; value: unknown }> - > { - if (!this.persistence.adapter.loadCollectionMetadata) { + private async loadCollectionMetadataSnapshot( + adapter: HydrationPersistenceAdapter, + ): Promise> { + if (!adapter.loadCollectionMetadata) { return [] } - return this.persistence.adapter.loadCollectionMetadata(this.collectionId) + return adapter.loadCollectionMetadata(this.collectionId) } private replaceCollectionMetadataSnapshot( @@ -1535,17 +1759,33 @@ class PersistedCollectionRuntime< const routeRemoteDemandDuringHydration = this.canRouteRemoteDemandThroughCoordinator() this.activeSubsets.set(this.getSubsetKey(options), options) - const appliedCursor = this.appliedReceiptSequence try { - await this.applyMutex.run(() => - this.hydrateSubsetUnsafe(options, { - requestRemoteEnsure: - this.mode === `sync-present` && !routeRemoteDemandDuringHydration, - lifecycleGeneration, - requestLocalLoadFailure: true, - }), - ) + await this.applyMutex.run(async () => { + try { + await this.runInHydrationScope((adapter) => + this.hydrateSubsetUnsafe( + options, + { + requestRemoteEnsure: + this.mode === `sync-present` && + !routeRemoteDemandDuringHydration, + lifecycleGeneration, + requestLocalLoadFailure: true, + rejectBufferedReplayFailure: true, + }, + adapter, + ), + ) + } finally { + if ( + lifecycleGeneration === this.lifecycleGeneration && + !this.getCurrentTerminalFailure() + ) { + await this.flushQueuedTxCommittedUnsafe() + } + } + }) if (lifecycleGeneration !== this.lifecycleGeneration) return await this.waitForAppliedReceiptsAfter(appliedCursor) } catch (error) { @@ -1642,12 +1882,22 @@ class PersistedCollectionRuntime< this.throwIfTerminal() const lifecycleGeneration = this.lifecycleGeneration // A one-shot refresh does not acquire an enduring subscription lease. - await this.applyMutex.run(() => - this.hydrateSubsetUnsafe(options, { - requestRemoteEnsure: false, - lifecycleGeneration, - }), - ) + await this.applyMutex.run(async () => { + await this.runInHydrationScope((adapter) => + this.hydrateSubsetUnsafe( + options, + { + requestRemoteEnsure: false, + lifecycleGeneration, + rejectBufferedReplayFailure: true, + }, + adapter, + ), + ) + if (lifecycleGeneration === this.lifecycleGeneration) { + await this.flushQueuedTxCommittedUnsafe() + } + }) } queueHydrationBufferedTransaction( @@ -1677,7 +1927,10 @@ class PersistedCollectionRuntime< ) } return this.applyMutex.run(async () => { - await this.applyBufferedSyncTransactionUnsafe(transaction) + await this.applyBufferedSyncTransactionUnsafe( + transaction, + this.persistence.adapter, + ) }) } @@ -1834,7 +2087,6 @@ class PersistedCollectionRuntime< private advanceLifecycle(): void { this.lifecycleGeneration++ - this.started = false this.startupMetadataPromise = null this.startPromise = null this.resumeBaselinePromise = null @@ -1866,8 +2118,9 @@ class PersistedCollectionRuntime< private loadSubsetRowsUnsafe( options: LoadSubsetOptions, + adapter: HydrationPersistenceAdapter, ): Promise> { - return this.persistence.adapter.loadSubset(this.collectionId, options, { + return adapter.loadSubset(this.collectionId, options, { requiredIndexSignatures: this.getRequiredIndexSignatures(), }) as Promise> } @@ -1899,9 +2152,13 @@ class PersistedCollectionRuntime< lifecycleGeneration: number bindKeySetEvidence?: boolean requestLocalLoadFailure?: boolean + rejectBufferedReplayFailure?: boolean }, + adapter: HydrationPersistenceAdapter, ): Promise { let rowsLoaded = false + let replayFailure: { reason: unknown } | undefined + const resetSequence = this.resetSequence this.hydrationSequence++ const hydrationContext = { suppliedRowKeys: new Set() } this.activeHydrationContext = hydrationContext @@ -1911,25 +2168,24 @@ class PersistedCollectionRuntime< try { let rows: Array<{ key: TKey; value: T; metadata?: unknown }> if (config.bindKeySetEvidence) { - const snapshot = await this.persistence.adapter.loadResumeSnapshot( - this.collectionId, - { - requiredIndexSignatures: this.getRequiredIndexSignatures(), - includeRows: true, - }, - ) + const snapshot = await adapter.loadResumeSnapshot(this.collectionId, { + requiredIndexSignatures: this.getRequiredIndexSignatures(), + includeRows: true, + }) rows = snapshot.rows as Array<{ key: TKey value: T metadata?: unknown }> if (config.lifecycleGeneration !== this.lifecycleGeneration) return + if (resetSequence !== this.resetSequence) return this.bindResumeSnapshotEvidence(snapshot) } else { - rows = await this.loadSubsetRowsUnsafe(options) + rows = await this.loadSubsetRowsUnsafe(options, adapter) } rowsLoaded = true if (config.lifecycleGeneration !== this.lifecycleGeneration) return + if (resetSequence !== this.resetSequence) return if ( !config.bindKeySetEvidence || @@ -1942,15 +2198,19 @@ class PersistedCollectionRuntime< if (applied !== true) await applied } } finally { - if (this.hydratingGeneration === config.lifecycleGeneration) { + if ( + this.activeHydrationContext === hydrationContext && + this.hydratingGeneration === config.lifecycleGeneration + ) { this.hydratingGeneration = null } } - await this.flushQueuedHydrationTransactionsUnsafe() - await this.flushQueuedTxCommittedUnsafe() + if (config.lifecycleGeneration !== this.lifecycleGeneration) return + replayFailure = await this.flushQueuedHydrationTransactionsUnsafe(adapter) + if (config.lifecycleGeneration !== this.lifecycleGeneration) return - if (config.requestRemoteEnsure) { + if (config.requestRemoteEnsure && !replayFailure) { this.queueRemoteSubsetEnsure(options) } } catch (error) { @@ -1964,6 +2224,7 @@ class PersistedCollectionRuntime< try { await this.recoverBufferedTransactionsAfterLocalLoadFailureUnsafe( config.lifecycleGeneration, + adapter, ) } catch (baselineError) { const terminalError = this.markTerminalFailure( @@ -1976,7 +2237,6 @@ class PersistedCollectionRuntime< this.hydratingGeneration = null } } - await this.flushQueuedTxCommittedUnsafe() throw error } throw this.markTerminalFailure(error, config.lifecycleGeneration) @@ -1985,10 +2245,14 @@ class PersistedCollectionRuntime< this.activeHydrationContext = undefined } } + if (replayFailure && config.rejectBufferedReplayFailure) { + throw replayFailure.reason + } } private async recoverBufferedTransactionsAfterLocalLoadFailureUnsafe( lifecycleGeneration: number, + adapter: HydrationPersistenceAdapter, ): Promise { let snapshotRows: | Map @@ -2040,15 +2304,13 @@ class PersistedCollectionRuntime< getPresence(key) === `unknown` ) { if (snapshotRows === undefined) { - const snapshot = - await this.persistence.adapter.loadResumeSnapshot( - this.collectionId, - { - requiredIndexSignatures: - this.getRequiredIndexSignatures(), - includeRows: true, - }, - ) + const snapshot = await adapter.loadResumeSnapshot( + this.collectionId, + { + requiredIndexSignatures: this.getRequiredIndexSignatures(), + includeRows: true, + }, + ) this.throwIfLifecycleReplaced(lifecycleGeneration) if (transaction.signal?.aborted) break snapshotRows = new Map( @@ -2086,9 +2348,9 @@ class PersistedCollectionRuntime< } } - const transactionApplied = - await this.applyBufferedSyncTransactionUnsafe(transaction) - if (transactionApplied) { + const transactionOutcome = + await this.applyBufferedSyncTransactionUnsafe(transaction, adapter) + if (transactionOutcome.applied) { if (transaction.truncate) { snapshotInvalidatedByTruncate = true recoveredPresence.clear() @@ -2253,14 +2515,21 @@ class PersistedCollectionRuntime< }) } - private async flushQueuedHydrationTransactionsUnsafe(): Promise { + private async flushQueuedHydrationTransactionsUnsafe( + adapter: HydrationPersistenceAdapter, + ): Promise<{ reason: unknown } | undefined> { + let operationFailure: { reason: unknown } | undefined while (this.queuedHydrationTransactions.length > 0) { const transaction = this.queuedHydrationTransactions.shift() if (!transaction) { continue } try { - await this.applyBufferedSyncTransactionUnsafe(transaction) + const outcome = await this.applyBufferedSyncTransactionUnsafe( + transaction, + adapter, + ) + operationFailure ??= outcome.hydrationFailure } catch (error) { transaction.rejectApplied?.(error) for (const abandoned of this.queuedHydrationTransactions) { @@ -2270,18 +2539,28 @@ class PersistedCollectionRuntime< throw error } } + return operationFailure } private async applyBufferedSyncTransactionUnsafe( transaction: BufferedSyncTransaction, - ): Promise { + adapter: HydrationPersistenceAdapter, + ): Promise<{ + applied: boolean + hydrationFailure?: { reason: unknown } + }> { this.throwIfTerminal() this.throwIfLifecycleReplaced(transaction.lifecycleGeneration) + const abortedBeforeApplication = transaction.signal?.aborted === true + let abortedDuringApplication = false + let applicationReturned = false try { const applied = transaction.internal ? this.withInternalApply(transaction.applyToCollection) : transaction.applyToCollection() + applicationReturned = true + abortedDuringApplication = transaction.signal?.aborted === true if (applied !== true) { this.sourcePublicationWaitDepth++ try { @@ -2293,11 +2572,17 @@ class PersistedCollectionRuntime< this.throwIfLifecycleReplaced(transaction.lifecycleGeneration) if (!transaction.internal) { - await this.persistAndBroadcastExternalSyncTransactionUnsafe(transaction) + await this.persistAndBroadcastExternalSyncTransactionUnsafe( + transaction, + adapter, + ) } transaction.resolveApplied?.() - return true + return { applied: true } } catch (error) { + if (!applicationReturned) { + abortedDuringApplication = transaction.signal?.aborted === true + } const aborted = transaction.signal?.aborted || error instanceof SyncTransactionAbortedError @@ -2309,7 +2594,13 @@ class PersistedCollectionRuntime< ) transaction.rejectApplied?.(terminalError) if (aborted && !shouldFailStop && transaction.rejectApplied) { - return false + return { + applied: false, + hydrationFailure: + !abortedBeforeApplication && abortedDuringApplication + ? { reason: terminalError } + : undefined, + } } throw terminalError } @@ -2333,6 +2624,7 @@ class PersistedCollectionRuntime< private async persistAndBroadcastExternalSyncTransactionUnsafe( transaction: BufferedSyncTransaction, + adapter: HydrationPersistenceAdapter = this.persistence.adapter, ): Promise { if (transaction.internal) { return @@ -2353,6 +2645,7 @@ class PersistedCollectionRuntime< const response = await this.persistence.coordinator.requestApplyCommittedTx( this.collectionId, tx, + adapter, ) if (!response.ok) { if (response.code === `PERSISTENCE_ERROR`) { @@ -2856,6 +3149,10 @@ class PersistedCollectionRuntime< return } + this.persistence.coordinator.setAdapterForCollection?.( + this.collectionId, + this.persistence.adapter, + ) this.coordinatorUnsubscribe = this.persistence.coordinator.subscribe( this.collectionId, (message) => { @@ -2903,7 +3200,14 @@ class PersistedCollectionRuntime< if (isCollectionResetPayload(payload)) { void this.applyMutex - .run(() => this.truncateAndReloadUnsafe(lifecycleGeneration)) + .run(async () => { + await this.runInHydrationScope((adapter) => + this.truncateAndReloadUnsafe(adapter, lifecycleGeneration), + ) + if (lifecycleGeneration === this.lifecycleGeneration) { + await this.flushQueuedTxCommittedUnsafe() + } + }) .catch((error) => { this.markTerminalFailure(error, lifecycleGeneration) }) @@ -2964,7 +3268,13 @@ class PersistedCollectionRuntime< txCommitted.latestRowVersion, ) - await this.invalidateFromCommittedTxUnsafe(txCommitted) + await this.invalidateFromCommittedTxUnsafe( + txCommitted, + this.persistence.adapter, + ) + if (lifecycleGeneration === this.lifecycleGeneration) { + await this.flushQueuedTxCommittedUnsafe() + } } private async recoverFromSeqGapUnsafe( @@ -2991,34 +3301,51 @@ class PersistedCollectionRuntime< pullResponse.latestSeq, pullResponse.latestRowVersion, ) - if (pullResponse.requiresFullReload || !pullResponse.deltas) { - await this.reloadActiveSubsetsUnsafe() + if (pullResponse.requiresFullReload) { + await this.runInHydrationScope((adapter) => + this.reloadActiveSubsetsUnsafe(adapter), + ) return } - - for (const delta of pullResponse.deltas) { - if (lifecycleGeneration !== this.lifecycleGeneration) return - await this.invalidateFromCommittedTxUnsafe({ - type: `tx:committed`, - term: pullResponse.latestTerm, - seq: pullResponse.latestSeq, - txId: delta.txId, - latestRowVersion: delta.latestRowVersion, - requiresFullReload: false, - changedRows: delta.changedRows, - deletedKeys: delta.deletedKeys, - rowMetadataMutations: delta.rowMetadataMutations, - collectionMetadataMutations: delta.collectionMetadataMutations, - }) - if (lifecycleGeneration !== this.lifecycleGeneration) return + const deltas = pullResponse.deltas + if (!deltas) { + await this.runInHydrationScope((adapter) => + this.reloadActiveSubsetsUnsafe(adapter), + ) + return } + + await this.runInHydrationScope(async (adapter) => { + for (const delta of deltas) { + if (lifecycleGeneration !== this.lifecycleGeneration) return + await this.invalidateFromCommittedTxUnsafe( + { + type: `tx:committed`, + term: pullResponse.latestTerm, + seq: pullResponse.latestSeq, + txId: delta.txId, + latestRowVersion: delta.latestRowVersion, + requiresFullReload: false, + changedRows: delta.changedRows, + deletedKeys: delta.deletedKeys, + rowMetadataMutations: delta.rowMetadataMutations, + collectionMetadataMutations: + delta.collectionMetadataMutations, + }, + adapter, + ) + if (lifecycleGeneration !== this.lifecycleGeneration) return + } + }) return } } } if (lifecycleGeneration !== this.lifecycleGeneration) return - await this.truncateAndReloadUnsafe(lifecycleGeneration) + await this.runInHydrationScope((adapter) => + this.truncateAndReloadUnsafe(adapter, lifecycleGeneration), + ) if (this.mode === `sync-present`) { for (const options of this.activeSubsets.values()) { @@ -3028,9 +3355,11 @@ class PersistedCollectionRuntime< } private async truncateAndReloadUnsafe( + adapter: HydrationPersistenceAdapter, lifecycleGeneration = this.lifecycleGeneration, ): Promise { if (lifecycleGeneration !== this.lifecycleGeneration) return + this.resetSequence++ if (this.syncControls.begin && this.syncControls.commit) { const applied = this.withInternalApply(() => { this.syncControls.begin?.({ immediate: true }) @@ -3041,21 +3370,28 @@ class PersistedCollectionRuntime< } if (lifecycleGeneration !== this.lifecycleGeneration) return - await this.reloadActiveSubsetsUnsafe() + await this.reloadActiveSubsetsUnsafe(adapter) } private async invalidateFromCommittedTxUnsafe( txCommitted: TxCommitted, + adapter: HydrationPersistenceAdapter, ): Promise { + const reloadActiveSubsets = () => + this.runInHydrationScope( + (scopedAdapter) => this.reloadActiveSubsetsUnsafe(scopedAdapter), + adapter, + ) + if (txCommitted.requiresFullReload) { - await this.reloadActiveSubsetsUnsafe() + await reloadActiveSubsets() return } const changedKeyCount = txCommitted.changedRows.length + txCommitted.deletedKeys.length if (changedKeyCount > TARGETED_INVALIDATION_KEY_LIMIT) { - await this.reloadActiveSubsetsUnsafe() + await reloadActiveSubsets() return } @@ -3070,7 +3406,7 @@ class PersistedCollectionRuntime< // Has paginated subsets — fall back to full reload. // Targeted invalidation for paginated subsets is deferred to a future iteration. - await this.reloadActiveSubsetsUnsafe() + await reloadActiveSubsets() } private async applyTargetedInvalidationUnsafe( @@ -3130,7 +3466,9 @@ class PersistedCollectionRuntime< if (applied !== true) await applied } - private async reloadActiveSubsetsUnsafe(): Promise { + private async reloadActiveSubsetsUnsafe( + adapter: HydrationPersistenceAdapter, + ): Promise { const lifecycleGeneration = this.lifecycleGeneration const activeSubsetOptions = this.activeSubsets.size > 0 @@ -3143,10 +3481,11 @@ class PersistedCollectionRuntime< this.hydratingGeneration = lifecycleGeneration try { const mergedRows = new Map() - const collectionMetadata = await this.loadCollectionMetadataSnapshot() + const collectionMetadata = + await this.loadCollectionMetadataSnapshot(adapter) if (lifecycleGeneration !== this.lifecycleGeneration) return for (const options of activeSubsetOptions) { - const subsetRows = await this.loadSubsetRowsUnsafe(options) + const subsetRows = await this.loadSubsetRowsUnsafe(options, adapter) if (lifecycleGeneration !== this.lifecycleGeneration) return for (const row of subsetRows) { mergedRows.set(row.key, { @@ -3170,7 +3509,10 @@ class PersistedCollectionRuntime< ) if (applied !== true) await applied } finally { - if (this.hydratingGeneration === lifecycleGeneration) { + if ( + this.activeHydrationContext === hydrationContext && + this.hydratingGeneration === lifecycleGeneration + ) { this.hydratingGeneration = null } if (this.activeHydrationContext === hydrationContext) { @@ -3178,8 +3520,9 @@ class PersistedCollectionRuntime< } } - await this.flushQueuedHydrationTransactionsUnsafe() - await this.flushQueuedTxCommittedUnsafe() + if (lifecycleGeneration === this.lifecycleGeneration) { + await this.flushQueuedHydrationTransactionsUnsafe(adapter) + } } private attachIndexLifecycleListeners(): void { @@ -3204,16 +3547,33 @@ class PersistedCollectionRuntime< private async bootstrapPersistedIndexes( indexMetadataSnapshot?: Array, - ): Promise { + adapter: HydrationPersistenceAdapter = this.persistence.adapter, + ): Promise> { const collection = this.collection if (!collection && !indexMetadataSnapshot) { - return + return new Set() } const indexMetadata = indexMetadataSnapshot ?? collection?.getIndexMetadata() ?? [] + const completedLocalIndexSignatures = new Set() for (const metadata of indexMetadata) { - await this.ensurePersistedIndex(metadata) + if (await this.ensureLocalPersistedIndex(metadata, adapter)) { + completedLocalIndexSignatures.add(metadata.signature) + } + } + return completedLocalIndexSignatures + } + + private async requestCoordinatorPersistedIndexes( + indexMetadata: Array, + completedLocalIndexSignatures: ReadonlySet, + ): Promise { + for (const metadata of indexMetadata) { + await this.requestCoordinatorPersistedIndex( + metadata, + completedLocalIndexSignatures.has(metadata.signature), + ) } } @@ -3232,24 +3592,47 @@ class PersistedCollectionRuntime< private async ensurePersistedIndex( indexMetadata: CollectionIndexMetadata, + adapter: HydrationPersistenceAdapter = this.persistence.adapter, ): Promise { + const completedLocally = await this.ensureLocalPersistedIndex( + indexMetadata, + adapter, + ) + await this.requestCoordinatorPersistedIndex(indexMetadata, completedLocally) + } + + private async ensureLocalPersistedIndex( + indexMetadata: CollectionIndexMetadata, + adapter: HydrationPersistenceAdapter, + ): Promise { const spec = this.buildPersistedIndexSpec(indexMetadata) try { - await this.persistence.adapter.ensureIndex( + await adapter.ensureIndex( this.collectionId, indexMetadata.signature, spec, ) + return true } catch (error) { console.warn(`Failed to ensure persisted index in adapter:`, error) + return false } + } + + private async requestCoordinatorPersistedIndex( + indexMetadata: CollectionIndexMetadata, + completedLocally: boolean, + ): Promise { + const spec = this.buildPersistedIndexSpec(indexMetadata) try { await this.persistence.coordinator.requestEnsurePersistedIndex( this.collectionId, indexMetadata.signature, spec, + completedLocally ? this.persistence.adapter : undefined, + completedLocally, ) } catch (error) { console.warn( @@ -3475,7 +3858,14 @@ function createWrappedSyncConfig< } transaction.deferredHydrationMetadataDeleteKeys.clear() - params.begin(transaction.beginOptions) + // A buffered source replay is part of the hydrate that owns it. + // It must not wait for a mutation whose persistence is queued + // behind that hydrate's apply mutex. + params.begin( + transaction.queuedBecauseHydrating + ? { immediate: true } + : transaction.beginOptions, + ) if (transaction.truncate) params.truncate() for (const operation of transaction.operations) { if (operation.type === `delete`) { diff --git a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts index 1d68098752..bcd92a98de 100644 --- a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts +++ b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts @@ -8,6 +8,7 @@ import { InvalidPersistedStorageKeyEncodingError, } from './errors' import { + SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY, createPersistedTableName, decodePersistedStorageKey, encodePersistedStorageKey, @@ -20,6 +21,7 @@ import { } from './sqlite-value' import type { LoadSubsetOptions } from '@tanstack/db' import type { + HydrationPersistenceAdapter, PersistedIndexSpec, PersistedKeySetEvidence, PersistedRowScanOptions, @@ -80,6 +82,159 @@ export type SQLitePullSinceResult = deltas: Array, TKey>> } +type ScheduledOperationKind = `regular` | `hydrate` + +type ScheduledOperation = { + kind: ScheduledOperationKind + task: () => Promise + resolve: (value: T) => void + reject: (error: unknown) => void +} + +class SharedPersistenceScheduler { + private readonly regularQueue: Array> = [] + private readonly hydrateQueue: Array> = [] + private running = false + private lastCompletedKind: ScheduledOperationKind | undefined + + runRegular(task: () => Promise): Promise { + return this.enqueue(`regular`, task) + } + + runHydrate(task: () => Promise): Promise { + return this.enqueue(`hydrate`, task) + } + + adoptRunningHydrate(completion: Promise): void { + if (this.running) return + this.running = true + const finish = () => { + this.lastCompletedKind = `hydrate` + this.running = false + this.drain() + } + void completion.then(finish, finish) + } + + private enqueue( + kind: ScheduledOperationKind, + task: () => Promise, + ): Promise { + const result = new Promise((resolve, reject) => { + const operation: ScheduledOperation = { + kind, + task, + resolve, + reject, + } + const queue = kind === `hydrate` ? this.hydrateQueue : this.regularQueue + queue.push(operation as ScheduledOperation) + }) + this.drain() + return result + } + + private drain(): void { + if (this.running) return + + const operation = this.takeNext() + if (!operation) return + + this.running = true + void this.execute(operation) + } + + private async execute(operation: ScheduledOperation): Promise { + try { + operation.resolve(await operation.task()) + } catch (error) { + operation.reject(error) + } finally { + this.lastCompletedKind = operation.kind + this.running = false + this.drain() + } + } + + private takeNext(): ScheduledOperation | undefined { + // Hydrates get priority after the currently running non-preemptible unit. + // While both lanes remain queued, alternate one regular operation after + // each hydrate (K=1), preserving FIFO order within each lane. + if (this.hydrateQueue.length > 0) { + if ( + this.regularQueue.length > 0 && + this.lastCompletedKind === `hydrate` + ) { + return this.regularQueue.shift() + } + return this.hydrateQueue.shift() + } + return this.regularQueue.shift() + } +} + +const sharedPersistenceSchedulers = new WeakMap< + object, + SharedPersistenceScheduler +>() +const observedDriverSchedulingKeys = new WeakMap() + +function getSharedPersistenceScheduler( + key: object, +): SharedPersistenceScheduler { + let scheduler = sharedPersistenceSchedulers.get(key) + if (!scheduler) { + scheduler = new SharedPersistenceScheduler() + sharedPersistenceSchedulers.set(key, scheduler) + } + return scheduler +} + +function getSharedLogicalSchedulingKey(value: unknown): object | undefined { + if ((typeof value !== `object` && typeof value !== `function`) || !value) { + return undefined + } + const key = ( + value as { + [SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY]?: unknown + } + )[SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY] + return key !== null && (typeof key === `object` || typeof key === `function`) + ? key + : undefined +} + +function observeSharedLogicalSchedulingSupport( + driver: SQLiteDriver, + onSupport: (key: object) => void, +): SQLiteDriver { + let observationPending = true + const observe = (promise: Promise): Promise => { + if (!observationPending) return promise + observationPending = false + const key = getSharedLogicalSchedulingKey(promise) + if (key) onSupport(key) + return promise + } + + return { + exec: (sql) => observe(driver.exec(sql)), + query: (sql: string, params: ReadonlyArray = []) => + observe(driver.query(sql, params)), + run: (sql, params = []) => observe(driver.run(sql, params)), + transaction: (fn: (transactionDriver: SQLiteDriver) => Promise) => + observe(driver.transaction(fn)), + transactionWithDriver: ( + fn: (transactionDriver: SQLiteDriver) => Promise, + ) => + observe( + driver.transactionWithDriver + ? driver.transactionWithDriver(fn) + : driver.transaction(fn), + ), + } +} + const DEFAULT_SCHEMA_VERSION = 1 const DEFAULT_PULL_SINCE_RELOAD_THRESHOLD = 128 @@ -1027,6 +1182,10 @@ function buildIndexName(collectionId: string, signature: string): string { export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { private readonly driver: SQLiteDriver + private readonly schedulingIdentitySource: SQLiteDriver + private scheduler: SharedPersistenceScheduler | undefined + private activeUnscheduledHydration: Promise | undefined + private readonly hydrationAdapter: HydrationPersistenceAdapter private readonly schemaVersion: number private readonly schemaMismatchPolicy: SQLiteCoreAdapterSchemaMismatchPolicy private readonly appliedTxPruneMaxRows: number | undefined @@ -1082,13 +1241,87 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { ) } - this.driver = options.driver + this.schedulingIdentitySource = options.driver + const schedulingKey = + getSharedLogicalSchedulingKey(options.driver) ?? + observedDriverSchedulingKeys.get(options.driver) + this.scheduler = schedulingKey + ? getSharedPersistenceScheduler(schedulingKey) + : undefined + this.driver = schedulingKey + ? options.driver + : observeSharedLogicalSchedulingSupport(options.driver, (key) => { + observedDriverSchedulingKeys.set(options.driver, key) + const scheduler = getSharedPersistenceScheduler(key) + this.scheduler ??= scheduler + if (this.activeUnscheduledHydration) { + scheduler.adoptRunningHydrate(this.activeUnscheduledHydration) + } + }) this.schemaVersion = schemaVersion this.schemaMismatchPolicy = options.schemaMismatchPolicy ?? `sync-present-reset` this.appliedTxPruneMaxRows = options.appliedTxPruneMaxRows this.appliedTxPruneMaxAgeSeconds = options.appliedTxPruneMaxAgeSeconds this.pullSinceReloadThreshold = pullSinceReloadThreshold + this.hydrationAdapter = { + loadSubset: (collectionId, loadOptions, context) => + this.loadSubsetUnscheduled(collectionId, loadOptions, context), + loadResumeSnapshot: (collectionId, context) => + this.loadResumeSnapshotUnscheduled(collectionId, context), + applyCommittedTx: (collectionId, tx) => + this.applyCommittedTxUnscheduled(collectionId, tx), + loadCollectionMetadata: (collectionId) => + this.loadCollectionMetadataUnscheduled(collectionId), + scanRows: (collectionId, scanOptions) => + this.scanRowsUnscheduled(collectionId, scanOptions), + ensureIndex: (collectionId, signature, spec) => + this.ensureIndexUnscheduled(collectionId, signature, spec), + markIndexRemoved: (collectionId, signature) => + this.markIndexRemovedUnscheduled(collectionId, signature), + getStreamPosition: (collectionId) => + this.getStreamPositionUnscheduled(collectionId), + pullSince: (collectionId, fromRowVersion) => + this.pullSinceUnscheduled(collectionId, fromRowVersion), + runInHydrationScope: async (task) => task(this.hydrationAdapter), + } + } + + runInHydrationScope( + task: (adapter: HydrationPersistenceAdapter) => Promise, + ): Promise { + const scheduler = this.resolveScheduler() + if (scheduler) { + return scheduler.runHydrate(() => task(this.hydrationAdapter)) + } + + const hydration = Promise.resolve().then(() => task(this.hydrationAdapter)) + this.activeUnscheduledHydration = hydration + const clear = () => { + if (this.activeUnscheduledHydration === hydration) { + this.activeUnscheduledHydration = undefined + } + } + void hydration.then(clear, clear) + return hydration + } + + isHydrationScopeScheduled(): boolean { + return this.resolveScheduler() !== undefined + } + + private runRegular(task: () => Promise): Promise { + const scheduler = this.resolveScheduler() + return scheduler ? scheduler.runRegular(task) : task() + } + + private resolveScheduler(): SharedPersistenceScheduler | undefined { + if (this.scheduler) return this.scheduler + const key = observedDriverSchedulingKeys.get(this.schedulingIdentitySource) + if (key) { + this.scheduler = getSharedPersistenceScheduler(key) + } + return this.scheduler } private runInTransaction( @@ -1123,7 +1356,23 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { } } - async loadSubset( + loadSubset( + collectionId: string, + options: LoadSubsetOptions, + ctx?: { requiredIndexSignatures?: ReadonlyArray }, + ): Promise< + Array<{ + key: string | number + value: Record + metadata?: unknown + }> + > { + return this.runRegular(() => + this.loadSubsetUnscheduled(collectionId, options, ctx), + ) + } + + private async loadSubsetUnscheduled( collectionId: string, options: LoadSubsetOptions, ctx?: { requiredIndexSignatures?: ReadonlyArray }, @@ -1208,7 +1457,19 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { }) } - async loadResumeSnapshot( + loadResumeSnapshot( + collectionId: string, + ctx?: { + requiredIndexSignatures?: ReadonlyArray + includeRows?: boolean + }, + ) { + return this.runRegular(() => + this.loadResumeSnapshotUnscheduled(collectionId, ctx), + ) + } + + private async loadResumeSnapshotUnscheduled( collectionId: string, ctx?: { requiredIndexSignatures?: ReadonlyArray @@ -1291,7 +1552,16 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { }) } - async applyCommittedTx(collectionId: string, tx: PersistedTx): Promise { + applyCommittedTx(collectionId: string, tx: PersistedTx): Promise { + return this.runRegular(() => + this.applyCommittedTxUnscheduled(collectionId, tx), + ) + } + + private async applyCommittedTxUnscheduled( + collectionId: string, + tx: PersistedTx, + ): Promise { const tableMapping = await this.ensureCollectionReady(collectionId) const collectionTableSql = quoteIdentifier(tableMapping.tableName) const tombstoneTableSql = quoteIdentifier(tableMapping.tombstoneTableName) @@ -1569,7 +1839,15 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { }) } - async loadCollectionMetadata( + loadCollectionMetadata( + collectionId: string, + ): Promise> { + return this.runRegular(() => + this.loadCollectionMetadataUnscheduled(collectionId), + ) + } + + private async loadCollectionMetadataUnscheduled( collectionId: string, ): Promise> { await this.ensureCollectionReady(collectionId) @@ -1596,7 +1874,16 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { }) } - async scanRows( + scanRows( + collectionId: string, + options?: PersistedRowScanOptions, + ): Promise> { + return this.runRegular(() => + this.scanRowsUnscheduled(collectionId, options), + ) + } + + private async scanRowsUnscheduled( collectionId: string, options?: PersistedRowScanOptions, ): Promise> { @@ -1625,7 +1912,17 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { }) } - async ensureIndex( + ensureIndex( + collectionId: string, + signature: string, + spec: PersistedIndexSpec, + ): Promise { + return this.runRegular(() => + this.ensureIndexUnscheduled(collectionId, signature, spec), + ) + } + + private async ensureIndexUnscheduled( collectionId: string, signature: string, spec: PersistedIndexSpec, @@ -1718,7 +2015,13 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { }) } - async markIndexRemoved( + markIndexRemoved(collectionId: string, signature: string): Promise { + return this.runRegular(() => + this.markIndexRemovedUnscheduled(collectionId, signature), + ) + } + + private async markIndexRemovedUnscheduled( collectionId: string, signature: string, ): Promise { @@ -1755,7 +2058,17 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { }) } - async getStreamPosition(collectionId: string): Promise<{ + getStreamPosition(collectionId: string): Promise<{ + latestTerm: number + latestSeq: number + latestRowVersion: number + }> { + return this.runRegular(() => + this.getStreamPositionUnscheduled(collectionId), + ) + } + + private async getStreamPositionUnscheduled(collectionId: string): Promise<{ latestTerm: number latestSeq: number latestRowVersion: number @@ -1857,7 +2170,16 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { } } - async pullSince( + pullSince( + collectionId: string, + fromRowVersion: number, + ): Promise> { + return this.runRegular(() => + this.pullSinceUnscheduled(collectionId, fromRowVersion), + ) + } + + private async pullSinceUnscheduled( collectionId: string, fromRowVersion: number, ): Promise> { diff --git a/packages/db-sqlite-persistence-core/tests/persisted-real-adapter-lifecycle.test.ts b/packages/db-sqlite-persistence-core/tests/persisted-real-adapter-lifecycle.test.ts new file mode 100644 index 0000000000..8b39be10f5 --- /dev/null +++ b/packages/db-sqlite-persistence-core/tests/persisted-real-adapter-lifecycle.test.ts @@ -0,0 +1,292 @@ +/** + * # Does real-adapter restart preserve the sync-run start boundary? + * + * A stale lifecycle may keep its already-admitted hydrate pending before its + * SQLite row read, but it must not retain the wrapper mutex that gates startup + * metadata for the replacement sync run. The replacement upstream sync + * function may start once its metadata is loaded, before the stale hydrate is + * released. This file uses the real core adapter because recording adapters + * do not expose its public hydration-scope method and therefore select a + * different startup branch. + */ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { expect, it } from 'vitest' +import fc from 'fast-check' +import { createCollection } from '@tanstack/db' +import { + SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY, + createSQLiteCorePersistenceAdapter, + persistedCollectionOptions, +} from '../src' +import { SqliteCliDriver } from './sqlite-core-adapter.test' +import type { PersistenceAdapter, SQLiteDriver } from '../src' + +type Deferred = { + promise: Promise + resolve: () => void +} + +function createDeferred(): Deferred { + let resolve!: () => void + const promise = new Promise((settle) => { + resolve = settle + }) + return { promise, resolve } +} + +async function reachCheckpoint( + promise: Promise, + checkpoint: string, +): Promise { + let timer: ReturnType | undefined + try { + await Promise.race([ + promise, + new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error(`Did not reach checkpoint: ${checkpoint}`)), + 2_000, + ) + }), + ]) + } finally { + if (timer !== undefined) clearTimeout(timer) + } +} + +class QueryObservingDriver implements SQLiteDriver { + readonly queries: Array = [] + + constructor( + private readonly driver: SQLiteDriver, + schedulingKey?: object, + ) { + if (schedulingKey) { + Object.defineProperty(this, SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY, { + value: schedulingKey, + }) + } + } + + exec(sql: string): Promise { + return this.driver.exec(sql) + } + + async query( + sql: string, + params: ReadonlyArray = [], + ): Promise> { + return this.queryThroughDriver(sql, params, this.driver) + } + + private async queryThroughDriver( + sql: string, + params: ReadonlyArray, + queryDriver: SQLiteDriver, + ): Promise> { + const normalizedSql = sql.replace(/\s+/g, ` `).trim() + this.queries.push(normalizedSql) + if ( + normalizedSql.includes(`FROM leader_term`) || + normalizedSql.includes(`FROM collection_version`) || + normalizedSql.includes(`FROM applied_tx`) || + normalizedSql.includes(`FROM collection_metadata`) + ) { + return [] + } + return queryDriver.query(sql, params) + } + + run(sql: string, params: ReadonlyArray = []): Promise { + return this.driver.run(sql, params) + } + + transaction( + fn: (transactionDriver: SQLiteDriver) => Promise, + ): Promise { + return this.driver.transaction((transactionDriver) => + fn(this.observeTransactionDriver(transactionDriver)), + ) + } + + transactionWithDriver( + fn: (transactionDriver: SQLiteDriver) => Promise, + ): Promise { + return this.driver.transactionWithDriver + ? this.driver.transactionWithDriver((transactionDriver) => + fn(this.observeTransactionDriver(transactionDriver)), + ) + : this.transaction(fn) + } + + private observeTransactionDriver(driver: SQLiteDriver): SQLiteDriver { + return { + exec: (sql) => driver.exec(sql), + query: (sql, params = []) => this.queryThroughDriver(sql, params, driver), + run: (sql, params = []) => driver.run(sql, params), + transaction: (fn) => + driver.transaction((nestedDriver) => + fn(this.observeTransactionDriver(nestedDriver)), + ), + } + } +} + +function holdFirstHydrationRead(adapter: PersistenceAdapter): { + entered: Promise + release: () => void +} { + const entered = createDeferred() + const gate = createDeferred() + if (!adapter.runInHydrationScope) { + throw new Error(`The real adapter must expose a hydration scope`) + } + const runInHydrationScope = adapter.runInHydrationScope.bind(adapter) + let held = false + adapter.runInHydrationScope = (task) => + runInHydrationScope((scopedAdapter) => + task({ + ...scopedAdapter, + loadResumeSnapshot: async (...args) => { + if (!held && args[1]?.includeRows !== false) { + held = true + entered.resolve() + await gate.promise + } + return scopedAdapter.loadResumeSnapshot(...args) + }, + }), + ) + return { entered: entered.promise, release: gate.resolve } +} + +async function observeRestartOrder(options: { + id: string + scheduled: boolean + schemaVersion: number +}): Promise<{ + sourceStartsBeforeRelease: number + sourceStartsAfterRelease: number + streamPositionReadsBeforeRelease: number + collectionMetadataReadsBeforeRelease: number +}> { + const directory = mkdtempSync(join(tmpdir(), `persisted-real-lifecycle-`)) + const driver = new QueryObservingDriver( + new SqliteCliDriver(join(directory, `state.sqlite`)), + options.scheduled ? {} : undefined, + ) + const adapter = createSQLiteCorePersistenceAdapter({ + driver, + schemaVersion: options.schemaVersion, + }) + const staleRows = holdFirstHydrationRead(adapter) + let sourceStarts = 0 + const freshSourceStarted = createDeferred() + + const collection = createCollection( + persistedCollectionOptions<{ id: string }, string>({ + id: `real-adapter-restart-order-${options.id}`, + getKey: (row) => row.id, + sync: { + sync: ({ markReady }) => { + sourceStarts++ + if (sourceStarts === 2) freshSourceStarted.resolve() + markReady() + }, + }, + persistence: { adapter }, + }), + ) + const stalePreload = Promise.resolve(collection.preload()) + void stalePreload.catch(() => undefined) + let freshReady: Promise | undefined + let observation = { + sourceStartsBeforeRelease: -1, + sourceStartsAfterRelease: -1, + streamPositionReadsBeforeRelease: -1, + collectionMetadataReadsBeforeRelease: -1, + } + + try { + await staleRows.entered + for (let microtask = 0; microtask < 12 && sourceStarts === 0; microtask++) { + await Promise.resolve() + } + expect(sourceStarts).toBe(1) + await collection.cleanup() + + collection.startSyncImmediate() + freshReady = collection.stateWhenReady() + void freshReady.catch(() => undefined) + if (options.scheduled) { + for (let microtask = 0; microtask < 12; microtask++) { + await Promise.resolve() + } + } else { + await reachCheckpoint( + freshSourceStarted.promise, + `replacement source started while stale hydration is held`, + ) + } + + observation = { + sourceStartsBeforeRelease: sourceStarts, + sourceStartsAfterRelease: -1, + streamPositionReadsBeforeRelease: driver.queries.filter((sql) => + sql.includes(`SELECT latest_term`), + ).length, + collectionMetadataReadsBeforeRelease: driver.queries.filter((sql) => + sql.includes(`FROM collection_metadata`), + ).length, + } + } finally { + staleRows.release() + await stalePreload.catch(() => undefined) + await freshReady?.catch(() => undefined) + observation.sourceStartsAfterRelease = sourceStarts + await collection.cleanup() + rmSync(directory, { recursive: true, force: true }) + } + + return observation +} + +it(`starts a rebound source before a stale real-adapter row read settles`, async () => { + await expect( + observeRestartOrder({ id: `shrink`, scheduled: false, schemaVersion: 1 }), + ).resolves.toEqual({ + sourceStartsBeforeRelease: 2, + sourceStartsAfterRelease: 2, + streamPositionReadsBeforeRelease: 4, + collectionMetadataReadsBeforeRelease: 2, + }) +}) + +it(`preserves generated restart order across real-adapter scheduler capabilities`, async () => { + await fc.assert( + fc.asyncProperty( + fc.record({ + scheduled: fc.boolean(), + schemaVersion: fc.integer({ min: 0, max: 3 }), + }), + async ({ scheduled, schemaVersion }) => { + const observation = await observeRestartOrder({ + id: `generated-${scheduled}-${schemaVersion}`, + scheduled, + schemaVersion, + }) + expect(observation.sourceStartsBeforeRelease).toBe(scheduled ? 1 : 2) + expect(observation.sourceStartsAfterRelease).toBe(2) + expect(observation.streamPositionReadsBeforeRelease).toBe( + scheduled ? 2 : 4, + ) + expect(observation.collectionMetadataReadsBeforeRelease).toBe( + scheduled ? 1 : 2, + ) + }, + ), + { seed: 1868, numRuns: 6, endOnFailure: true }, + ) +}, 15_000) diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index 2e4de00d1a..9b9a2b4faa 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -31,6 +31,7 @@ import { toTransportedLoadSubsetOptions, } from '../src' import type { + CollectionReset, PersistedCollectionCoordinator, PersistedCollectionPersistence, PersistedSyncWrappedOptions, @@ -67,9 +68,13 @@ import type { * Histories cross hydration, held adapters, source FIFO ordering, * independent/dependent aborts, open-transaction failure boundaries, ambient owner * operations, applied-receipt rejection, remote subset demand, acquisition - * release, retry, coordinator replay, cleanup, and restart. Tests drive the + * release, retry, coordinator replay, queued startup reloads, unscheduled + * startup/reset overlap, cleanup, and restart. Tests drive the * real persisted wrapper, Collection, coordinator, adapter, transactions, * indexes, and local mutation path. + * A source abort before core application rejects that transaction's receipt. + * It does not invalidate the durable baseline or an independent queued source + * transaction when the failed transaction made no public or durable change. * * Refinement checkpoints compare public rows, durable state, metadata, request * data, sequence evidence, exact errors, and late-work fencing. Fixed hostile @@ -351,7 +356,7 @@ function createNoopAdapter(): PersistenceAdapter { } type CoordinatorHarness = PersistedCollectionCoordinator & { - emit: (payload: TxCommitted, senderId?: string) => void + emit: (payload: TxCommitted | CollectionReset, senderId?: string) => void pullSinceCalls: number setPullSinceResponse: (response: PullSinceResponse) => void } @@ -498,6 +503,17 @@ async function flushAsyncWork(delayMs: number = 0): Promise { await new Promise((resolve) => setTimeout(resolve, delayMs)) } +function createDeferred(): { + promise: Promise + resolve: (value: T) => void +} { + let resolve!: (value: T) => void + const promise = new Promise((settle) => { + resolve = settle + }) + return { promise, resolve } +} + function createEventGate(): { promise: Promise resolve: () => void @@ -8589,6 +8605,16 @@ describeUnlessOracleReplay(`persistedCollectionOptions`, () => { const adapter = createRecordingAdapter() const hydrationEntered = createEventGate() const hydration = createEventGate() + const prefixPersistenceEntered = createEventGate() + const releasePrefixPersistence = createEventGate() + const applyCommittedTx = adapter.applyCommittedTx.bind(adapter) + adapter.applyCommittedTx = async (...args) => { + if (args[1].mutations.some((mutation) => mutation.key === `prefix`)) { + prefixPersistenceEntered.resolve() + await releasePrefixPersistence.promise + } + return applyCommittedTx(...args) + } overrideBaselineRows(adapter, async () => { hydrationEntered.resolve() await hydration.promise @@ -8608,11 +8634,8 @@ describeUnlessOracleReplay(`persistedCollectionOptions`, () => { persistence: { adapter }, }), ) - const localPersistence = createEventGate() - const localTransaction = createTransaction({ - mutationFn: () => localPersistence.promise, - }) const aborted = new AbortController() + let prefixReceipt: Promise | undefined let abortedReceipt: Promise | undefined let independentReceipt: Promise | undefined const ready = collection.stateWhenReady() @@ -8624,10 +8647,15 @@ describeUnlessOracleReplay(`persistedCollectionOptions`, () => { hydrationEntered.promise, `independent abort hydration entered`, ) - localTransaction.mutate(() => { - collection.insert({ id: `local-gate`, title: `local pending` }) + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `prefix`, title: `durability held` }, }) - expect(localTransaction.state).toBe(`persisting`) + prefixReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + void prefixReceipt.catch(() => undefined) sourceParams.begin() sourceParams.write({ @@ -8650,21 +8678,11 @@ describeUnlessOracleReplay(`persistedCollectionOptions`, () => { void independentReceipt.catch(() => undefined) hydration.resolve() - await vi.waitFor(() => - expect( - collection._state.pendingSyncedTransactions.some( - (transaction) => transaction.committed, - ), - ).toBe(true), + await atPersistedOracleCheckpoint( + prefixPersistenceEntered.promise, + `in-flight hydration prefix entered durability`, ) aborted.abort() - await expect( - atPersistedOracleCheckpoint( - abortedReceipt, - `in-flight hydration sibling aborted`, - ), - ).rejects.toMatchObject({ name: `AbortError` }) - const independentSettlement = observeSettlement(independentReceipt) await Promise.resolve() expect({ @@ -8677,11 +8695,17 @@ describeUnlessOracleReplay(`persistedCollectionOptions`, () => { publicError: undefined, }) - localPersistence.resolve() + releasePrefixPersistence.resolve() await atPersistedOracleCheckpoint( - localTransaction.isPersisted.promise, - `independent abort local gate settled`, + prefixReceipt, + `independent abort prefix settled`, ) + await expect( + atPersistedOracleCheckpoint( + abortedReceipt, + `in-flight hydration sibling aborted`, + ), + ).rejects.toMatchObject({ name: `AbortError` }) await atPersistedOracleCheckpoint( independentReceipt, `independent hydration sibling applied`, @@ -8689,10 +8713,14 @@ describeUnlessOracleReplay(`persistedCollectionOptions`, () => { await atPersistedOracleCheckpoint(ready, `independent abort ready`) expect({ status: collection.status, + prefix: stripVirtualProps(collection.get(`prefix`)), + aborted: collection.get(`aborted`), independent: stripVirtualProps(collection.get(`independent`)), durable: adapter.rows.get(`independent`), }).toEqual({ status: `ready`, + prefix: { id: `prefix`, title: `durability held` }, + aborted: undefined, independent: { id: `independent`, title: `must survive sibling abort`, @@ -8708,10 +8736,10 @@ describeUnlessOracleReplay(`persistedCollectionOptions`, () => { } finally { hydration.resolve() aborted.abort() - localPersistence.resolve() - await localTransaction.isPersisted.promise.catch(() => undefined) + releasePrefixPersistence.resolve() await cleanupPersistedOracle( [ + () => prefixReceipt?.catch(() => undefined), () => abortedReceipt?.catch(() => undefined), () => independentReceipt?.catch(() => undefined), () => ready.catch(() => undefined), @@ -8722,6 +8750,188 @@ describeUnlessOracleReplay(`persistedCollectionOptions`, () => { } }) + it.each([`startup`, `resume`, `reset`] as const)( + `keeps the %s hydration ready after a buffered pre-application abort`, + async (entry) => { + const adapter = createRecordingAdapter() + const hydrationEntered = createEventGate() + const releaseHydration = createEventGate() + const coordinator = + entry === `reset` ? createCoordinatorHarness() : undefined + if (coordinator) { + coordinator.requestApplyCommittedTx = async (collectionId, tx) => { + await adapter.applyCommittedTx(collectionId, tx) + return { + type: `rpc:applyCommittedTx:res`, + rpcId: tx.txId, + ok: true, + term: tx.term, + seq: tx.seq, + latestRowVersion: tx.rowVersion, + } + } + } else { + overrideBaselineRows(adapter, async () => { + hydrationEntered.resolve() + await releaseHydration.promise + return [] + }) + } + let sourceParams!: TodoSyncParams + let hydrateResumeBaseline: (() => Promise) | undefined + const collection = createCollection( + persistedCollectionOptions({ + id: coordinator ? `sync-present` : `buffered-abort-${entry}`, + getKey: (row) => row.id, + syncMode: entry === `resume` ? `on-demand` : `eager`, + sync: { + sync: (params) => { + sourceParams = params + hydrateResumeBaseline = + params.metadata?.persistence?.hydrateBaseline + params.markReady() + return { loadSubset: () => true } + }, + }, + persistence: { adapter, coordinator }, + }), + ) + const aborted = new AbortController() + let keyReads = 0 + let hydration: Promise | undefined + let abortedReceipt: Promise | undefined + let independentReceipt: Promise | undefined + let hasPrimaryFailure = false + + try { + if (entry === `resume`) { + collection.startSyncImmediate() + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `resume collection ready before baseline`, + ) + expect(hydrateResumeBaseline).toBeTypeOf(`function`) + hydration = hydrateResumeBaseline!() + } else if (coordinator) { + await atPersistedOracleCheckpoint( + collection.stateWhenReady(), + `reset collection ready before reload`, + ) + const loadSubset = adapter.loadSubset.bind(adapter) + adapter.loadSubset = async (...args) => { + hydrationEntered.resolve() + await releaseHydration.promise + return loadSubset(...args) + } + coordinator.emit({ + type: `collection:reset`, + schemaVersion: 1, + resetEpoch: 1, + }) + hydration = Promise.resolve() + } else { + hydration = collection.stateWhenReady() + } + void hydration.catch(() => undefined) + await atPersistedOracleCheckpoint( + hydrationEntered.promise, + `${entry} hydration entered`, + ) + + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `prefix`, title: `staged before abort` }, + }) + sourceParams.write({ + type: `insert`, + value: { + get id() { + keyReads++ + if (keyReads === 2) aborted.abort() + return `aborting` + }, + title: `cancel before core application`, + }, + }) + abortedReceipt = Promise.resolve( + sourceParams.commit(aborted.signal), + ).then(() => undefined) + void abortedReceipt.catch(() => undefined) + + sourceParams.begin() + sourceParams.write({ + type: `insert`, + value: { id: `independent`, title: `survives abort` }, + }) + independentReceipt = Promise.resolve(sourceParams.commit()).then( + () => undefined, + ) + void independentReceipt.catch(() => undefined) + + releaseHydration.resolve() + const outcomes = await atPersistedOracleCheckpoint( + Promise.allSettled([hydration, abortedReceipt, independentReceipt]), + `${entry} buffered abort settled`, + ) + const expected = foldDurabilityLedger([ + { type: `begin`, transactionId: `aborted` }, + { + type: `write`, + transactionId: `aborted`, + row: { id: `prefix`, title: `staged before abort` }, + }, + { type: `abort`, transactionId: `aborted` }, + { type: `begin`, transactionId: `independent` }, + { + type: `write`, + transactionId: `independent`, + row: { id: `independent`, title: `survives abort` }, + }, + { type: `commit`, transactionId: `independent` }, + ]) + + expect(keyReads).toBeGreaterThanOrEqual(2) + expect(outcomes.map((outcome) => outcome.status)).toEqual([ + `fulfilled`, + `rejected`, + `fulfilled`, + ]) + if (outcomes[1].status === `rejected`) { + expect(outcomes[1].reason).toBeInstanceOf(SyncTransactionAbortedError) + } + expect(collection.status).toBe(`ready`) + expect(collection._lifecycle.getSyncError()).toBeUndefined() + expect(collection.get(`prefix`)).toBeUndefined() + expect(collection.get(`aborting`)).toBeUndefined() + expect(stripVirtualProps(collection.get(`independent`))).toEqual( + expected.committedRows.get(`independent`), + ) + expect(adapter.rows).toEqual(expected.committedRows) + expect( + adapter.applyCommittedTxCalls.map(({ tx }) => + tx.mutations.map(({ key }) => key), + ), + ).toEqual([[`independent`]]) + } catch (error) { + hasPrimaryFailure = true + throw error + } finally { + releaseHydration.resolve() + aborted.abort() + await cleanupPersistedOracle( + [ + () => hydration?.catch(() => undefined), + () => abortedReceipt?.catch(() => undefined), + () => independentReceipt?.catch(() => undefined), + () => collection.cleanup(), + ], + hasPrimaryFailure, + ) + } + }, + ) + it(`rejects every hydration-buffered receipt when replay fails`, async () => { const adapter = createRecordingAdapter() let resolveLoadSubset: (() => void) | undefined @@ -15665,6 +15875,1274 @@ describeUnlessOracleReplay(`persistedCollectionOptions`, () => { await collection.cleanup() } }) + + // Focused collection-reset refinement: metadata and rows must come from one + // hydration scope. The adapter makes an interleaved v2 write possible only + // outside that scope, so the public v1 metadata and row are the independent + // coherence checkpoint. This fixed history does not model arbitrary resets. + it(`keeps a collection-reset reload inside one hydration scope`, async () => { + const adapter = createRecordingAdapter([{ id: `1`, title: `Initial row` }]) + adapter.collectionMetadata.set(`snapshot`, `initial`) + const coordinator = createCoordinatorHarness() + const loadSubset = adapter.loadSubset.bind(adapter) + let inHydrationScope = false + let interleaveArmed = false + let interleaveRan = false + + const runInterleavedWrite = () => { + interleaveRan = true + adapter.collectionMetadata.set(`snapshot`, `v2`) + adapter.rows.set(`1`, { id: `1`, title: `v2 row` }) + } + + adapter.loadSubset = async (...args) => { + if (interleaveArmed && !inHydrationScope && !interleaveRan) { + runInterleavedWrite() + } + return loadSubset(...args) + } + adapter.runInHydrationScope = async (task) => { + inHydrationScope = true + try { + return await task(adapter) + } finally { + inHydrationScope = false + if (interleaveArmed && !interleaveRan) runInterleavedWrite() + } + } + + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + + await collection.preload() + adapter.collectionMetadata.set(`snapshot`, `v1`) + adapter.rows.set(`1`, { id: `1`, title: `v1 row` }) + interleaveArmed = true + + coordinator.emit({ + type: `collection:reset`, + schemaVersion: 1, + resetEpoch: 1, + }) + + await vi.waitFor(() => expect(interleaveRan).toBe(true)) + expect(collection._state.syncedCollectionMetadata.get(`snapshot`)).toBe( + `v1`, + ) + expect(collection.get(`1`)?.title).toBe(`v1 row`) + await collection.cleanup() + }) + + // A remote commit queued during startup must run after the local hydration + // scope releases the shared driver. A reset racing an unscheduled startup + // baseline may publish before or after that baseline, but must own the final + // public snapshot and may never be overwritten by an older baseline. + it(`does not let an unscheduled startup baseline overwrite a newer collection reset`, async () => { + const adapter = createRecordingAdapter([{ id: `1`, title: `old` }]) + const coordinator = createCoordinatorHarness() + const baselineEntered = createEventGate() + const releaseBaseline = createEventGate() + const resetReloadEntered = createEventGate() + const loadResumeSnapshot = adapter.loadResumeSnapshot.bind(adapter) + const loadSubset = adapter.loadSubset.bind(adapter) + let heldBaseline = false + adapter.loadResumeSnapshot = async (...args) => { + const snapshot = await loadResumeSnapshot(...args) + if (args[1]?.includeRows && !heldBaseline) { + heldBaseline = true + baselineEntered.resolve() + await releaseBaseline.promise + } + return snapshot + } + adapter.loadSubset = (...args) => { + resetReloadEntered.resolve() + return loadSubset(...args) + } + + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present`, + getKey: (item) => item.id, + sync: { sync: ({ markReady }) => markReady() }, + persistence: { adapter, coordinator }, + }), + ) + const preload = collection.preload() + void preload.catch(() => undefined) + const publishedTitles: Array = [] + const subscription = collection.subscribeChanges( + () => publishedTitles.push(collection.get(`1`)?.title), + { includeInitialState: false }, + ) + + try { + await atPersistedOracleCheckpoint( + baselineEntered.promise, + `unscheduled baseline entered`, + ) + adapter.rows.set(`1`, { id: `1`, title: `new` }) + coordinator.emit({ + type: `collection:reset`, + schemaVersion: 1, + resetEpoch: 1, + }) + // Give a concurrent reset its event-loop turn. Serialization may instead + // hold it behind the baseline; both orders must converge without a + // public new -> old reversion. + await flushAsyncWork() + expect([undefined, `new`]).toContain(collection.get(`1`)?.title) + releaseBaseline.resolve() + await atPersistedOracleCheckpoint(preload, `startup after reset`) + await atPersistedOracleCheckpoint( + resetReloadEntered.promise, + `reset reload after startup`, + ) + await vi.waitFor(() => expect(collection.get(`1`)?.title).toBe(`new`)) + const firstNew = publishedTitles.indexOf(`new`) + expect(firstNew).toBeGreaterThanOrEqual(0) + expect(publishedTitles.slice(firstNew)).not.toContain(`old`) + } finally { + releaseBaseline.resolve() + await preload.catch(() => undefined) + subscription.unsubscribe() + await collection.cleanup() + } + }) + + it.each([`contiguous-reload`, `sequence-gap`] as const)( + `finishes queued %s after scheduled startup hydration releases its scope`, + async (route) => { + const adapter = createRecordingAdapter([{ id: `1`, title: `old` }]) + const coordinator = createCoordinatorHarness() + const baselineEntered = createEventGate() + const releaseBaseline = createEventGate() + const nestedScopeRequested = createEventGate() + const queuedScopes: Array<() => void> = [] + const loadResumeSnapshot = adapter.loadResumeSnapshot.bind(adapter) + let heldBaseline = false + let scopeActive = false + + adapter.loadResumeSnapshot = async (...args) => { + const snapshot = await loadResumeSnapshot(...args) + if (args[1]?.includeRows && !heldBaseline) { + heldBaseline = true + baselineEntered.resolve() + await releaseBaseline.promise + } + return snapshot + } + + const scopedAdapter: RecordingAdapter = { ...adapter } + scopedAdapter.runInHydrationScope = (task) => task(scopedAdapter) + adapter.runInHydrationScope = (task) => { + if (scopeActive) { + nestedScopeRequested.resolve() + return new Promise((resolve, reject) => { + queuedScopes.push(() => { + void task(scopedAdapter).then(resolve, reject) + }) + }) + } + scopeActive = true + return Promise.resolve() + .then(() => task(scopedAdapter)) + .finally(() => { + scopeActive = false + while (queuedScopes.length > 0) queuedScopes.shift()?.() + }) + } + + if (route === `sequence-gap`) { + coordinator.setPullSinceResponse({ + type: `rpc:pullSince:res`, + rpcId: `review-gap`, + ok: true, + latestTerm: 1, + latestSeq: 2, + latestRowVersion: 2, + requiresFullReload: true, + }) + } + const pullSince = coordinator.pullSince!.bind(coordinator) + let coordinatorEnteredDuringScope: boolean | undefined + coordinator.pullSince = (...args) => { + coordinatorEnteredDuringScope = scopeActive + return pullSince(...args) + } + + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present`, + getKey: (item) => item.id, + sync: { sync: ({ markReady }) => markReady() }, + persistence: { adapter, coordinator }, + }), + ) + const preload = collection.preload() + void preload.catch(() => undefined) + + try { + await atPersistedOracleCheckpoint( + baselineEntered.promise, + `${route} baseline entered`, + ) + adapter.rows.set(`1`, { id: `1`, title: `new` }) + const committed: TxCommitted = + route === `sequence-gap` + ? { + type: `tx:committed`, + term: 1, + seq: 2, + txId: `review-sequence-gap`, + latestRowVersion: 2, + requiresFullReload: false, + changedRows: [], + deletedKeys: [], + } + : { + type: `tx:committed`, + term: 1, + seq: 1, + txId: `review-contiguous-reload`, + latestRowVersion: 1, + requiresFullReload: true, + } + coordinator.emit(committed) + releaseBaseline.resolve() + const first = await atPersistedOracleCheckpoint( + Promise.race([ + preload.then(() => `startup-settled` as const), + nestedScopeRequested.promise.then(() => `nested-scope` as const), + ]), + `${route} startup or nested scope`, + ) + expect(first).toBe(`startup-settled`) + expect(collection.get(`1`)?.title).toBe(`new`) + if (route === `sequence-gap`) { + expect(coordinator.pullSinceCalls).toBe(1) + expect(coordinatorEnteredDuringScope).toBe(false) + } + } finally { + releaseBaseline.resolve() + while (queuedScopes.length > 0) queuedScopes.shift()?.() + await preload.catch(() => undefined) + await collection.cleanup() + } + }, + ) + + // Focused receipt-ownership refinements. A source receipt created by the + // hydration operation belongs to its waiter even if it rejects before the + // waiter snapshots; a receipt created after hydration work returns does not. + // The public load result and exact rejection identity distinguish those two + // boundaries without treating every pending source receipt as related. + it(`propagates an operation-owned receipt rejection that settles before the hydration waiter snapshots`, async () => { + const adapter = createRecordingAdapter() + const hydrateLoadEntered = createDeferred() + const allowHydrateLoad = createDeferred() + let gateHydrationLoad = true + adapter.loadSubset = async () => { + if (gateHydrationLoad) { + gateHydrationLoad = false + hydrateLoadEntered.resolve() + await allowHydrateLoad.promise + } + return [] + } + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: + | ((signal?: AbortSignal) => true | Promise) + | undefined + + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present-settled-receipt-boundary`, + getKey: (item) => item.id, + syncMode: `on-demand`, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit + markReady() + return { loadSubset: () => true } + }, + }, + persistence: { adapter }, + }), + ) + let load: Promise | undefined + let receipt: Promise | undefined + const abortController = new AbortController() + + try { + collection.startSyncImmediate() + await collection.stateWhenReady() + load = Promise.resolve(collection._sync.loadSubset({ limit: 1 })) + await hydrateLoadEntered.promise + + let keyReads = 0 + const establishingRow = { + get id() { + keyReads++ + if (keyReads === 2) abortController.abort() + return `establishing` + }, + title: `Abort during buffered replay`, + } + remoteBegin?.() + remoteWrite?.({ type: `insert`, value: establishingRow }) + const applied = remoteCommit?.(abortController.signal) + if (!(applied instanceof Promise)) { + throw new Error(`expected a buffered establishing receipt`) + } + receipt = applied + void receipt.catch(() => undefined) + + allowHydrateLoad.resolve() + const [loadResult, receiptResult] = await Promise.allSettled([ + load, + receipt, + ]) + + expect(keyReads).toBeGreaterThanOrEqual(2) + expect(abortController.signal.aborted).toBe(true) + expect(receiptResult.status).toBe(`rejected`) + expect(loadResult.status).toBe(`rejected`) + if ( + loadResult.status === `rejected` && + receiptResult.status === `rejected` + ) { + expect(loadResult.reason).toBe(receiptResult.reason) + } + } finally { + abortController.abort() + allowHydrateLoad.resolve() + await receipt?.catch(() => undefined) + await load?.catch(() => undefined) + await collection.cleanup() + } + }) + + it(`does not adopt an unrelated source receipt created after hydration work returns`, async () => { + const adapter = createRecordingAdapter() + const mutationEntered = createDeferred() + const releaseMutation = createDeferred() + const unrelatedStarted = createDeferred<{ + abortController: AbortController + receipt: Promise + }>() + const trace: Array = [] + let probeActive = false + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `insert`; value: Todo }) => void) + | undefined + let remoteCommit: + | ((signal?: AbortSignal) => true | Promise) + | undefined + + adapter.runInHydrationScope = async (task) => { + if (!probeActive) return task(adapter) + probeActive = false + + const result = await task(adapter) + trace.push(`hydrate-task-returned`) + + const abortController = new AbortController() + remoteBegin?.() + remoteWrite?.({ + type: `insert`, + value: { id: `unrelated`, title: `Outside hydrate boundary` }, + }) + const receipt = remoteCommit?.(abortController.signal) + if (!(receipt instanceof Promise)) { + throw new Error(`expected a pending unrelated receipt`) + } + trace.push(`unrelated-receipt-created`) + unrelatedStarted.resolve({ abortController, receipt }) + return result + } + + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present-unrelated-receipt-boundary`, + getKey: (item) => item.id, + syncMode: `on-demand`, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `insert` + value: Todo + }) => void + remoteCommit = commit + markReady() + return { loadSubset: () => true } + }, + }, + persistence: { adapter }, + onInsert: async () => { + mutationEntered.resolve() + await releaseMutation.promise + }, + }), + ) + let mutation: ReturnType | undefined + let load: Promise | undefined + let unrelated: + | { abortController: AbortController; receipt: Promise } + | undefined + + try { + collection.startSyncImmediate() + await collection.stateWhenReady() + mutation = collection.insert({ id: `local`, title: `Persisting gate` }) + await mutationEntered.promise + + probeActive = true + load = Promise.resolve(collection._sync.loadSubset({ limit: 1 })) + unrelated = await unrelatedStarted.promise + expect(trace).toEqual([ + `hydrate-task-returned`, + `unrelated-receipt-created`, + ]) + + // Give the public load continuation the opportunity to snapshot receipts. + await flushAsyncWork() + unrelated.abortController.abort() + await unrelated.receipt.catch(() => undefined) + + await expect(load).resolves.toBeUndefined() + } finally { + unrelated?.abortController.abort() + releaseMutation.resolve() + await mutation?.isPersisted.promise.catch(() => undefined) + await load?.catch(() => undefined) + await collection.cleanup() + } + }) + + it(`only signals completed leader-local index work after local success`, async () => { + const adapter = createRecordingAdapter() + const coordinator = createCoordinatorHarness() + const localFailure = new Error(`local index creation failed`) + const completedLocalMarkers: Array = [] + const warn = vi.spyOn(console, `warn`).mockImplementation(() => {}) + + adapter.ensureIndex = async (collectionId, signature) => { + adapter.ensureIndexCalls.push({ collectionId, signature }) + throw localFailure + } + coordinator.requestEnsurePersistedIndex = async ( + _collectionId, + _signature, + _spec, + completedLocalAdapter, + localEnsureCompleted, + ) => { + completedLocalMarkers.push( + completedLocalAdapter !== undefined && localEnsureCompleted === true, + ) + } + + const collection = createCollection( + persistedCollectionOptions({ + id: `failed-local-index-bootstrap`, + getKey: (item) => item.id, + defaultIndexType: BasicIndex, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + collection.createIndex((row) => row.title, { name: `startup-title` }) + + try { + await collection.preload() + expect(completedLocalMarkers).toEqual([false]) + } finally { + await collection.cleanup() + warn.mockRestore() + } + }) + + it(`releases the hydration scope before invoking a coordinator that uses its own adapter`, async () => { + const adapter = createRecordingAdapter() + const coordinatorEntered = createDeferred() + const breakSchedulerCycle = createDeferred() + let hydrationScopeActive = false + let coordinatorEnteredDuringHydration: boolean | undefined + + const publicEnsureIndex = adapter.ensureIndex.bind(adapter) + const scopedAdapter: PersistenceAdapter = { + ...adapter, + ensureIndex: publicEnsureIndex, + } + adapter.runInHydrationScope = async (task) => { + hydrationScopeActive = true + try { + return await task(scopedAdapter) + } finally { + hydrationScopeActive = false + } + } + adapter.ensureIndex = async (...args) => { + if (hydrationScopeActive) { + // A public core-adapter call queues behind the active hydrate. The + // hydrate cannot release until this coordinator call returns. + await breakSchedulerCycle.promise + } + await publicEnsureIndex(...args) + } + + const coordinator = createCoordinatorHarness() + coordinator.requestEnsurePersistedIndex = async ( + collectionId, + signature, + spec, + ) => { + coordinatorEnteredDuringHydration = hydrationScopeActive + coordinatorEntered.resolve() + // Deliberately ignore the optional scoped adapter, as existing public + // coordinator implementations are allowed to do. + await adapter.ensureIndex(collectionId, signature, spec) + } + + const collection = createCollection( + persistedCollectionOptions({ + id: `custom-coordinator-hydration-scope`, + getKey: (item) => item.id, + defaultIndexType: BasicIndex, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + collection.createIndex((row) => row.title, { + name: `startup-title`, + }) + const preload = Promise.resolve(collection.preload()) + void preload.catch(() => undefined) + + try { + await coordinatorEntered.promise + expect(coordinatorEnteredDuringHydration).toBe(false) + } finally { + breakSchedulerCycle.resolve() + await preload.catch(() => undefined) + await collection.cleanup() + } + }) + + it(`releases crossed follower hydration scopes before leader index RPC work`, async () => { + const bothLocalIndexesEntered = createDeferred() + const bothCoordinatorRequestsEntered = createDeferred() + let localIndexEntries = 0 + let coordinatorEntries = 0 + + const createTabAdapter = () => { + const adapter = createRecordingAdapter() + const breakSchedulerCycle = createDeferred() + let hydrationScopeActive = false + const remoteScopeObservations: Array = [] + const publicEnsureIndex = adapter.ensureIndex.bind(adapter) + const scopedAdapter: PersistenceAdapter = { + ...adapter, + ensureIndex: async (...args) => { + localIndexEntries++ + if (localIndexEntries === 2) bothLocalIndexesEntered.resolve() + await bothLocalIndexesEntered.promise + await publicEnsureIndex(...args) + }, + } + adapter.runInHydrationScope = async (task) => { + hydrationScopeActive = true + try { + return await task(scopedAdapter) + } finally { + hydrationScopeActive = false + } + } + adapter.ensureIndex = async (...args) => { + remoteScopeObservations.push(hydrationScopeActive) + coordinatorEntries++ + if (coordinatorEntries === 2) bothCoordinatorRequestsEntered.resolve() + if (hydrationScopeActive) await breakSchedulerCycle.promise + await publicEnsureIndex(...args) + } + return { + adapter, + breakSchedulerCycle, + remoteScopeObservations, + } + } + + const tab1 = createTabAdapter() + const tab2 = createTabAdapter() + const coordinator1 = createCoordinatorHarness() + const coordinator2 = createCoordinatorHarness() + coordinator1.isLeader = () => false + coordinator2.isLeader = () => false + coordinator1.requestEnsurePersistedIndex = ( + collectionId, + signature, + spec, + ) => tab2.adapter.ensureIndex(collectionId, signature, spec) + coordinator2.requestEnsurePersistedIndex = ( + collectionId, + signature, + spec, + ) => tab1.adapter.ensureIndex(collectionId, signature, spec) + + const createFollowerCollection = ( + id: string, + persistence: PersistedCollectionPersistence, + ) => { + const collection = createCollection( + persistedCollectionOptions({ + id, + getKey: (item) => item.id, + defaultIndexType: BasicIndex, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence, + }), + ) + collection.createIndex((row) => row.title, { name: `${id}-title` }) + return collection + } + const followerA = createFollowerCollection(`follower-a`, { + adapter: tab1.adapter, + coordinator: coordinator1, + }) + const followerB = createFollowerCollection(`follower-b`, { + adapter: tab2.adapter, + coordinator: coordinator2, + }) + const preloadA = Promise.resolve(followerA.preload()) + const preloadB = Promise.resolve(followerB.preload()) + void preloadA.catch(() => undefined) + void preloadB.catch(() => undefined) + + try { + await bothCoordinatorRequestsEntered.promise + expect({ + tab1: tab1.remoteScopeObservations, + tab2: tab2.remoteScopeObservations, + }).toEqual({ tab1: [false], tab2: [false] }) + } finally { + tab1.breakSchedulerCycle.resolve() + tab2.breakSchedulerCycle.resolve() + await Promise.all([preloadA, preloadB]).catch(() => undefined) + await Promise.all([followerA.cleanup(), followerB.cleanup()]) + } + }) + + it(`keeps generated crossed-leadership index RPC histories outside local hydration scopes`, async () => { + let run = 0 + await fc.assert( + fc.asyncProperty( + fc.record({ + tabCount: fc.integer({ min: 2, max: 4 }), + indexCount: fc.integer({ min: 1, max: 2 }), + direction: fc.constantFrom(-1, 1), + }), + async ({ tabCount, indexCount, direction }) => { + run++ + const allTabsAtLocalIndex = createDeferred() + let tabsAtLocalIndex = 0 + const tabs = Array.from({ length: tabCount }, () => { + const adapter = createRecordingAdapter() + let hydrationScopeActive = false + let localIndexCalls = 0 + const remoteScopeObservations: Array = [] + const publicEnsureIndex = adapter.ensureIndex.bind(adapter) + const scopedAdapter: PersistenceAdapter = { + ...adapter, + ensureIndex: async (...args) => { + localIndexCalls++ + if (localIndexCalls === 1) { + tabsAtLocalIndex++ + if (tabsAtLocalIndex === tabCount) { + allTabsAtLocalIndex.resolve() + } + await allTabsAtLocalIndex.promise + } + await publicEnsureIndex(...args) + }, + } + adapter.runInHydrationScope = async (task) => { + hydrationScopeActive = true + try { + return await task(scopedAdapter) + } finally { + hydrationScopeActive = false + } + } + adapter.ensureIndex = async (...args) => { + remoteScopeObservations.push(hydrationScopeActive) + await publicEnsureIndex(...args) + } + return { adapter, remoteScopeObservations } + }) + + const collections = tabs.map((tab, index) => { + const remoteIndex = (index + direction + tabCount) % tabCount + const coordinator = createCoordinatorHarness() + coordinator.isLeader = () => false + coordinator.requestEnsurePersistedIndex = ( + collectionId, + signature, + spec, + ) => + tabs[remoteIndex]!.adapter.ensureIndex( + collectionId, + signature, + spec, + ) + const collection = createCollection( + persistedCollectionOptions({ + id: `generated-crossed-${run}-${index}`, + getKey: (item) => item.id, + defaultIndexType: BasicIndex, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { adapter: tab.adapter, coordinator }, + }), + ) + for ( + let indexOrdinal = 0; + indexOrdinal < indexCount; + indexOrdinal++ + ) { + collection.createIndex( + indexOrdinal % 2 === 0 ? (row) => row.title : (row) => row.id, + { name: `idx-${indexOrdinal}` }, + ) + } + return collection + }) + + try { + await Promise.all( + collections.map((collection) => collection.preload()), + ) + for (const tab of tabs) { + expect(tab.remoteScopeObservations).toEqual( + Array.from({ length: indexCount }, () => false), + ) + } + } finally { + await Promise.all( + collections.map((collection) => collection.cleanup()), + ) + } + }, + ), + { seed: 1868, numRuns: 8, endOnFailure: true }, + ) + }) + + // Focused R7 causal-replay witness: after hydration releases its buffer, the + // source receipt must replay without awaiting the persisting operation whose + // callback is itself awaiting that receipt. Persistence reach, both public + // settlements, and the final source row expose the otherwise hidden cycle. + it(`replays a buffered source receipt without blocking its persisting predecessor`, async () => { + const adapter = createRecordingAdapter() + const hydrateLoadEntered = createDeferred() + const allowHydrateLoad = createDeferred() + let gateHydrationLoad = true + const replayState = { persisted: false } + + const loadResumeSnapshot = adapter.loadResumeSnapshot.bind(adapter) + adapter.loadResumeSnapshot = async (...args) => { + if (args[1]?.includeRows === true && gateHydrationLoad) { + gateHydrationLoad = false + hydrateLoadEntered.resolve() + await allowHydrateLoad.promise + } + return loadResumeSnapshot(...args) + } + const applyCommittedTx = adapter.applyCommittedTx + adapter.applyCommittedTx = async (...args) => { + replayState.persisted = true + await applyCommittedTx(...args) + } + adapter.runInHydrationScope = (task) => task(adapter) + + let remoteBegin: (() => void) | undefined + let remoteWrite: + | ((message: { type: `update`; value: Todo }) => void) + | undefined + let remoteCommit: (() => true | Promise) | undefined + const sourceReady = createDeferred() + const bufferedCommitReturned = createDeferred<{ + receipt: Promise + }>() + + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present-buffered-causal-replay`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + remoteBegin = begin + remoteWrite = write as (message: { + type: `update` + value: Todo + }) => void + remoteCommit = commit + sourceReady.resolve() + markReady() + return { loadSubset: () => true } + }, + }, + persistence: { adapter }, + onInsert: async () => { + if (!remoteBegin || !remoteWrite || !remoteCommit) { + throw new Error(`source sync is not ready`) + } + remoteBegin() + remoteWrite({ + type: `update`, + value: { id: `source-row`, title: `Buffered during hydrate` }, + }) + const applied = remoteCommit() + if (applied === true) { + throw new Error(`source commit was not buffered during hydration`) + } + bufferedCommitReturned.resolve({ receipt: applied }) + await applied + }, + }), + ) + + const preload = Promise.resolve(collection.preload()) + void preload.catch(() => undefined) + let mutationPersisted: Promise | undefined + let bufferedReceipt: Promise | undefined + + try { + await atPersistedOracleCheckpoint( + hydrateLoadEntered.promise, + `buffered causal replay hydration entered`, + ) + await atPersistedOracleCheckpoint( + sourceReady.promise, + `buffered causal replay source ready`, + ) + + const mutation = collection.insert({ id: `local`, title: `Pending` }) + mutationPersisted = mutation.isPersisted.promise + void mutationPersisted.catch(() => undefined) + const bufferedCommit = await atPersistedOracleCheckpoint( + bufferedCommitReturned.promise, + `buffered causal replay commit returned`, + ) + bufferedReceipt = bufferedCommit.receipt + void bufferedReceipt.catch(() => undefined) + + allowHydrateLoad.resolve() + + let causalCycleObserved = false + for ( + let attempt = 0; + attempt < 100 && !replayState.persisted; + attempt++ + ) { + causalCycleObserved = collection._state.pendingSyncedTransactions.some( + (transaction) => + transaction.committed && transaction.applied.isPending(), + ) + if (causalCycleObserved) break + await Promise.resolve() + } + + expect(causalCycleObserved).toBe(false) + expect(replayState.persisted).toBe(true) + await expect( + atPersistedOracleCheckpoint( + bufferedReceipt, + `buffered causal replay source receipt`, + ), + ).resolves.toBeUndefined() + await expect( + atPersistedOracleCheckpoint( + mutationPersisted, + `buffered causal replay mutation persisted`, + ), + ).resolves.toBeDefined() + await expect( + atPersistedOracleCheckpoint( + preload, + `buffered causal replay preload settled`, + ), + ).resolves.toBeUndefined() + expect(stripVirtualProps(collection.get(`source-row`))).toEqual({ + id: `source-row`, + title: `Buffered during hydrate`, + }) + } finally { + allowHydrateLoad.resolve() + await collection.cleanup() + } + }) + + it(`releases the hydration scope before a gap coordinator uses its own adapter`, async () => { + const adapter = createRecordingAdapter() + const coordinator = createCoordinatorHarness() + const coordinatorEntered = createDeferred() + const breakSchedulerCycle = createDeferred() + let hydrationScopeActive = false + let coordinatorEnteredDuringHydration: boolean | undefined + + const pullSince = async () => ({ + latestRowVersion: 0, + requiresFullReload: false as const, + changedKeys: [], + deletedKeys: [], + deltas: [], + }) + const publicAdapter = adapter as RecordingAdapter & { + pullSince: typeof pullSince + } + const scopedAdapter = { + ...adapter, + pullSince, + } + publicAdapter.pullSince = async () => { + if (hydrationScopeActive) await breakSchedulerCycle.promise + return pullSince() + } + adapter.runInHydrationScope = async (task) => { + hydrationScopeActive = true + try { + return await task(scopedAdapter) + } finally { + hydrationScopeActive = false + } + } + coordinator.pullSince = async (_collectionId, _fromRowVersion) => { + coordinatorEnteredDuringHydration = hydrationScopeActive + coordinatorEntered.resolve() + const result = await publicAdapter.pullSince() + return { + type: `rpc:pullSince:res`, + rpcId: `legacy-gap-coordinator`, + ok: true, + latestTerm: 1, + latestSeq: 2, + latestRowVersion: result.latestRowVersion, + requiresFullReload: result.requiresFullReload, + changedKeys: result.changedKeys, + deletedKeys: result.deletedKeys, + deltas: result.deltas, + } + } + + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + await collection.preload() + + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 2, + txId: `tx-gap-legacy-coordinator`, + latestRowVersion: 2, + requiresFullReload: false, + changedRows: [], + deletedKeys: [], + }) + + try { + await coordinatorEntered.promise + expect(coordinatorEnteredDuringHydration).toBe(false) + } finally { + breakSchedulerCycle.resolve() + await flushAsyncWork() + await collection.cleanup() + } + }) + + // Focused invalidation-reload refinements. Whether recovery follows a + // sequence gap or a contiguous committed notification, metadata and rows + // must be read inside one hydration scope. The adapter schedules v2 only + // outside the scope; coherent public v1 state is the checkpoint. + it(`keeps sequence-gap recovery inside one hydration scope`, async () => { + const adapter = createRecordingAdapter([{ id: `1`, title: `Initial row` }]) + adapter.collectionMetadata.set(`snapshot`, `initial`) + const coordinator = createCoordinatorHarness() + coordinator.setPullSinceResponse({ + type: `rpc:pullSince:res`, + rpcId: `pull-gap-scope`, + ok: true, + latestTerm: 1, + latestSeq: 1, + latestRowVersion: 1, + requiresFullReload: true, + }) + const loadSubset = adapter.loadSubset.bind(adapter) + let inHydrationScope = false + let interleaveArmed = false + let interleaveRan = false + + const runInterleavedWrite = () => { + interleaveRan = true + adapter.collectionMetadata.set(`snapshot`, `v2`) + adapter.rows.set(`1`, { id: `1`, title: `v2 row` }) + } + + adapter.loadSubset = async (...args) => { + if (interleaveArmed && !inHydrationScope && !interleaveRan) { + runInterleavedWrite() + } + return loadSubset(...args) + } + adapter.runInHydrationScope = async (task) => { + inHydrationScope = true + try { + return await task(adapter) + } finally { + inHydrationScope = false + if (interleaveArmed && !interleaveRan) runInterleavedWrite() + } + } + + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + + await collection.preload() + adapter.collectionMetadata.set(`snapshot`, `v1`) + adapter.rows.set(`1`, { id: `1`, title: `v1 row` }) + interleaveArmed = true + + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 2, + txId: `tx-gap-scope`, + latestRowVersion: 2, + requiresFullReload: false, + changedRows: [], + deletedKeys: [], + }) + + await vi.waitFor(() => expect(interleaveRan).toBe(true)) + expect(collection._state.syncedCollectionMetadata.get(`snapshot`)).toBe( + `v1`, + ) + expect(collection.get(`1`)?.title).toBe(`v1 row`) + await collection.cleanup() + }) + + it(`keeps contiguous committed reload inside one hydration scope`, async () => { + const adapter = createRecordingAdapter([{ id: `1`, title: `Initial row` }]) + adapter.collectionMetadata.set(`snapshot`, `initial`) + const coordinator = createCoordinatorHarness() + const loadSubset = adapter.loadSubset.bind(adapter) + let inHydrationScope = false + let interleaveArmed = false + let interleaveRan = false + + const runInterleavedWrite = () => { + interleaveRan = true + adapter.collectionMetadata.set(`snapshot`, `v2`) + adapter.rows.set(`1`, { id: `1`, title: `v2 row` }) + } + + adapter.loadSubset = async (...args) => { + if (interleaveArmed && !inHydrationScope && !interleaveRan) { + runInterleavedWrite() + } + return loadSubset(...args) + } + adapter.runInHydrationScope = async (task) => { + inHydrationScope = true + try { + return await task(adapter) + } finally { + inHydrationScope = false + if (interleaveArmed && !interleaveRan) runInterleavedWrite() + } + } + + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { adapter, coordinator }, + }), + ) + + await collection.preload() + adapter.collectionMetadata.set(`snapshot`, `v1`) + adapter.rows.set(`1`, { id: `1`, title: `v1 row` }) + interleaveArmed = true + + coordinator.emit({ + type: `tx:committed`, + term: 1, + seq: 1, + txId: `tx-contiguous-reload-scope`, + latestRowVersion: 1, + requiresFullReload: true, + }) + + await vi.waitFor(() => expect(interleaveRan).toBe(true)) + expect(collection._state.syncedCollectionMetadata.get(`snapshot`)).toBe( + `v1`, + ) + expect(collection.get(`1`)?.title).toBe(`v1 row`) + await collection.cleanup() + }) + + // Focused lifecycle-fencing witness: generation-zero startup is held across + // cleanup and rebound, then released while generation one is still loading. + // Zero ensure-index calls for the rebound signatures prove stale bootstrap + // and listener work did not cross the public lifecycle boundary. + it(`does not let stale startup install index work on a rebound lifecycle`, async () => { + const adapter = createRecordingAdapter() + const g0MetadataEntered = createDeferred() + const allowG0Metadata = createDeferred() + const g1MetadataEntered = createDeferred() + const allowG1Metadata = createDeferred() + let metadataCalls = 0 + const loadResumeSnapshot = adapter.loadResumeSnapshot.bind(adapter) + adapter.loadResumeSnapshot = async (...args) => { + if (args[1]?.includeRows === false) { + metadataCalls++ + if (metadataCalls === 1) { + g0MetadataEntered.resolve() + await allowG0Metadata.promise + } else if (metadataCalls === 2) { + g1MetadataEntered.resolve() + await allowG1Metadata.promise + } + } + return loadResumeSnapshot(...args) + } + + const collection = createCollection( + persistedCollectionOptions({ + id: `sync-present-startup-index-generation`, + getKey: (item) => item.id, + defaultIndexType: BasicIndex, + sync: { + sync: ({ markReady }) => { + markReady() + }, + }, + persistence: { adapter }, + }), + ) + const stalePreload = Promise.resolve(collection.preload()) + void stalePreload.catch(() => undefined) + let freshReady: Promise | undefined + + try { + await g0MetadataEntered.promise + await collection.cleanup() + + const reboundIndex = collection.createIndex((row) => row.title, { + name: `rebound-bootstrap`, + }) + const reboundSignature = collection + .getIndexMetadata() + .find((metadata) => metadata.indexId === reboundIndex.id)?.signature + expect(reboundSignature).toBeDefined() + freshReady = collection.stateWhenReady() + + allowG0Metadata.resolve() + await g1MetadataEntered.promise + + const staleBootstrapCalls = adapter.ensureIndexCalls.filter( + (call) => call.signature === reboundSignature, + ) + const listenerIndex = collection.createIndex((row) => row.id, { + name: `rebound-listener`, + }) + const listenerSignature = collection + .getIndexMetadata() + .find((metadata) => metadata.indexId === listenerIndex.id)?.signature + expect(listenerSignature).toBeDefined() + const staleListenerCalls = adapter.ensureIndexCalls.filter( + (call) => call.signature === listenerSignature, + ) + + expect({ + staleBootstrapCalls: staleBootstrapCalls.length, + staleListenerCalls: staleListenerCalls.length, + }).toEqual({ + staleBootstrapCalls: 0, + staleListenerCalls: 0, + }) + + allowG1Metadata.resolve() + await freshReady + } finally { + allowG0Metadata.resolve() + allowG1Metadata.resolve() + await stalePreload.catch(() => undefined) + await freshReady?.catch(() => undefined) + await collection.cleanup() + } + }) }) describeUnlessOracleReplay(`persisted key and identifier helpers`, () => { diff --git a/packages/db-sqlite-persistence-core/tests/shared-logical-scheduling.test.ts b/packages/db-sqlite-persistence-core/tests/shared-logical-scheduling.test.ts new file mode 100644 index 0000000000..4813fa4746 --- /dev/null +++ b/packages/db-sqlite-persistence-core/tests/shared-logical-scheduling.test.ts @@ -0,0 +1,386 @@ +/** + * # Which adapters share one logical scheduling boundary? + * + * Contract and source: the RFC #1659 driver protocol keys scheduling by the + * exact shared driver identity. Two fresh core adapters over that identity + * must not interleave a complete hydration scope with regular adapter work. + * Transparent wrappers must forward the identity unchanged, including when + * the identity itself is a function object. + * + * Independent relation and legal domain: a two-adapter history starts one + * hydration metadata query, holds it at the driver boundary, then requests one + * regular metadata query. Before release, exactly the first query may be + * admitted; after release, both operations must finish. The three legal driver + * forms are direct, transparently wrapped, and direct with a function-valued + * key. This relation records admissions without copying the production + * scheduler or invoking the opaque key. Promise-only discovery is a fourth + * form: the first operation reveals the shared identity and its complete + * hydration scope becomes the scheduler's already-running unit. + * + * Production boundary and checkpoint: both operations use + * `createSQLiteCorePersistenceAdapter`; the hydration operation enters through + * `runInHydrationScope`. The first checkpoint is immediately after the regular + * request enters the adapter, while the first query remains held. Losing + * wrapper identity or treating a function key as a getter synchronously admits + * the second query and fails the exact admission assertion. + * + * Known omissions: promise-only discovery coordinates adapters over the same + * wrapper identity. Distinct unbranded wrappers must forward the shared key + * before adapter construction. This focused contract test does not establish + * K=1 lane fairness, SQL result correctness, eventual progress under arbitrary + * I/O, or cross-process coordination. Those belong to the shared-driver oracle + * and provider refinements. + */ +import { describe, expect, it } from 'vitest' +import fc from 'fast-check' +import { + SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY, + createSQLiteCorePersistenceAdapter, + forwardSQLiteDriverSharedLogicalScheduling, +} from '../src' +import type { SQLiteDriver } from '../src' + +type Deferred = { + promise: Promise + resolve: () => void +} + +function createDeferred(): Deferred { + let resolve!: () => void + const promise = new Promise((settle) => { + resolve = settle + }) + return { promise, resolve } +} + +function fixtureQueryRows(sql: string): ReadonlyArray { + if (sql.includes(`FROM collection_registry`)) { + return [ + { + table_name: `c_fixture`, + tombstone_table_name: `t_fixture`, + schema_version: 1, + } as T, + ] + } + return [] +} + +function isMetadataQuery(sql: string): boolean { + return sql.includes(`FROM collection_metadata`) +} + +class FirstQueryGatedDriver implements SQLiteDriver { + readonly [SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY]: object + readonly admissions: Array = [] + readonly metadataAdmissions: Array = [] + readonly firstQueryEntered = createDeferred() + firstQueryAdmissionCount = 0 + onQuery: (() => void) | undefined + private readonly firstQueryGate = createDeferred() + private holdFirstQuery = true + + constructor(sharedLogicalSchedulingKey: object = {}) { + this[SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY] = + sharedLogicalSchedulingKey + } + + exec(): Promise { + this.admissions.push(`exec`) + return Promise.resolve() + } + + async query(sql: string): Promise> { + this.admissions.push(`query`) + if (isMetadataQuery(sql)) { + this.metadataAdmissions.push(`query`) + this.onQuery?.() + } + if (this.holdFirstQuery && isMetadataQuery(sql)) { + this.holdFirstQuery = false + this.firstQueryAdmissionCount = this.admissions.length + this.firstQueryEntered.resolve() + await this.firstQueryGate.promise + } + return fixtureQueryRows(sql) + } + + run(): Promise { + this.admissions.push(`run`) + return Promise.resolve() + } + + transaction( + fn: (transactionDriver: SQLiteDriver) => Promise, + ): Promise { + this.admissions.push(`transaction`) + return fn(this) + } + + releaseFirstQuery(): void { + this.firstQueryGate.resolve() + } +} + +class PromiseBrandedFirstQueryGatedDriver implements SQLiteDriver { + readonly admissions: Array = [] + readonly metadataAdmissions: Array = [] + readonly firstQueryEntered = createDeferred() + firstQueryAdmissionCount = 0 + private readonly firstQueryGate = createDeferred() + private holdFirstQuery = true + + constructor(private readonly schedulingKey: object = {}) {} + + exec(): Promise { + this.admissions.push(`exec`) + return this.brand(Promise.resolve()) + } + + query(sql: string): Promise> { + this.admissions.push(`query`) + if (isMetadataQuery(sql)) this.metadataAdmissions.push(`query`) + const result = (async () => { + if (this.holdFirstQuery && isMetadataQuery(sql)) { + this.holdFirstQuery = false + this.firstQueryAdmissionCount = this.admissions.length + this.firstQueryEntered.resolve() + await this.firstQueryGate.promise + } + return fixtureQueryRows(sql) + })() + return this.brand(result) + } + + run(): Promise { + this.admissions.push(`run`) + return this.brand(Promise.resolve()) + } + + transaction( + fn: (transactionDriver: SQLiteDriver) => Promise, + ): Promise { + this.admissions.push(`transaction`) + return this.brand(fn(this)) + } + + releaseFirstQuery(): void { + this.firstQueryGate.resolve() + } + + private brand(promise: Promise): Promise { + Object.defineProperty( + promise, + SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY, + { value: this.schedulingKey }, + ) + return promise + } +} + +class UnbrandedPromiseLookupDriver implements SQLiteDriver { + schedulingKeyLookups = 0 + + exec(): Promise { + return this.unbranded(Promise.resolve()) + } + + query(sql: string): Promise> { + return this.unbranded(Promise.resolve(fixtureQueryRows(sql))) + } + + run(): Promise { + return this.unbranded(Promise.resolve()) + } + + transaction( + fn: (transactionDriver: SQLiteDriver) => Promise, + ): Promise { + return this.unbranded(fn(this)) + } + + private unbranded(promise: Promise): Promise { + Object.defineProperty( + promise, + SQLITE_DRIVER_SHARED_LOGICAL_SCHEDULING_KEY, + { + configurable: true, + get: () => { + this.schedulingKeyLookups++ + return undefined + }, + }, + ) + return promise + } +} + +function createTransparentWrapper(driver: SQLiteDriver): SQLiteDriver { + return forwardSQLiteDriverSharedLogicalScheduling(driver, { + exec: (sql) => driver.exec(sql), + query: (sql: string, params: ReadonlyArray = []) => + driver.query(sql, params), + run: (sql, params = []) => driver.run(sql, params), + transaction: (fn: (transactionDriver: SQLiteDriver) => Promise) => + driver.transaction(fn), + transactionWithDriver: ( + fn: (transactionDriver: SQLiteDriver) => Promise, + ) => + driver.transactionWithDriver + ? driver.transactionWithDriver(fn) + : driver.transaction(fn), + }) +} + +describe(`shared logical scheduling`, () => { + it.each([ + { + name: `direct driver`, + key: {}, + wrap: (driver: SQLiteDriver) => driver, + }, + { + name: `transparent delegating driver`, + key: {}, + wrap: createTransparentWrapper, + }, + { + name: `twice-transparent delegating driver`, + key: {}, + wrap: (driver: SQLiteDriver) => + createTransparentWrapper(createTransparentWrapper(driver)), + }, + { + name: `function-valued key`, + key: () => undefined, + wrap: (driver: SQLiteDriver) => driver, + }, + ])( + `shares a scheduler across two fresh adapters through a $name`, + async ({ key, wrap }) => { + const underlying = new FirstQueryGatedDriver(key) + const driver = wrap(underlying) + const hydrateAdapter = createSQLiteCorePersistenceAdapter({ driver }) + const regularAdapter = createSQLiteCorePersistenceAdapter({ driver }) + + const hydrate = hydrateAdapter.runInHydrationScope!(async (scoped) => { + await scoped.loadCollectionMetadata!(`hydrate`) + }) + await underlying.firstQueryEntered.promise + + const regular = regularAdapter.loadCollectionMetadata!(`regular`) + + expect(underlying.admissions).toHaveLength( + underlying.firstQueryAdmissionCount, + ) + + underlying.releaseFirstQuery() + await Promise.all([hydrate, regular]) + expect(underlying.metadataAdmissions).toEqual([`query`, `query`]) + }, + ) + + it(`adopts a late-discovered promise identity before admitting peer work`, async () => { + const driver = new PromiseBrandedFirstQueryGatedDriver() + const hydrateAdapter = createSQLiteCorePersistenceAdapter({ driver }) + const regularAdapter = createSQLiteCorePersistenceAdapter({ driver }) + + const hydrate = hydrateAdapter.runInHydrationScope!(async (scoped) => { + await scoped.loadCollectionMetadata!(`hydrate`) + }) + await driver.firstQueryEntered.promise + + const regular = regularAdapter.loadCollectionMetadata!(`regular`) + + expect(driver.admissions).toHaveLength(driver.firstQueryAdmissionCount) + + driver.releaseFirstQuery() + await Promise.all([hydrate, regular]) + expect(driver.metadataAdmissions).toEqual([`query`, `query`]) + }) + + it(`keeps generated promise-discovered hydrate units non-preemptible`, async () => { + await fc.assert( + fc.asyncProperty( + fc.record({ + hydrateQueries: fc.integer({ min: 1, max: 4 }), + peerQueries: fc.integer({ min: 1, max: 3 }), + }), + async ({ hydrateQueries, peerQueries }) => { + const driver = new PromiseBrandedFirstQueryGatedDriver() + const hydrateAdapter = createSQLiteCorePersistenceAdapter({ driver }) + const peerAdapters = Array.from({ length: peerQueries }, () => + createSQLiteCorePersistenceAdapter({ driver }), + ) + + const hydrate = hydrateAdapter.runInHydrationScope!( + async (scoped) => { + for (let index = 0; index < hydrateQueries; index++) { + await scoped.loadCollectionMetadata!(`hydrate-${index}`) + } + }, + ) + await driver.firstQueryEntered.promise + + const peers = peerAdapters.map((adapter, index) => + adapter.loadCollectionMetadata!(`peer-${index}`), + ) + expect(driver.admissions).toHaveLength( + driver.firstQueryAdmissionCount, + ) + + driver.releaseFirstQuery() + await Promise.all([hydrate, ...peers]) + expect(driver.metadataAdmissions).toEqual( + Array.from({ length: hydrateQueries + peerQueries }, () => `query`), + ) + }, + ), + { seed: 1868, numRuns: 12, endOnFailure: true }, + ) + }) + + it(`counts a rejected hydrate as a completed K=1 lane unit`, async () => { + const driver = new FirstQueryGatedDriver() + const hydrateAdapter = createSQLiteCorePersistenceAdapter({ driver }) + const regularAdapter = createSQLiteCorePersistenceAdapter({ driver }) + const events: Array = [] + + const initialRegular = regularAdapter.loadCollectionMetadata!(`initial`) + await driver.firstQueryEntered.promise + driver.onQuery = () => events.push(`query`) + + const failedHydrate = hydrateAdapter.runInHydrationScope!(() => { + events.push(`failed-hydrate`) + throw new Error(`expected hydrate failure`) + }) + const failedHydrateExpectation = expect(failedHydrate).rejects.toThrow( + `expected hydrate failure`, + ) + const regular = regularAdapter.loadCollectionMetadata!(`regular`) + const nextHydrate = hydrateAdapter.runInHydrationScope!(async (scoped) => { + events.push(`next-hydrate`) + await scoped.loadCollectionMetadata!(`hydrate`) + }) + + driver.releaseFirstQuery() + await Promise.all([ + initialRegular, + failedHydrateExpectation, + regular, + nextHydrate, + ]) + + expect(events).toEqual([`failed-hydrate`, `query`, `next-hydrate`, `query`]) + }) + + it(`stops probing after the first returned promise lacks scheduling support`, async () => { + const driver = new UnbrandedPromiseLookupDriver() + const adapter = createSQLiteCorePersistenceAdapter({ driver }) + + await adapter.loadCollectionMetadata!(`first`) + await adapter.loadCollectionMetadata!(`second`) + + expect(driver.schedulingKeyLookups).toBe(1) + }) +}) diff --git a/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts b/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts index 2e5303c4a2..94d4b1f3f6 100644 --- a/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts +++ b/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts @@ -67,7 +67,7 @@ function interpolateSql(sql: string, params: ReadonlyArray): string { return renderedSql } -class SqliteCliDriver implements SQLiteDriver { +export class SqliteCliDriver implements SQLiteDriver { private readonly transactionDbPath = new AsyncLocalStorage() private queue: Promise = Promise.resolve() @@ -639,6 +639,47 @@ export type SQLiteCoreAdapterHarnessFactory = ( >, ) => SQLiteCoreAdapterContractHarness +function holdAndRejectFirstSubsetLoad( + adapter: PersistenceAdapter, + failure: Error, +): { entered: Promise; release: () => void } { + let enter!: () => void + let release!: () => void + const entered = new Promise((resolve) => { + enter = resolve + }) + const held = new Promise((resolve) => { + release = resolve + }) + let loadCalls = 0 + const intercept = + (loadSubset: PersistenceAdapter[`loadSubset`]) => + async (...args: Parameters) => { + loadCalls++ + if (loadCalls === 1) { + enter() + await held + throw failure + } + return loadSubset(...args) + } + + const runInHydrationScope = adapter.runInHydrationScope?.bind(adapter) + if (runInHydrationScope) { + adapter.runInHydrationScope = (task) => + runInHydrationScope((scopedAdapter) => + task({ + ...scopedAdapter, + loadSubset: intercept(scopedAdapter.loadSubset), + }), + ) + } else { + adapter.loadSubset = intercept(adapter.loadSubset.bind(adapter)) + } + + return { entered, release } +} + export function runSQLiteCoreAdapterContractSuite( suiteName: string = `SQLiteCorePersistenceAdapter`, harnessFactory: SQLiteCoreAdapterHarnessFactory = createHarness, @@ -781,25 +822,8 @@ export function runSQLiteCoreAdapterContractSuite( }) const loadSubset = adapter.loadSubset.bind(adapter) - let enterLoad!: () => void - let rejectLoad!: () => void - const loadEntered = new Promise((resolve) => { - enterLoad = resolve - }) - const loadRejected = new Promise((resolve) => { - rejectLoad = resolve - }) const subsetFailure = new Error(`controlled incremental subset failure`) - let loadCalls = 0 - adapter.loadSubset = async (...args) => { - loadCalls++ - if (loadCalls === 1) { - enterLoad() - await loadRejected - throw subsetFailure - } - return loadSubset(...args) - } + const subsetLoad = holdAndRejectFirstSubsetLoad(adapter, subsetFailure) let source!: Parameters[`sync`]>[0] const collection = createCollection( @@ -834,7 +858,7 @@ export function runSQLiteCoreAdapterContractSuite( () => undefined, ) void load.catch(() => undefined) - await loadEntered + await subsetLoad.entered source.begin() source.write({ @@ -853,7 +877,7 @@ export function runSQLiteCoreAdapterContractSuite( ) expect(receiptStatus).toBe(`pending`) - rejectLoad() + subsetLoad.release() await expect(load).rejects.toBe(subsetFailure) await receipt const durableBeforeRetry = await loadSubset(collectionId, {}) @@ -882,7 +906,7 @@ export function runSQLiteCoreAdapterContractSuite( publicError: undefined, }) } finally { - rejectLoad() + subsetLoad.release() await load?.catch(() => undefined) await receipt?.catch(() => undefined) await collection.cleanup() @@ -911,27 +935,10 @@ export function runSQLiteCoreAdapterContractSuite( } const loadSubset = adapter.loadSubset.bind(adapter) - let enterLoad!: () => void - let rejectLoad!: () => void - const loadEntered = new Promise((resolve) => { - enterLoad = resolve - }) - const loadRejected = new Promise((resolve) => { - rejectLoad = resolve - }) const subsetFailure = new Error( `controlled ${operation} subset failure`, ) - let loadCalls = 0 - adapter.loadSubset = async (...args) => { - loadCalls++ - if (loadCalls === 1) { - enterLoad() - await loadRejected - throw subsetFailure - } - return loadSubset(...args) - } + const subsetLoad = holdAndRejectFirstSubsetLoad(adapter, subsetFailure) let source!: Parameters[`sync`]>[0] const collection = createCollection( @@ -959,7 +966,7 @@ export function runSQLiteCoreAdapterContractSuite( collection._sync.loadSubset({ limit: 1 }), ).then(() => undefined) void load.catch(() => undefined) - await loadEntered + await subsetLoad.entered source.begin() source.write( @@ -979,7 +986,7 @@ export function runSQLiteCoreAdapterContractSuite( ) expect(receiptStatus).toBe(`pending`) - rejectLoad() + subsetLoad.release() await expect(load).rejects.toBe(subsetFailure) await receipt const expectedRows = operation === `delete` ? [] : [row] @@ -1020,7 +1027,7 @@ export function runSQLiteCoreAdapterContractSuite( publicError: undefined, }) } finally { - rejectLoad() + subsetLoad.release() await load?.catch(() => undefined) await receipt?.catch(() => undefined) await collection.cleanup() diff --git a/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts b/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts index 3bd4a37862..9745b7fcb5 100644 --- a/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts +++ b/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts @@ -300,17 +300,24 @@ describe(`SQLite resume snapshots`, () => { const initialSnapshotRelease = deferred() releaseInitialSnapshot = initialSnapshotRelease.resolve let snapshotCalls = 0 - adapter.loadResumeSnapshot = async (...args) => { - const snapshot = await loadResumeSnapshot(...args) - snapshotCalls++ - if (snapshotCalls === 1) { - reachedInitialSnapshot.resolve() - await initialSnapshotRelease.promise - } else if (snapshotCalls === 2) { - reachedHydrationSnapshot.resolve() - } - return snapshot - } + const runInHydrationScope = adapter.runInHydrationScope.bind(adapter) + adapter.runInHydrationScope = (task) => + runInHydrationScope((scopedAdapter) => + task({ + ...scopedAdapter, + loadResumeSnapshot: async (...args) => { + const snapshot = await scopedAdapter.loadResumeSnapshot(...args) + snapshotCalls++ + if (snapshotCalls === 1) { + reachedInitialSnapshot.resolve() + await initialSnapshotRelease.promise + } else if (snapshotCalls === 2) { + reachedHydrationSnapshot.resolve() + } + return snapshot + }, + }), + ) collection = createCollection( persistedCollectionOptions<{ id: string; title: string }, string>({ diff --git a/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts b/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts index e39d5a0d63..6bafb1d5b8 100644 --- a/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts +++ b/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts @@ -11,6 +11,7 @@ import { electricCollectionOptions } from '../src/electric' import type { Message, Row } from '@electric-sql/client' import type { Collection } from '@tanstack/db' import type { + HydrationPersistenceAdapter, PersistenceAdapter, SQLiteDriver, } from '../../db-sqlite-persistence-core/src' @@ -137,6 +138,7 @@ async function runRace( | `unknown` | `missing` | `incompatible` = `unchanged`, + forwardHydrationScope = false, ): Promise { const database = new DatabaseSync(`:memory:`) const driver = createDriver(database) @@ -219,53 +221,77 @@ async function runRace( const receiverFailure = new Promise((resolve) => { reportReceiverFailure = resolve }) - const gatedAdapter = new Proxy(restartedAdapter, { - get(target, property) { - if (property === `loadResumeSnapshot`) { - return async function ( - this: PersistenceAdapter, - ...args: Parameters - ) { - if (this !== gatedAdapter) { - const error = new Error( - `Persistence adapter lost its receiver during resume certification`, - ) - reportReceiverFailure(error) - throw error + const gateSnapshotAdapter = ( + adapter: TAdapter, + ): TAdapter => { + const gatedAdapter = new Proxy(adapter, { + get(target, property) { + if (property === `runInHydrationScope`) { + // Exercise both an adapter without optional hydration scopes and + // an adapter that forwards its scoped snapshot reads. + if (!forwardHydrationScope || !target.runInHydrationScope) { + return undefined } - snapshotCalls++ - const isLaterSnapshot = snapshotCalls > 1 - if (isLaterSnapshot) { - laterSnapshotIncludedRows = args[1]?.includeRows - if (startupReset !== `none`) { - resumeStateAtLaterSnapshot = ( - await target.loadCollectionMetadata(collectionId) - ).find(({ key }) => key === `electric:resume`)?.value + return ( + task: (scopedAdapter: HydrationPersistenceAdapter) => Promise, + ): Promise => + target.runInHydrationScope!((scopedAdapter) => + task(gateSnapshotAdapter(scopedAdapter)), + ) + } + if (property === `loadResumeSnapshot`) { + return async function ( + this: PersistenceAdapter, + ...args: Parameters + ) { + if (this !== gatedAdapter) { + const error = new Error( + `Persistence adapter lost its receiver during resume certification`, + ) + reportReceiverFailure(error) + throw error } - laterSnapshotEntered.resolve() - await releaseLaterSnapshot.promise - } - const snapshot = await target.loadResumeSnapshot(...args) - if (isLaterSnapshot && laterKeySetEvidence !== `unchanged`) { - return { - ...snapshot, - keySet: - laterKeySetEvidence === `unknown` - ? { status: `unknown` as const } - : laterKeySetEvidence === `incompatible` - ? { status: `incompatible` as const } - : undefined, + snapshotCalls++ + const isLaterSnapshot = snapshotCalls > 1 + if (isLaterSnapshot) { + laterSnapshotIncludedRows = args[1]?.includeRows + if (startupReset !== `none`) { + if (!target.loadCollectionMetadata) { + throw new Error( + `Expected collection metadata in snapshot scope`, + ) + } + resumeStateAtLaterSnapshot = ( + await target.loadCollectionMetadata(collectionId) + ).find(({ key }) => key === `electric:resume`)?.value + } + laterSnapshotEntered.resolve() + await releaseLaterSnapshot.promise + } + const snapshot = await target.loadResumeSnapshot(...args) + if (isLaterSnapshot && laterKeySetEvidence !== `unchanged`) { + return { + ...snapshot, + keySet: + laterKeySetEvidence === `unknown` + ? { status: `unknown` as const } + : laterKeySetEvidence === `incompatible` + ? { status: `incompatible` as const } + : undefined, + } } + return missingKeySetEvidence + ? { ...snapshot, keySet: undefined } + : snapshot } - return missingKeySetEvidence - ? { ...snapshot, keySet: undefined } - : snapshot } - } - const value = Reflect.get(target, property, target) as unknown - return typeof value === `function` ? value.bind(target) : value - }, - }) as unknown as PersistenceAdapter + const value = Reflect.get(target, property, target) as unknown + return typeof value === `function` ? value.bind(target) : value + }, + }) + return gatedAdapter + } + const gatedAdapter = gateSnapshotAdapter(restartedAdapter) const electricOptions = electricCollectionOptions({ id: collectionId, @@ -357,9 +383,6 @@ async function runRace( } const replacesUncertifiedBaseline = legacyUnknown || missingKeySetEvidence || startupReset !== `none` - if (startupReset !== `none`) { - expect(resumeStateAtLaterSnapshot).toMatchObject({ kind: `reset` }) - } if (metadataWrapper === `shallow-persistence`) { expect(forwardedPersistenceCapability).toBe(receivedPersistenceCapability) expect(getExpectedCommitCallCount()).toBe(1) @@ -535,6 +558,27 @@ async function runRace( expect(laterSnapshotIncludedRows).toBe( replacesUncertifiedBaseline || syncMode !== `on-demand`, ) + if (startupReset !== `none`) { + // The marker write may commit before or after the held hydration read. + // Neither schedule may invent a new resume cursor before the fresh + // source snapshot. The request and final rows are checked above. + if ( + typeof resumeStateAtLaterSnapshot !== `object` || + resumeStateAtLaterSnapshot === null || + !(`kind` in resumeStateAtLaterSnapshot) + ) { + throw new Error(`Expected the prior resume or reset marker`) + } + if (resumeStateAtLaterSnapshot.kind === `reset`) { + expect(resumeStateAtLaterSnapshot).not.toHaveProperty(`offset`) + } else { + expect(resumeStateAtLaterSnapshot).toMatchObject({ + kind: `resume`, + offset: `10_0`, + handle: `shape-old`, + }) + } + } } catch (error) { primaryFailure = error } finally { @@ -819,11 +863,11 @@ describe(`Electric resume snapshot races`, () => { vi.clearAllMocks() }) - it(`keeps a healthy tagged cache when a fresh reset commits before hydration`, async () => { + it(`keeps a healthy tagged cache when startup invalidates its resume cursor`, async () => { await runRace(`none`, `eager`, false, false, `tag-state`) }) - it(`keeps a healthy cache when a changed shape commits its reset before hydration`, async () => { + it(`keeps a healthy cache when a changed shape invalidates its resume cursor`, async () => { await runRace(`none`, `eager`, false, false, `shape-identity`) }) @@ -838,6 +882,26 @@ describe(`Electric resume snapshot races`, () => { ) }) + it.each([ + [`tag-state`, `none`], + [`shape-identity`, `none`], + [`shape-identity`, `shallow-persistence`], + ] as const)( + `keeps a fresh %s reset safe when the %s source wrapper forwards hydration scope`, + async (startupReset, metadataWrapper) => { + await runRace( + `none`, + `eager`, + false, + false, + startupReset, + metadataWrapper, + `unchanged`, + true, + ) + }, + ) + it(`rejects row loss between resume metadata and baseline hydration`, async () => { await runRace(`external-row-loss`) }) diff --git a/packages/electron-db-sqlite-persistence/src/renderer.ts b/packages/electron-db-sqlite-persistence/src/renderer.ts index 12e3c37617..8d974450e6 100644 --- a/packages/electron-db-sqlite-persistence/src/renderer.ts +++ b/packages/electron-db-sqlite-persistence/src/renderer.ts @@ -397,6 +397,9 @@ export function createElectronSQLitePersistence( `sync-absent`, undefined, ) + if (coordinator instanceof ElectronCollectionCoordinator) { + coordinator.setAdapter(defaultPersistence.adapter) + } return { ...defaultPersistence, diff --git a/packages/electron-db-sqlite-persistence/tests/electron-coordinator.test.ts b/packages/electron-db-sqlite-persistence/tests/electron-coordinator.test.ts new file mode 100644 index 0000000000..17bafa1461 --- /dev/null +++ b/packages/electron-db-sqlite-persistence/tests/electron-coordinator.test.ts @@ -0,0 +1,618 @@ +/** + * Browser and Electron coordinators implement the same cross-window protocol. + * The generated Browser law owns the history dimensions; these shrink + * witnesses keep the Electron implementation on the same adapter-routing, + * retry-result, and disposal boundaries. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { ElectronCollectionCoordinator } from '../src/electron-coordinator' +import type { ElectronCollectionCoordinatorOptions } from '../src/electron-coordinator' +import type { PersistenceAdapter } from '@tanstack/db-sqlite-persistence-core' + +type MessageHandler = (event: { data: unknown }) => void +const channels = new Map>() +let dropNextMessageWhen: ((data: unknown) => boolean) | undefined +let observePostedMessage: ((data: unknown) => void) | undefined + +class MockBroadcastChannel { + onmessage: MessageHandler | null = null + + constructor(readonly name: string) { + const peers = channels.get(name) ?? new Set() + peers.add(this) + channels.set(name, peers) + } + + postMessage(data: unknown): void { + observePostedMessage?.(data) + if (dropNextMessageWhen?.(data)) { + dropNextMessageWhen = undefined + return + } + for (const peer of channels.get(this.name) ?? []) { + if (peer !== this && peer.onmessage) { + const handler = peer.onmessage + queueMicrotask(() => handler({ data: structuredClone(data) })) + } + } + } + + close(): void { + channels.get(this.name)?.delete(this) + } +} + +type LockCallback = (lock: { name: string }) => Promise +type QueuedLock = { + callback: LockCallback + signal?: AbortSignal + resolve: (value: unknown) => void + reject: (error: Error) => void +} +const heldLocks = new Set() +const lockQueues = new Map>() + +function grantNextLock(name: string): void { + if (heldLocks.has(name)) return + const next = lockQueues.get(name)?.shift() + if (!next) return + if (next.signal?.aborted) { + grantNextLock(name) + return + } + heldLocks.add(name) + void Promise.resolve(next.callback({ name })).then( + (value) => { + heldLocks.delete(name) + next.resolve(value) + grantNextLock(name) + }, + (error: unknown) => { + heldLocks.delete(name) + next.reject(error instanceof Error ? error : new Error(String(error))) + grantNextLock(name) + }, + ) +} + +const mockLocks = { + request: ( + name: string, + optionsOrCallback: { signal?: AbortSignal } | LockCallback, + maybeCallback?: LockCallback, + ): Promise => { + const callback = + typeof optionsOrCallback === `function` + ? optionsOrCallback + : maybeCallback! + const signal = + typeof optionsOrCallback === `function` + ? undefined + : optionsOrCallback.signal + return new Promise((resolve, reject) => { + const entry = { callback, signal, resolve, reject } + const queue = lockQueues.get(name) ?? [] + queue.push(entry) + lockQueues.set(name, queue) + signal?.addEventListener(`abort`, () => { + const index = queue.indexOf(entry) + if (index >= 0) { + queue.splice(index, 1) + reject(new DOMException(`Lock request aborted`, `AbortError`)) + } + }) + grantNextLock(name) + }) + }, +} + +type StubAdapter = PersistenceAdapter & { + appliedTxs: Array + pullSince: () => Promise<{ + latestRowVersion: number + requiresFullReload: false + changedKeys: Array + deletedKeys: Array + }> + getStreamPosition: () => Promise<{ + latestTerm: number + latestSeq: number + latestRowVersion: number + }> +} + +function createStubAdapter(): StubAdapter { + const appliedTxs: Array = [] + return { + appliedTxs, + loadSubset: async () => [], + loadResumeSnapshot: async () => ({ + rows: [], + collectionMetadata: [], + latestTerm: 0, + latestSeq: 0, + latestRowVersion: 0, + resetEpoch: 0, + }), + applyCommittedTx: async (_collectionId, tx) => { + appliedTxs.push(tx.txId) + }, + ensureIndex: async () => {}, + pullSince: async () => ({ + latestRowVersion: 0, + requiresFullReload: false, + changedKeys: [], + deletedKeys: [], + }), + getStreamPosition: async () => ({ + latestTerm: 0, + latestSeq: 0, + latestRowVersion: 0, + }), + } +} + +function createCoordinator( + adapter: StubAdapter, +): ElectronCollectionCoordinator { + const options: ElectronCollectionCoordinatorOptions = { + dbName: `electron-coordinator-law`, + adapter, + } + return new ElectronCollectionCoordinator(options) +} + +async function waitForLeadership( + coordinator: ElectronCollectionCoordinator, + collectionId: string, +): Promise { + for (let microtask = 0; microtask < 12; microtask++) { + if (coordinator.isLeader(collectionId)) return + await Promise.resolve() + } + expect(coordinator.isLeader(collectionId)).toBe(true) +} + +type CoordinatorInspection = { + collectionAdapters: Map + collections: Map + appliedEnvelopes: Map +} + +function inspectCoordinator( + coordinator: ElectronCollectionCoordinator, +): CoordinatorInspection { + return coordinator as unknown as CoordinatorInspection +} + +describe(`ElectronCollectionCoordinator parity`, () => { + beforeEach(() => { + ;(globalThis as Record).BroadcastChannel = + MockBroadcastChannel as unknown + Object.defineProperty(globalThis, `navigator`, { + value: { locks: mockLocks }, + configurable: true, + }) + }) + + afterEach(() => { + dropNextMessageWhen = undefined + observePostedMessage = undefined + channels.clear() + heldLocks.clear() + lockQueues.clear() + vi.useRealTimers() + }) + + it(`keeps follower RPC work on the collection adapter`, async () => { + const todosAdapter = createStubAdapter() + const notesAdapter = createStubAdapter() + let leadershipRead!: () => void + const leadershipReadPromise = new Promise((resolve) => { + leadershipRead = resolve + }) + todosAdapter.getStreamPosition = async () => { + leadershipRead() + return { latestTerm: 0, latestSeq: 0, latestRowVersion: 0 } + } + todosAdapter.ensureIndex = vi.fn().mockResolvedValue(undefined) + notesAdapter.ensureIndex = vi.fn().mockResolvedValue(undefined) + + const leader = createCoordinator(todosAdapter) + const follower = createCoordinator(notesAdapter) + try { + leader.setAdapterForCollection(`todos`, todosAdapter) + leader.subscribe(`todos`, () => {}) + await leadershipReadPromise + await waitForLeadership(leader, `todos`) + follower.subscribe(`todos`, () => {}) + leader.setAdapter(notesAdapter) + + const postedTypes: Array = [] + observePostedMessage = (data) => { + postedTypes.push( + (data as { payload?: { type?: string } }).payload?.type, + ) + } + const request = follower.requestEnsurePersistedIndex(`todos`, `idx`, { + expressionSql: [`title`], + }) + for (let microtask = 0; microtask < 12; microtask++) { + await Promise.resolve() + } + expect(postedTypes).toContain(`rpc:ensurePersistedIndex:res`) + await request + + expect(todosAdapter.ensureIndex).toHaveBeenCalledOnce() + expect(notesAdapter.ensureIndex).not.toHaveBeenCalled() + } finally { + follower.dispose() + leader.dispose() + } + }) + + it(`does not repeat successful leader-local index creation`, async () => { + const adapter = createStubAdapter() + let leadershipRead!: () => void + const leadershipReadPromise = new Promise((resolve) => { + leadershipRead = resolve + }) + adapter.getStreamPosition = async () => { + leadershipRead() + return { latestTerm: 0, latestSeq: 0, latestRowVersion: 0 } + } + adapter.ensureIndex = vi.fn().mockResolvedValue(undefined) + + const coordinator = createCoordinator(adapter) + try { + coordinator.subscribe(`todos`, () => {}) + await leadershipReadPromise + await waitForLeadership(coordinator, `todos`) + const spec = { expressionSql: [`title`] } + + await adapter.ensureIndex(`todos`, `idx-once`, spec) + await coordinator.requestEnsurePersistedIndex( + `todos`, + `idx-once`, + spec, + adapter, + true, + ) + + expect(adapter.ensureIndex).toHaveBeenCalledOnce() + } finally { + coordinator.dispose() + } + }) + + it(`replays a successful mutation result after response loss`, async () => { + vi.useFakeTimers() + const adapter = createStubAdapter() + let leadershipRead!: () => void + const leadershipReadPromise = new Promise((resolve) => { + leadershipRead = resolve + }) + let firstSuccessDropped!: () => void + const firstSuccessDroppedPromise = new Promise((resolve) => { + firstSuccessDropped = resolve + }) + adapter.getStreamPosition = async () => { + leadershipRead() + return { latestTerm: 0, latestSeq: 0, latestRowVersion: 0 } + } + + const leader = createCoordinator(adapter) + const follower = createCoordinator(adapter) + try { + leader.subscribe(`todos`, () => {}) + await leadershipReadPromise + await waitForLeadership(leader, `todos`) + follower.subscribe(`todos`, () => {}) + dropNextMessageWhen = (data) => { + const payload = (data as { payload?: { type?: string; ok?: boolean } }) + .payload + if ( + payload?.type === `rpc:applyLocalMutations:res` && + payload.ok === true + ) { + firstSuccessDropped() + return true + } + return false + } + + const responsePromise = follower.requestApplyLocalMutations(`todos`, [ + { + mutationId: `mut-electron`, + type: `insert`, + key: `1`, + value: { id: `1` }, + }, + ]) + await firstSuccessDroppedPromise + await vi.advanceTimersByTimeAsync(10_200) + + await expect(responsePromise).resolves.toMatchObject({ + ok: true, + acceptedMutationIds: [`mut-electron`], + }) + expect(adapter.appliedTxs).toHaveLength(1) + } finally { + follower.dispose() + leader.dispose() + } + }) + + it(`scopes envelope deduplication by collection`, async () => { + const alphaAdapter = createStubAdapter() + const betaAdapter = createStubAdapter() + const coordinator = createCoordinator(alphaAdapter) + coordinator.setAdapterForCollection(`alpha`, alphaAdapter) + coordinator.setAdapterForCollection(`beta`, betaAdapter) + coordinator.subscribe(`alpha`, () => {}) + coordinator.subscribe(`beta`, () => {}) + + const internals = coordinator as unknown as { + handleApplyLocalMutations: ( + collectionId: string, + request: { + type: `rpc:applyLocalMutations:req` + rpcId: string + envelopeId: string + mutations: Array<{ + mutationId: string + type: `insert` + key: string + value: { id: string } + }> + }, + ) => Promise<{ ok: boolean; rpcId: string }> + } + + try { + await Promise.all([ + waitForLeadership(coordinator, `alpha`), + waitForLeadership(coordinator, `beta`), + ]) + const alpha = await internals.handleApplyLocalMutations(`alpha`, { + type: `rpc:applyLocalMutations:req`, + rpcId: `alpha-rpc`, + envelopeId: `shared-envelope`, + mutations: [ + { + mutationId: `alpha-mutation`, + type: `insert`, + key: `alpha`, + value: { id: `alpha` }, + }, + ], + }) + const beta = await internals.handleApplyLocalMutations(`beta`, { + type: `rpc:applyLocalMutations:req`, + rpcId: `beta-rpc`, + envelopeId: `shared-envelope`, + mutations: [ + { + mutationId: `beta-mutation`, + type: `insert`, + key: `beta`, + value: { id: `beta` }, + }, + ], + }) + + expect({ + alpha, + alphaApplies: alphaAdapter.appliedTxs.length, + beta, + betaApplies: betaAdapter.appliedTxs.length, + }).toMatchObject({ + alpha: { ok: true, rpcId: `alpha-rpc` }, + alphaApplies: 1, + beta: { ok: true, rpcId: `beta-rpc` }, + betaApplies: 1, + }) + } finally { + coordinator.dispose() + } + }) + + it(`coalesces an envelope retry while its first write is in flight`, async () => { + const adapter = createStubAdapter() + let enterFirstApply!: () => void + const firstApplyEntered = new Promise((resolve) => { + enterFirstApply = resolve + }) + let releaseFirstApply!: () => void + const firstApplyRelease = new Promise((resolve) => { + releaseFirstApply = resolve + }) + let applyCalls = 0 + adapter.applyCommittedTx = async (_collectionId, tx) => { + applyCalls++ + if (applyCalls === 1) { + enterFirstApply() + await firstApplyRelease + } + adapter.appliedTxs.push(tx.txId) + } + const coordinator = createCoordinator(adapter) + coordinator.subscribe(`todos`, () => {}) + + const internals = coordinator as unknown as { + handleApplyLocalMutations: ( + collectionId: string, + request: { + type: `rpc:applyLocalMutations:req` + rpcId: string + envelopeId: string + mutations: Array<{ + mutationId: string + type: `insert` + key: string + value: { id: string } + }> + }, + ) => Promise<{ + ok: boolean + rpcId: string + term?: number + seq?: number + latestRowVersion?: number + }> + } + const request = { + type: `rpc:applyLocalMutations:req` as const, + envelopeId: `in-flight-envelope`, + mutations: [ + { + mutationId: `mutation`, + type: `insert` as const, + key: `row`, + value: { id: `row` }, + }, + ], + } + + try { + await waitForLeadership(coordinator, `todos`) + const first = internals.handleApplyLocalMutations(`todos`, { + ...request, + rpcId: `first-rpc`, + }) + await firstApplyEntered + const retry = internals.handleApplyLocalMutations(`todos`, { + ...request, + rpcId: `retry-rpc`, + }) + await Promise.resolve() + releaseFirstApply() + + const [firstResponse, retryResponse] = await Promise.all([first, retry]) + expect({ + applyCalls, + first: { ...firstResponse, rpcId: undefined }, + retry: { ...retryResponse, rpcId: undefined }, + }).toEqual({ + applyCalls: 1, + first: { ...retryResponse, rpcId: undefined }, + retry: { ...firstResponse, rpcId: undefined }, + }) + } finally { + releaseFirstApply() + coordinator.dispose() + } + }) + + it(`releases collection-owned state and retry results with the last subscriber`, async () => { + const adapter = createStubAdapter() + const coordinator = createCoordinator(adapter) + const release = coordinator.subscribe(`todos`, () => {}) + try { + await waitForLeadership(coordinator, `todos`) + await coordinator.requestApplyLocalMutations(`todos`, [ + { + mutationId: `mut-release`, + type: `insert`, + key: `1`, + value: { id: `1` }, + }, + ]) + expect({ + adapters: inspectCoordinator(coordinator).collectionAdapters.size, + collections: inspectCoordinator(coordinator).collections.size, + envelopes: inspectCoordinator(coordinator).appliedEnvelopes.size, + }).toEqual({ adapters: 1, collections: 1, envelopes: 1 }) + + release() + expect({ + adapters: inspectCoordinator(coordinator).collectionAdapters.size, + collections: inspectCoordinator(coordinator).collections.size, + envelopes: inspectCoordinator(coordinator).appliedEnvelopes.size, + }).toEqual({ adapters: 0, collections: 0, envelopes: 0 }) + } finally { + release() + coordinator.dispose() + } + }) + + it(`expires retry results without later mutation traffic`, async () => { + vi.useFakeTimers() + vi.setSystemTime(0) + const adapter = createStubAdapter() + const coordinator = createCoordinator(adapter) + const release = coordinator.subscribe(`todos`, () => {}) + try { + await waitForLeadership(coordinator, `todos`) + await coordinator.requestApplyLocalMutations(`todos`, [ + { + mutationId: `mut-expire`, + type: `insert`, + key: `1`, + value: { id: `1` }, + }, + ]) + expect(inspectCoordinator(coordinator).appliedEnvelopes.size).toBe(1) + + await vi.advanceTimersByTimeAsync(60_000) + expect(inspectCoordinator(coordinator).appliedEnvelopes.size).toBe(0) + } finally { + release() + coordinator.dispose() + vi.useRealTimers() + } + }) + + it(`settles pending work at dispose without retrying`, async () => { + vi.useFakeTimers() + const adapter = createStubAdapter() + let leadershipRead!: () => void + const leadershipReadPromise = new Promise((resolve) => { + leadershipRead = resolve + }) + let firstRequestPosted!: () => void + const firstRequestPostedPromise = new Promise((resolve) => { + firstRequestPosted = resolve + }) + adapter.getStreamPosition = async () => { + leadershipRead() + return { latestTerm: 0, latestSeq: 0, latestRowVersion: 0 } + } + + const leader = createCoordinator(adapter) + const follower = createCoordinator(adapter) + let requestPosts = 0 + try { + leader.subscribe(`todos`, () => {}) + await leadershipReadPromise + await waitForLeadership(leader, `todos`) + follower.subscribe(`todos`, () => {}) + observePostedMessage = (data) => { + if ( + (data as { payload?: { type?: string } }).payload?.type === + `rpc:pullSince:req` + ) { + requestPosts++ + firstRequestPosted() + } + } + dropNextMessageWhen = (data) => + (data as { payload?: { type?: string } }).payload?.type === + `rpc:pullSince:req` + + const settled = follower.pullSince(`todos`, 0).then( + () => `resolved`, + () => `rejected`, + ) + await firstRequestPostedPromise + follower.dispose() + + await expect(settled).resolves.toBe(`rejected`) + await vi.advanceTimersByTimeAsync(21_000) + expect(requestPosts).toBe(1) + } finally { + follower.dispose() + leader.dispose() + } + }) +}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 328d4fe5ad..450ddd4bfe 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1072,6 +1072,9 @@ importers: '@journeyapps/wa-sqlite': specifier: ^1.4.1 version: 1.5.0 + '@playwright/test': + specifier: ^1.60.0 + version: 1.60.0 '@types/better-sqlite3': specifier: ^7.6.13 version: 7.6.13