From f92d1c9803ef4833cdcdb715a8f4718cebb68e7a Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 20 Sep 2026 23:35:52 +0100 Subject: [PATCH 1/8] fix(sqlite-persistence): preserve expression index use --- .../fix-sqlite-expression-index-planning.md | 5 + docs/contributing/oracle-coverage.md | 2 + .../src/sqlite-core-adapter.ts | 15 +- .../node-db-sqlite-persistence/package.json | 1 + .../tests/expression-index-oracle.test.ts | 524 ++++++++++++++++++ 5 files changed, 542 insertions(+), 5 deletions(-) create mode 100644 .changeset/fix-sqlite-expression-index-planning.md create mode 100644 packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts diff --git a/.changeset/fix-sqlite-expression-index-planning.md b/.changeset/fix-sqlite-expression-index-planning.md new file mode 100644 index 0000000000..7256a2821a --- /dev/null +++ b/.changeset/fix-sqlite-expression-index-planning.md @@ -0,0 +1,5 @@ +--- +'@tanstack/db-sqlite-persistence-core': patch +--- + +Use canonical SQL literals for serialized ref JSON paths so runtime predicates match persisted expression indexes and SQLite can use them. diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index cd17a6c50f..ea1c9b8438 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -55,6 +55,7 @@ comment and the current API/architecture contract before extending its model. | Electric and TrailBase | [Electric histories](../../packages/electric-db-collection/tests/electric-oracle.property.test.ts), [PostgreSQL semantics](../../packages/electric-db-collection/e2e/sql-predicate-semantics.e2e.test.ts), [TrailBase contract](../../packages/trailbase-db-collection/tests/ORACLE.md) | Installed SDK delivery/framing, independent predicates, exact subscription arguments and late errors. SDK fixtures and a real service test earn different credit. | | PowerSync | [tests](../../packages/powersync-db-collection/tests), `tests/correctness-oracle.test.ts` | Applied receipt positions crossed with held peers, native SQLite/SDK and cleanup evidence. Run the focused owner with the package's `test:oracles` command. A timeout mutant proves a progress failure, not every value assertion. | | SQLite persistence and native hosts | [persisted histories](../../packages/db-sqlite-persistence-core/tests/persisted.test.ts), [driver contracts](../../packages/db-sqlite-persistence-core/tests/contracts/sqlite-driver-contract.ts), [113-law manifest](../../packages/db-collection-e2e/src/fixtures/persisted-conformance-manifest.ts) | Cache/remote rejection/peer/reopen histories and exact driver results. The manifest excludes progressive and move suites; registration and shim runs are not device execution. | +| SQLite expression-index planning | [Node expression-index oracle](../../packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts) | RFC #1659 invariant 8 owns matching persisted-index and runtime-predicate JSON expression shapes. Generated paths use 1-6 character identifiers starting with `a/m/p/t/x` and continuing with `a/b/e/i/n/r/s/0/1`, plus up to three identifier or `0..3` index tails. Values are integer targets `-10_000..10_000` with adjacent distractors, `target-'${suffix}` strings with `before-${suffix}`/`after-${suffix}` distractors for suffixes `0..10_000`, and booleans with duplicated opposite distractors. `_root`/`Upper_2`, fractional-number, and empty-string examples are omitted from generation but same-path audit probes are GREEN; no permanent matrix is claimed. An independent full scan judges matching keys while captured real-driver SQL and bindings are replayed through `EXPLAIN QUERY PLAN` for a named-index search. Run the package's `test:oracles` campaign. Arbitrary raw SQL, result ordering, date/bigint/null/tagged-value coercions, and native-host execution are outside this law. | | Offline execution | [scheduler](../../packages/offline-transactions/tests/KeyScheduler.property.test.ts), [leadership](../../packages/offline-transactions/tests/leadership-replay.property.test.ts), [settlement](../../packages/offline-transactions/tests/transaction-settlement.property.test.ts), [serialization](../../packages/offline-transactions/tests/transaction-serializer.property.test.ts) | Declarative FIFO eligibility, per-transaction outcomes, durable state and typed wire trees. Issued work may finish after ownership loss, but new work must not start. Exactly-once network execution is not promised. | | Frameworks | [React conformance](../../packages/react-db/tests/conformance.test.tsx), [React pagination](../../packages/react-db/tests/infinite-query-conformance.test.tsx), [shared suites](../../packages/db-collection-e2e/src/suites) | Exact exposed rows/pages and each framework's own lifecycle cuts. A React witness does not prove Vue/Solid/Angular/Svelte scheduling. Preserve their receiving registrations. | | Small structures and test mechanics | [SortedMap](../../packages/db/tests/SortedMap.test.ts), [cleanup queue](../../packages/db/tests/cleanup-queue.property.test.ts), [guarded replay](../../packages/db/tests/oracle-replay.test.ts) | Map/full-sort and appointment-list models; executed target/seed/path checks. Callback-reentrant scheduling is outside the initial cleanup-queue domain. | @@ -97,6 +98,7 @@ pnpm --filter @tanstack/db-ivm build pnpm --filter @tanstack/db build pnpm --filter @tanstack/db test:oracles pnpm --filter @tanstack/powersync-db-collection test:oracles +pnpm --filter @tanstack/node-db-sqlite-persistence test:oracles # Service-dependent: requires Electric and PostgreSQL to be running. pnpm --filter @tanstack/electric-db-collection test:e2e pnpm run typecheck:tests diff --git a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts index 69f29fc604..6fd5f57dc8 100644 --- a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts +++ b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts @@ -594,18 +594,23 @@ function compileComparisonSql( function compileRefExpressionSql(jsonPath: string): CompiledSqlFragment { const typePath = `${jsonPath}.${PERSISTED_TYPE_TAG}` const taggedValuePath = `${jsonPath}.${PERSISTED_VALUE_TAG}` + // createJsonPath has already validated every segment. Keep these paths as + // canonical SQL literals so runtime refs match persisted expression indexes. + const typePathSql = toSqliteLiteral(typePath) + const taggedValuePathSql = toSqliteLiteral(taggedValuePath) + const jsonPathSql = toSqliteLiteral(jsonPath) return { supported: true, - sql: `(CASE json_extract(value, ?) - WHEN 'bigint' THEN CAST(json_extract(value, ?) AS NUMERIC) - WHEN 'date' THEN json_extract(value, ?) + sql: `(CASE json_extract(value, ${typePathSql}) + WHEN 'bigint' THEN CAST(json_extract(value, ${taggedValuePathSql}) AS NUMERIC) + WHEN 'date' THEN json_extract(value, ${taggedValuePathSql}) WHEN 'nan' THEN NULL WHEN 'infinity' THEN NULL WHEN '-infinity' THEN NULL - ELSE json_extract(value, ?) + ELSE json_extract(value, ${jsonPathSql}) END)`, - params: [typePath, taggedValuePath, taggedValuePath, jsonPath], + params: [], valueKind: `unknown`, } } diff --git a/packages/node-db-sqlite-persistence/package.json b/packages/node-db-sqlite-persistence/package.json index 83c4fddc1e..13b45d987f 100644 --- a/packages/node-db-sqlite-persistence/package.json +++ b/packages/node-db-sqlite-persistence/package.json @@ -22,6 +22,7 @@ "dev": "vite build --watch", "lint": "eslint . --fix", "test": "vitest --run", + "test:oracles": "vitest --run tests/expression-index-oracle.test.ts", "test:e2e": "pnpm --filter @tanstack/db-ivm build && pnpm --filter @tanstack/db build && pnpm --filter @tanstack/db-sqlite-persistence-core build && pnpm --filter @tanstack/node-db-sqlite-persistence build && vitest --config vitest.e2e.config.ts --run" }, "type": "module", diff --git a/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts b/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts new file mode 100644 index 0000000000..2f0ab9b4af --- /dev/null +++ b/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts @@ -0,0 +1,524 @@ +/** + * Law and source: RFC #1659 invariant 8 requires persisted index DDL and the + * indexed runtime predicate to have the same SQLite expression shape. SQLite + * requires syntactically matching expressions before an expression index can + * satisfy a predicate. + * + * Domain: constructively generated object-rooted SQLite JSON paths have a + * one-to-six-character identifier root and up to three identifier/array-index + * tail segments. Identifiers start with a/m/p/t/x and continue with + * a/b/e/i/n/r/s/0/1; array indices are 0..3. Independent equality values are + * integer targets -10_000..10_000 with adjacent distractors, + * `target-'${suffix}` strings with `before-${suffix}`/`after-${suffix}` + * distractors for suffixes 0..10_000, or booleans with duplicated opposite + * distractors. Legal `_root`/`Upper_2` paths, fractional numbers, and empty + * strings are omitted from the generated campaign; their same-path audit probes + * are GREEN, so they are not a permanent matrix. Null and persisted tagged + * values remain outside this query-planning law because their operators or + * coercions can change the indexed expression. Each generated history inserts + * rows, creates the serialized ref index, scans, and loads the subset. + * + * Reference: a full adapter scan followed by a small path walker and strict + * scalar equality. It does not call the SQL compiler or reuse its path logic. + * + * Production path and checkpoint: the public SQLite-core adapter factory with + * the real BetterSqlite3SQLiteDriver. The exact SQL and bindings passed to the + * driver's predicate query are captured, then replayed through EXPLAIN QUERY + * PLAN before cleanup. Result keys and the named expression index in the plan + * are separate observations; result order is outside this law. + * + * Reach, challenge, replay, and cleanup: every production case proves the + * index exists, the full-scan row count and target classification match the + * seed, the filter value stays bound, and indexed results equal the independent + * scan classification. Direct SQLite fault controls show that binding the DDL + * path is rejected and binding the predicate path returns the same rows but + * loses the index search. Replay a generated failure with TANSTACK_DB_WS5A_SEED + * and TANSTACK_DB_WS5A_PATH. Teardown retains the semantic failure as + * AggregateError.cause if cleanup also fails. + */ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import fc from 'fast-check' +import { describe, expect, it } from 'vitest' +import { IR } from '@tanstack/db' +import { + createPersistedTableName, + createSQLiteCorePersistenceAdapter, +} from '@tanstack/db-sqlite-persistence-core' +import { BetterSqlite3SQLiteDriver } from '../src/node-driver' +import type { SQLiteDriver } from '@tanstack/db-sqlite-persistence-core' + +const DEFAULT_ORACLE_SEED = 1_659_005 +const DEFAULT_ORACLE_RUNS = 24 + +type OracleScalar = boolean | number | string + +type OracleCase = { + path: Array + target: OracleScalar + distractors: [OracleScalar, OracleScalar] +} + +type CapturedQuery = { + sql: string + params: ReadonlyArray +} + +type QueryPlanRow = { + detail: string +} + +type ScanRow = { + key: string | number + value: Record +} + +const identifierStart = [`a`, `m`, `p`, `t`, `x`] as const +const identifierRest = [`a`, `b`, `e`, `i`, `n`, `r`, `s`, `0`, `1`] as const + +const identifierArbitrary = fc + .tuple( + fc.constantFrom(...identifierStart), + fc.array(fc.constantFrom(...identifierRest), { maxLength: 5 }), + ) + .map(([start, rest]) => `${start}${rest.join(``)}`) + +const legalPathArbitrary = fc + .tuple( + identifierArbitrary, + fc.array( + fc.oneof(identifierArbitrary, fc.integer({ min: 0, max: 3 }).map(String)), + { maxLength: 3 }, + ), + ) + .map(([root, tail]) => [root, ...tail]) + +const independentValuesArbitrary = fc.oneof( + fc.integer({ min: -10_000, max: 10_000 }).map((target) => ({ + target, + distractors: [target - 1, target + 1] as [number, number], + })), + fc.integer({ min: 0, max: 10_000 }).map((suffix) => ({ + target: `target-'${suffix}`, + distractors: [`before-${suffix}`, `after-${suffix}`] as [string, string], + })), + fc.boolean().map((target) => ({ + target, + distractors: [!target, !target] as [boolean, boolean], + })), +) + +// Path structure and scalar values come from separate arbitraries so neither +// can derive or restrict the other. +const expressionIndexCaseArbitrary = fc + .tuple(legalPathArbitrary, independentValuesArbitrary) + .map(([path, values]): OracleCase => ({ path, ...values })) + +function oracleRunConfiguration(): { + seed: number + path?: string + numRuns: number +} { + const seedText = process.env.TANSTACK_DB_WS5A_SEED + const path = process.env.TANSTACK_DB_WS5A_PATH + const seed = seedText === undefined ? DEFAULT_ORACLE_SEED : Number(seedText) + + if (!Number.isSafeInteger(seed)) { + throw new Error(`TANSTACK_DB_WS5A_SEED must be an integer`) + } + if (path !== undefined && !/^\d+(?::\d+)*$/.test(path)) { + throw new Error( + `TANSTACK_DB_WS5A_PATH must contain colon-separated nonnegative integers`, + ) + } + + return { + seed, + ...(path === undefined ? {} : { path }), + numRuns: path === undefined ? DEFAULT_ORACLE_RUNS : 1, + } +} + +function createNestedRow( + path: ReadonlyArray, + value: OracleScalar, +): Record { + let nested: unknown = value + + for (let index = path.length - 1; index >= 0; index--) { + const segment = path[index]! + if (/^\d+$/.test(segment)) { + const entries = Array.from({ length: Number(segment) + 1 }) + entries[Number(segment)] = nested + nested = entries + } else { + nested = { [segment]: nested } + } + } + + if (typeof nested !== `object` || nested === null || Array.isArray(nested)) { + throw new Error(`generated path must be rooted in an object`) + } + return nested as Record +} + +function readPath( + row: Record, + path: ReadonlyArray, +): unknown { + let value: unknown = row + + for (const segment of path) { + if (Array.isArray(value) && /^\d+$/.test(segment)) { + value = value[Number(segment)] + continue + } + if (typeof value !== `object` || value === null || Array.isArray(value)) { + return undefined + } + value = (value as Record)[segment] + } + + return value +} + +function expectedKeysFromScan( + rows: ReadonlyArray, + path: ReadonlyArray, + target: OracleScalar, +): Array { + return rows + .filter((row) => readPath(row.value, path) === target) + .map((row) => String(row.key)) + .sort() +} + +function sqliteJsonPath(path: ReadonlyArray): string { + return path.reduce((result, segment) => { + return /^\d+$/.test(segment) + ? `${result}[${segment}]` + : `${result}.${segment}` + }, `$`) +} + +function sqliteLiteral(value: string): string { + return `'${value.replace(/'/g, `''`)}'` +} + +function sqliteScalarParameter(value: OracleScalar): number | string { + return typeof value === `boolean` ? (value ? 1 : 0) : value +} + +function createQueryObservingDriver( + inner: SQLiteDriver, + observe: (query: CapturedQuery) => void, +): SQLiteDriver { + const wrap = (driver: SQLiteDriver): SQLiteDriver => ({ + exec: (sql) => driver.exec(sql), + query: async (sql: string, params: ReadonlyArray = []) => { + observe({ sql, params: [...params] }) + return driver.query(sql, params) + }, + run: (sql, params) => driver.run(sql, params), + transaction: (body) => + driver.transaction((transactionDriver) => body(wrap(transactionDriver))), + transactionWithDriver: driver.transactionWithDriver + ? (body) => + driver.transactionWithDriver!((transactionDriver) => + body(wrap(transactionDriver)), + ) + : undefined, + }) + + return wrap(inner) +} + +async function withFailurePreservingCleanup( + body: () => void | Promise, + cleanups: ReadonlyArray<() => void | Promise>, +): Promise { + let primary: { error: unknown } | undefined + try { + await body() + } catch (error) { + primary = { error } + } + + const cleanupFailures: Array = [] + for (const cleanup of cleanups) { + try { + await cleanup() + } catch (error) { + cleanupFailures.push(error) + } + } + + if (cleanupFailures.length > 0) { + if (primary) { + throw new AggregateError( + [primary.error, ...cleanupFailures], + `Expression-index oracle and cleanup failed`, + { cause: primary.error }, + ) + } + if (cleanupFailures.length === 1) throw cleanupFailures[0] + throw new AggregateError( + cleanupFailures, + `Expression-index oracle cleanup failed`, + ) + } + if (primary) throw primary.error +} + +function planUsesNamedIndex( + plan: ReadonlyArray, + tableName: string, + indexName: string, +): boolean { + return plan.some( + ({ detail }) => + detail.includes(`SEARCH ${tableName}`) && + detail.includes(`USING INDEX ${indexName}`), + ) +} + +function planScansTable( + plan: ReadonlyArray, + tableName: string, +): boolean { + return plan.some(({ detail }) => detail.startsWith(`SCAN ${tableName}`)) +} + +async function assertExpressionIndexHistory( + testCase: OracleCase, + label: string, +): Promise { + const tempDirectory = mkdtempSync(join(tmpdir(), `db-expression-index-`)) + const databasePath = join(tempDirectory, `state.sqlite`) + const baseDriver = new BetterSqlite3SQLiteDriver({ filename: databasePath }) + const collectionId = `expression-index-${label}` + const signature = `generated-ref` + const tableName = createPersistedTableName(collectionId, `c`) + let predicateQuery: CapturedQuery | undefined + + const observingDriver = createQueryObservingDriver(baseDriver, (query) => { + if ( + query.sql.includes(`FROM "${tableName}"`) && + query.sql.includes(` WHERE `) + ) { + predicateQuery = query + } + }) + const adapter = createSQLiteCorePersistenceAdapter({ + driver: observingDriver, + }) + const values = [ + testCase.target, + testCase.distractors[0], + testCase.target, + testCase.distractors[1], + testCase.distractors[0], + testCase.target, + ] + const seededRows = values.map((value, index) => ({ + key: `row-${index}`, + value: createNestedRow(testCase.path, value), + })) + + await withFailurePreservingCleanup(async () => { + await adapter.applyCommittedTx(collectionId, { + txId: `seed-${label}`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: seededRows.map((row) => ({ + type: `insert` as const, + key: row.key, + value: row.value, + })), + }) + + await adapter.ensureIndex(collectionId, signature, { + expressionSql: [JSON.stringify({ type: `ref`, path: testCase.path })], + }) + + if (!adapter.scanRows) { + throw new Error(`real SQLite adapter did not expose scanRows`) + } + const scannedRows = await adapter.scanRows(collectionId) + const expectedKeys = expectedKeysFromScan( + scannedRows, + testCase.path, + testCase.target, + ) + const independentlySeededExpectedKeys = expectedKeysFromScan( + seededRows, + testCase.path, + testCase.target, + ) + + expect( + expectedKeys, + `full-scan checkpoint must retain independently seeded values`, + ).toEqual(independentlySeededExpectedKeys) + expect(scannedRows).toHaveLength(seededRows.length) + + const indexedRows = await adapter.loadSubset(collectionId, { + where: new IR.Func(`eq`, [ + new IR.PropRef(testCase.path), + new IR.Value(testCase.target), + ]), + }) + const indexedKeys = indexedRows.map((row) => String(row.key)).sort() + + expect( + indexedKeys, + `indexed adapter results must equal the independent scan model`, + ).toEqual(expectedKeys) + + if (!predicateQuery) { + throw new Error(`predicate query checkpoint was not reached`) + } + expect( + predicateQuery.params, + `the filter value must remain a bound parameter`, + ).toContain(sqliteScalarParameter(testCase.target)) + + const registryRow = baseDriver + .getDatabase() + .prepare( + `SELECT index_name FROM persisted_index_registry + WHERE collection_id = ? AND signature = ?`, + ) + .get(collectionId, signature) as { index_name: string } | undefined + if (!registryRow) { + throw new Error(`expression index registry checkpoint was not reached`) + } + + const indexDefinition = baseDriver + .getDatabase() + .prepare( + `SELECT sql FROM sqlite_master WHERE type = 'index' AND name = ?`, + ) + .get(registryRow.index_name) as { sql: string } | undefined + if (!indexDefinition) { + throw new Error(`expression index DDL checkpoint was not reached`) + } + + const plan = baseDriver + .getDatabase() + .prepare(`EXPLAIN QUERY PLAN ${predicateQuery.sql}`) + .all(...predicateQuery.params) as Array + const checkpoint = { + name: `predicate-explain`, + path: sqliteJsonPath(testCase.path), + expectedKeys, + indexedKeys, + subsetSql: predicateQuery.sql, + subsetParams: predicateQuery.params, + indexName: registryRow.index_name, + indexSql: indexDefinition.sql, + plan: plan.map((row) => row.detail), + usedNamedExpressionIndex: planUsesNamedIndex( + plan, + tableName, + registryRow.index_name, + ), + scannedCollectionTable: planScansTable(plan, tableName), + } + + expect( + checkpoint.usedNamedExpressionIndex, + `predicate checkpoint must use the matching expression index:\n${JSON.stringify(checkpoint, null, 2)}`, + ).toBe(true) + expect( + checkpoint.scannedCollectionTable, + `predicate checkpoint must not scan the collection table:\n${JSON.stringify(checkpoint, null, 2)}`, + ).toBe(false) + }, [ + () => baseDriver.close(), + () => rmSync(tempDirectory, { recursive: true, force: true }), + ]) +} + +describe(`SQLite expression-index oracle`, () => { + it(`distinguishes rejected DDL path binding from correct predicate rows without index use`, async () => { + const tempDirectory = mkdtempSync(join(tmpdir(), `db-index-controls-`)) + const databasePath = join(tempDirectory, `state.sqlite`) + const driver = new BetterSqlite3SQLiteDriver({ filename: databasePath }) + const database = driver.getDatabase() + const jsonPath = `$.payload.threadId` + const target = `thread-'1` + + await withFailurePreservingCleanup(() => { + database.exec( + `CREATE TABLE rows (key TEXT PRIMARY KEY, value TEXT NOT NULL)`, + ) + const insert = database.prepare( + `INSERT INTO rows (key, value) VALUES (?, ?)`, + ) + insert.run(`matching`, JSON.stringify({ payload: { threadId: target } })) + insert.run( + `different`, + JSON.stringify({ payload: { threadId: `thread-2` } }), + ) + + // SQLite rejects a DDL-path binding before an index can exist. This is + // a setup-rejection control only, not the production oracle's RED. + expect(() => + database + .prepare(`CREATE INDEX bound_ddl ON rows (json_extract(value, ?))`) + .run(jsonPath), + ).toThrow(/parameters prohibited in index expressions/i) + + database.exec( + `CREATE INDEX literal_ddl ON rows (json_extract(value, ${sqliteLiteral(jsonPath)}))`, + ) + const literalPredicate = `SELECT key FROM rows WHERE json_extract(value, ${sqliteLiteral(jsonPath)}) = ?` + const boundPredicate = `SELECT key FROM rows WHERE json_extract(value, ?) = ?` + const literalRows = database.prepare(literalPredicate).all(target) + const boundRows = database.prepare(boundPredicate).all(jsonPath, target) + const literalPlan = database + .prepare(`EXPLAIN QUERY PLAN ${literalPredicate}`) + .all(target) as Array + const boundPlan = database + .prepare(`EXPLAIN QUERY PLAN ${boundPredicate}`) + .all(jsonPath, target) as Array + + // Same correct rows, different plan: a row-only assertion is + // false-green. This predicate-plan mismatch kills the hostile mutant. + expect(boundRows).toEqual(literalRows) + expect(planUsesNamedIndex(literalPlan, `rows`, `literal_ddl`)).toBe(true) + expect(planScansTable(literalPlan, `rows`)).toBe(false) + expect(planUsesNamedIndex(boundPlan, `rows`, `literal_ddl`)).toBe(false) + expect(planScansTable(boundPlan, `rows`)).toBe(true) + }, [ + () => driver.close(), + () => rmSync(tempDirectory, { recursive: true, force: true }), + ]) + }) + + it(`uses the expression index for the fixed filter witness`, async () => { + await assertExpressionIndexHistory( + { + path: [`threadId`], + target: `thread-'1`, + distractors: [`thread-2`, `thread-3`], + }, + `fixed-thread-filter`, + ) + }) + + it(`uses matching expression indexes for generated legal paths and values`, async () => { + await fc.assert( + fc.asyncProperty(expressionIndexCaseArbitrary, async (testCase) => { + await assertExpressionIndexHistory(testCase, `generated`) + }), + { + ...oracleRunConfiguration(), + verbose: 2, + }, + ) + }) +}) From 672019444a39068d07b9feb25bd52f94947e8d9f Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 21 Sep 2026 10:01:36 +0100 Subject: [PATCH 2/8] docs(test): clarify expression index oracle model --- .../tests/expression-index-oracle.test.ts | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts b/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts index 2f0ab9b4af..56051e267e 100644 --- a/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts +++ b/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts @@ -1,13 +1,16 @@ /** - * Law and source: RFC #1659 invariant 8 requires persisted index DDL and the - * indexed runtime predicate to have the same SQLite expression shape. SQLite - * requires syntactically matching expressions before an expression index can - * satisfy a predicate. + * # When does SQLite use a persisted expression index? * - * Domain: constructively generated object-rooted SQLite JSON paths have a - * one-to-six-character identifier root and up to three identifier/array-index - * tail segments. Identifiers start with a/m/p/t/x and continue with - * a/b/e/i/n/r/s/0/1; array indices are 0..3. Independent equality values are + * Contract and source: RFC #1659 invariant 8 requires persisted index DDL and + * the indexed runtime predicate to have the same SQLite expression shape. + * SQLite requires syntactically matching expressions before an expression + * index can satisfy a predicate. + * + * History grammar and domain: constructively generated object-rooted SQLite + * JSON paths have a one-to-six-character identifier root and up to three + * identifier/array-index tail segments. Identifiers start with a/m/p/t/x and + * continue with a/b/e/i/n/r/s/0/1; array indices are 0..3. Independent + * equality values are * integer targets -10_000..10_000 with adjacent distractors, * `target-'${suffix}` strings with `before-${suffix}`/`after-${suffix}` * distractors for suffixes 0..10_000, or booleans with duplicated opposite @@ -18,8 +21,9 @@ * coercions can change the indexed expression. Each generated history inserts * rows, creates the serialized ref index, scans, and loads the subset. * - * Reference: a full adapter scan followed by a small path walker and strict - * scalar equality. It does not call the SQL compiler or reuse its path logic. + * Independent model: a full adapter scan followed by a small path walker and + * strict scalar equality. It does not call the SQL compiler or reuse its path + * logic. * * Production path and checkpoint: the public SQLite-core adapter factory with * the real BetterSqlite3SQLiteDriver. The exact SQL and bindings passed to the From 6db8fe7535c3f4a686e7172c17093f22535a4927 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 22 Sep 2026 12:14:22 +0100 Subject: [PATCH 3/8] test(sqlite): accept equivalent query plan formats --- .../tests/expression-index-oracle.test.ts | 54 +++++++++++++++++-- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts b/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts index 56051e267e..7ac696642f 100644 --- a/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts +++ b/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts @@ -280,10 +280,18 @@ function planUsesNamedIndex( tableName: string, indexName: string, ): boolean { + const tablePattern = sqlitePlanIdentifierPattern(tableName) + const indexPattern = sqlitePlanIdentifierPattern(indexName) + const searchPattern = new RegExp( + `\\bSEARCH(?: TABLE)? ${tablePattern}(?:\\s|$)`, + ) + const indexUsagePattern = new RegExp( + `\\bUSING INDEX ${indexPattern}(?:\\s|$)`, + ) + return plan.some( ({ detail }) => - detail.includes(`SEARCH ${tableName}`) && - detail.includes(`USING INDEX ${indexName}`), + searchPattern.test(detail) && indexUsagePattern.test(detail), ) } @@ -291,7 +299,14 @@ function planScansTable( plan: ReadonlyArray, tableName: string, ): boolean { - return plan.some(({ detail }) => detail.startsWith(`SCAN ${tableName}`)) + const tablePattern = sqlitePlanIdentifierPattern(tableName) + const scanPattern = new RegExp(`^SCAN(?: TABLE)? ${tablePattern}(?:\\s|$)`) + return plan.some(({ detail }) => scanPattern.test(detail)) +} + +function sqlitePlanIdentifierPattern(identifier: string): string { + const escaped = identifier.replace(/[.*+?^${}()|[\]\\]/g, `\\$&`) + return `(?:"${escaped}"|${escaped})` } async function assertExpressionIndexHistory( @@ -447,6 +462,39 @@ async function assertExpressionIndexHistory( } describe(`SQLite expression-index oracle`, () => { + it(`recognizes equivalent SQLite plan identifier formats without prefix collisions`, () => { + const tableName = `rows` + const indexName = `literal_ddl` + + for (const detail of [ + `SEARCH rows USING INDEX literal_ddl (=?)`, + `SEARCH TABLE rows USING INDEX literal_ddl (=?)`, + `SEARCH "rows" USING INDEX "literal_ddl" (=?)`, + `SEARCH TABLE "rows" USING INDEX "literal_ddl" (=?)`, + ]) { + expect(planUsesNamedIndex([{ detail }], tableName, indexName)).toBe(true) + } + + for (const detail of [`SCAN rows`, `SCAN TABLE rows`, `SCAN "rows"`]) { + expect(planScansTable([{ detail }], tableName)).toBe(true) + } + + expect( + planUsesNamedIndex( + [ + { + detail: `SEARCH rows_archive USING INDEX literal_ddl_backup (=?)`, + }, + ], + tableName, + indexName, + ), + ).toBe(false) + expect(planScansTable([{ detail: `SCAN rows_archive` }], tableName)).toBe( + false, + ) + }) + it(`distinguishes rejected DDL path binding from correct predicate rows without index use`, async () => { const tempDirectory = mkdtempSync(join(tmpdir(), `db-index-controls-`)) const databasePath = join(tempDirectory, `state.sqlite`) From 28bbddd1c9154fd62a5553664372209755469782 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 22 Sep 2026 17:42:55 +0100 Subject: [PATCH 4/8] docs(oracles): label expression-index limits --- .../tests/expression-index-oracle.test.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts b/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts index 7ac696642f..77b5926277 100644 --- a/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts +++ b/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts @@ -16,10 +16,11 @@ * distractors for suffixes 0..10_000, or booleans with duplicated opposite * distractors. Legal `_root`/`Upper_2` paths, fractional numbers, and empty * strings are omitted from the generated campaign; their same-path audit probes - * are GREEN, so they are not a permanent matrix. Null and persisted tagged - * values remain outside this query-planning law because their operators or - * coercions can change the indexed expression. Each generated history inserts - * rows, creates the serialized ref index, scans, and loads the subset. + * are GREEN, so they are not a permanent matrix. Known omissions: null and + * persisted tagged values remain outside this query-planning law because their + * operators or coercions can change the indexed expression. Each generated + * history inserts rows, creates the serialized ref index, scans, and loads the + * subset. * * Independent model: a full adapter scan followed by a small path walker and * strict scalar equality. It does not call the SQL compiler or reuse its path From 7bcd8b621783e69403b7190f508306fa4c8d430e Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 23 Sep 2026 12:20:33 +0100 Subject: [PATCH 5/8] fix(sqlite): preserve expression index semantics --- .../fix-sqlite-expression-index-planning.md | 3 +- docs/contributing/oracle-coverage.md | 3 +- .../db-sqlite-persistence-core/src/errors.ts | 9 + .../db-sqlite-persistence-core/src/index.ts | 1 + .../src/persisted.ts | 3 +- .../src/sqlite-core-adapter.ts | 261 ++-- .../src/sqlite-value.ts | 30 + .../tests/sqlite-core-adapter.test.ts | 43 +- packages/db/src/collection/subscription.ts | 7 +- packages/db/src/query/builder/ref-proxy.ts | 66 +- packages/db/src/query/compiler/evaluators.ts | 11 +- packages/db/src/query/compiler/expressions.ts | 21 +- packages/db/src/query/compiler/group-by.ts | 9 +- .../db/src/query/compiler/lazy-targets.ts | 6 +- packages/db/src/query/compiler/select.ts | 3 +- packages/db/src/query/ir-stable-identity.ts | 33 + packages/db/src/query/ir.ts | 38 + .../db/tests/query/builder/ref-proxy.test.ts | 2 + .../tests/query/compiler/evaluators.test.ts | 26 +- .../db/tests/query/ir-stable-identity.test.ts | 15 + packages/db/tests/query/optimizer.test.ts | 22 + .../tests/expression-index-oracle.test.ts | 1245 ++++++++++++++++- 22 files changed, 1647 insertions(+), 210 deletions(-) create mode 100644 packages/db-sqlite-persistence-core/src/sqlite-value.ts diff --git a/.changeset/fix-sqlite-expression-index-planning.md b/.changeset/fix-sqlite-expression-index-planning.md index 7256a2821a..d22921dd29 100644 --- a/.changeset/fix-sqlite-expression-index-planning.md +++ b/.changeset/fix-sqlite-expression-index-planning.md @@ -1,5 +1,6 @@ --- +'@tanstack/db': patch '@tanstack/db-sqlite-persistence-core': patch --- -Use canonical SQL literals for serialized ref JSON paths so runtime predicates match persisted expression indexes and SQLite can use them. +Preserve explicit source aliases without changing legacy property paths. Compile SQLite expression-index queries consistently and reject BigInts outside SQLite's signed 64-bit range. diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 669dd39878..1dbfdb5de8 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -111,7 +111,7 @@ comment and the current API/architecture contract before extending its model. | Electric and TrailBase | [Electric histories](https://github.com/TanStack/db/blob/main/packages/electric-db-collection/tests/electric-oracle.property.test.ts), [PostgreSQL semantics](https://github.com/TanStack/db/blob/main/packages/electric-db-collection/e2e/sql-predicate-semantics.e2e.test.ts), [TrailBase contract](https://github.com/TanStack/db/blob/main/packages/trailbase-db-collection/tests/ORACLE.md) | Installed SDK delivery/framing, independent predicates, exact subscription arguments and late errors. SDK fixtures and a real service test earn different credit. | | PowerSync | [tests](https://github.com/TanStack/db/tree/main/packages/powersync-db-collection/tests), `tests/correctness-oracle.test.ts` | Applied receipt positions crossed with held peers, native SQLite/SDK and cleanup evidence. Run the focused owner with the package's `test:oracles` command. A timeout mutant proves a progress failure, not every value assertion. | | SQLite persistence and native hosts | [persisted histories](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/persisted.test.ts), [driver contracts](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/contracts/sqlite-driver-contract.ts), [browser OPFS lifecycle](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/opfs-page-lifecycle-oracle.test.ts), [worker diagnostics](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/opfs-worker-diagnostics-oracle.test.ts), [113-law manifest](https://github.com/TanStack/db/blob/main/packages/db-collection-e2e/src/fixtures/persisted-conformance-manifest.ts) | Cache/remote rejection/peer/reopen histories, exact driver results, controlled page/worker ownership, and diagnostic-cause retention. Fake workers and synthetic page events do not prove native handle release or real bfcache admission. The manifest excludes progressive and move suites; registration and shim runs are not device execution. | -| SQLite expression-index planning | [Node expression-index oracle](https://github.com/TanStack/db/blob/main/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts) | RFC #1659 invariant 8 owns matching persisted-index and runtime-predicate JSON expression shapes. Generated paths use 1-6 character identifiers starting with `a/m/p/t/x` and continuing with `a/b/e/i/n/r/s/0/1`, plus up to three identifier or `0..3` index tails. Values are integer targets `-10_000..10_000` with adjacent distractors, `target-'${suffix}` strings with `before-${suffix}`/`after-${suffix}` distractors for suffixes `0..10_000`, and booleans with duplicated opposite distractors. `_root`/`Upper_2`, fractional-number, and empty-string examples are omitted from generation but same-path audit probes are GREEN; no permanent matrix is claimed. An independent full scan judges matching keys while captured real-driver SQL and bindings are replayed through `EXPLAIN QUERY PLAN` for a named-index search. Run the package's `test:oracles` campaign. Arbitrary raw SQL, result ordering, date/bigint/null/tagged-value coercions, and native-host execution are outside this law. | +| SQLite expression-index planning | [Node expression-index oracle](https://github.com/TanStack/db/blob/main/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts) | RFC #1659 invariant 8 owns identical persisted-index and runtime-expression shapes. Independent expected keys are checked against direct captured SQL, adapter results, and named-index plans. Limits: bounded unqualified JSON paths/scalars, signed-range BigInts, Node BetterSQLite, and no null, arbitrary raw SQL, or native-host planning. Run the package's `test:oracles` campaign. | | Offline execution | [scheduler](https://github.com/TanStack/db/blob/main/packages/offline-transactions/tests/KeyScheduler.property.test.ts), [leadership](https://github.com/TanStack/db/blob/main/packages/offline-transactions/tests/leadership-replay.property.test.ts), [settlement](https://github.com/TanStack/db/blob/main/packages/offline-transactions/tests/transaction-settlement.property.test.ts), [serialization](https://github.com/TanStack/db/blob/main/packages/offline-transactions/tests/transaction-serializer.property.test.ts) | Declarative FIFO eligibility, per-transaction outcomes, durable state and typed wire trees. Issued work may finish after ownership loss, but new work must not start. Exactly-once network execution is not promised. | | Frameworks | [React conformance](https://github.com/TanStack/db/blob/main/packages/react-db/tests/conformance.test.tsx), [React pagination](https://github.com/TanStack/db/blob/main/packages/react-db/tests/infinite-query-conformance.test.tsx), [shared suites](https://github.com/TanStack/db/tree/main/packages/db-collection-e2e/src/suites) | Exact exposed rows/pages and each framework's own lifecycle cuts. A React witness does not prove Vue/Solid/Angular/Svelte scheduling. Preserve their receiving registrations. | | Structural values and ordered primitives | [hash values](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash.property.test.ts), [hash graphs](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash-graph.property.test.ts), [mixed hash graphs](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash-mixed-graph.property.test.ts), [hash retry](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash-failure-retry.property.test.ts), [comparison](https://github.com/TanStack/db/blob/main/packages/db/tests/comparison.property.test.ts), [deep equality](https://github.com/TanStack/db/blob/main/packages/db/tests/utils.property.test.ts), [cursor](https://github.com/TanStack/db/blob/main/packages/db/tests/cursor.property.test.ts), [indexes](https://github.com/TanStack/db/blob/main/packages/db/tests/index-update.property.test.ts), [query identity](https://github.com/TanStack/db/blob/main/packages/db/tests/query/identity-output-shape-oracle.test.ts) | Independent flat values, graph topology, algebraic laws, Map/group/sort recomputation, expression denotation, and compiled output bags. Hash collision freedom is not promised. Unsupported composite cursors reject. | @@ -156,6 +156,7 @@ pnpm --filter @tanstack/db-ivm build pnpm --filter @tanstack/db build pnpm --filter @tanstack/db test:oracles pnpm --filter @tanstack/powersync-db-collection test:oracles +pnpm --filter @tanstack/db-sqlite-persistence-core build pnpm --filter @tanstack/node-db-sqlite-persistence test:oracles # Service-dependent: requires Electric and PostgreSQL to be running. pnpm --filter @tanstack/electric-db-collection test:e2e diff --git a/packages/db-sqlite-persistence-core/src/errors.ts b/packages/db-sqlite-persistence-core/src/errors.ts index f0ffc330eb..5035539b98 100644 --- a/packages/db-sqlite-persistence-core/src/errors.ts +++ b/packages/db-sqlite-persistence-core/src/errors.ts @@ -61,6 +61,15 @@ export class InvalidPersistedStorageKeyEncodingError extends InvalidPersistedCol } } +export class SQLiteBigIntOutOfRangeError extends PersistedCollectionCoreError { + constructor(value: bigint, minimum: bigint, maximum: bigint) { + super( + `SQLite BigInt value ${value} is outside the signed 64-bit range [${minimum}, ${maximum}]`, + ) + this.name = `SQLiteBigIntOutOfRangeError` + } +} + export class PersistenceUnavailableError extends PersistedCollectionCoreError { constructor(details?: string) { super( diff --git a/packages/db-sqlite-persistence-core/src/index.ts b/packages/db-sqlite-persistence-core/src/index.ts index 9e2bb9faae..0e1eeacbff 100644 --- a/packages/db-sqlite-persistence-core/src/index.ts +++ b/packages/db-sqlite-persistence-core/src/index.ts @@ -1,5 +1,6 @@ export * from './persisted' export * from './errors' export * from './sqlite-core-adapter' +export * from './sqlite-value' // Re-export for use in non-secure browser contexts (see #1541) export { safeRandomUUID } from '@tanstack/db' diff --git a/packages/db-sqlite-persistence-core/src/persisted.ts b/packages/db-sqlite-persistence-core/src/persisted.ts index 0f1e4112ac..5179578c6c 100644 --- a/packages/db-sqlite-persistence-core/src/persisted.ts +++ b/packages/db-sqlite-persistence-core/src/persisted.ts @@ -13,6 +13,7 @@ import { InvalidPersistenceAdapterError, InvalidSyncConfigError, } from './errors' +import { serializeSQLiteBigInt } from './sqlite-value' import type { StandardSchemaV1 } from '@standard-schema/spec' import type { ChangeMessageOrDeleteKeyMessage, @@ -640,7 +641,7 @@ function toStableSerializable(value: unknown): unknown { case `boolean`: return value case `bigint`: - return value.toString() + return serializeSQLiteBigInt(value) case `function`: case `symbol`: case `undefined`: diff --git a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts index 6fd5f57dc8..e03d01f91c 100644 --- a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts +++ b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts @@ -12,6 +12,12 @@ import { decodePersistedStorageKey, encodePersistedStorageKey, } from './persisted' +import { + PERSISTED_TYPE_TAG, + PERSISTED_VALUE_TAG, + assertSQLiteBigIntInRange, + serializeSQLiteBigInt, +} from './sqlite-value' import type { LoadSubsetOptions } from '@tanstack/db' import type { PersistedIndexSpec, @@ -37,6 +43,11 @@ type CompiledSqlFragment = { valueKind?: CompiledValueKind } +type SqlExpressionCompilationContext = + | `predicate` + | `index-expression` + | `comparison-target` + type StoredSqliteRow = { key: string value: string @@ -91,9 +102,6 @@ export const DEFAULT_APPLIED_TX_PRUNE_MAX_AGE_SECONDS = 24 * 60 * 60 const SQLITE_MAX_IN_BATCH_SIZE = 900 const SAFE_IDENTIFIER_PATTERN = /^[a-zA-Z_][a-zA-Z0-9_]*$/ const FORBIDDEN_SQL_FRAGMENT_PATTERN = /(;|--|\/\*)/ -const PERSISTED_TYPE_TAG = `__tanstack_db_persisted_type__` -const PERSISTED_VALUE_TAG = `value` - type CompiledValueKind = `unknown` | `bigint` | `date` | `datetime` type PersistedTaggedValueType = | `bigint` @@ -171,10 +179,7 @@ function encodePersistedJsonValue(value: unknown): unknown { } if (typeof value === `bigint`) { - return { - [PERSISTED_TYPE_TAG]: `bigint`, - [PERSISTED_VALUE_TAG]: value.toString(), - } satisfies PersistedTaggedValue + return serializeSQLiteBigInt(value) satisfies PersistedTaggedValue } if (value instanceof Date) { @@ -233,7 +238,7 @@ function decodePersistedJsonValue(value: unknown): unknown { if (isPersistedTaggedValue(value)) { switch (value[PERSISTED_TYPE_TAG]) { case `bigint`: - return BigInt(value[PERSISTED_VALUE_TAG]) + return assertSQLiteBigIntInRange(BigInt(value[PERSISTED_VALUE_TAG])) case `date`: { const parsedDate = new Date(value[PERSISTED_VALUE_TAG]) return Number.isNaN(parsedDate.getTime()) ? null : parsedDate @@ -287,6 +292,7 @@ function toSqliteParameterValue(value: unknown): SqliteSupportedValue { } if (typeof value === `bigint`) { + assertSQLiteBigIntInRange(value) return value.toString() } @@ -317,10 +323,20 @@ function toSqliteLiteral(value: SqliteSupportedValue): string { return `'${value.replace(/'/g, `''`)}'` } +function toSqliteExpressionLiteral(value: unknown): string { + if (typeof value === `bigint`) { + return assertSQLiteBigIntInRange(value).toString() + } + return toSqliteLiteral(toSqliteParameterValue(value)) +} + function inlineSqlParams( sql: string, params: ReadonlyArray, ): string { + // Every question mark in this compiler-owned SQL is a placeholder. Ref-path + // literals cannot contain one because createJsonPath rejects such segments. + // Callers must bypass this helper when SQL already contains other literals. let index = 0 const inlinedSql = sql.replace(/\?/g, () => { const paramValue = params[index] @@ -339,57 +355,6 @@ function inlineSqlParams( type CompiledRowExpressionEvaluator = (row: Record) => unknown -function collectAliasQualifiedRefSegments( - expression: IR.BasicExpression, - segments: Set = new Set(), -): Set { - if (expression.type === `ref`) { - if (expression.path.length > 1) { - const rootSegment = String(expression.path[0]) - if (rootSegment.length > 0) { - segments.add(rootSegment) - } - } - return segments - } - - if (expression.type === `func`) { - for (const arg of expression.args) { - collectAliasQualifiedRefSegments(arg, segments) - } - } - - return segments -} - -function createAliasAwareRowProxy( - row: Record, - aliasSegments: ReadonlySet, -): Record { - return new Proxy(row, { - get(target, prop, receiver) { - if (typeof prop !== `string`) { - return Reflect.get(target, prop, receiver) - } - - if (Object.prototype.hasOwnProperty.call(target, prop)) { - const value = Reflect.get(target, prop, receiver) - if (value !== undefined || !aliasSegments.has(prop)) { - return value - } - - return target - } - - if (aliasSegments.has(prop)) { - return target - } - - return undefined - }, - }) -} - function compileRowExpressionEvaluator( expression: IR.BasicExpression, ): CompiledRowExpressionEvaluator { @@ -401,24 +366,7 @@ function compileRowExpressionEvaluator( `Unsupported expression for SQLite adapter fallback evaluator: ${(error as Error).message}`, ) } - - const aliasSegments = collectAliasQualifiedRefSegments(expression) - if (aliasSegments.size === 0) { - return (row) => baseEvaluator(row) - } - - const proxyCache = new WeakMap< - Record, - Record - >() - return (row) => { - let proxy = proxyCache.get(row) - if (!proxy) { - proxy = createAliasAwareRowProxy(row, aliasSegments) - proxyCache.set(row, proxy) - } - return baseEvaluator(proxy) - } + return (row) => baseEvaluator(row) } function getOrderByObjectId(value: object): number { @@ -575,20 +523,28 @@ function resolveComparisonValueKind( function compileComparisonSql( operator: `=` | `>` | `>=` | `<` | `<=`, + leftExpression: IR.BasicExpression, + rightExpression: IR.BasicExpression, leftSql: string, rightSql: string, valueKind: CompiledValueKind, + leftKind: CompiledValueKind, + rightKind: CompiledValueKind, ): string { - if (valueKind === `bigint`) { - return `(CAST(${leftSql} AS NUMERIC) ${operator} CAST(${rightSql} AS NUMERIC))` - } - if (valueKind === `date`) { - return `(date(${leftSql}) ${operator} date(${rightSql}))` - } - if (valueKind === `datetime`) { - return `(datetime(${leftSql}) ${operator} datetime(${rightSql}))` + const compileOperand = ( + expression: IR.BasicExpression, + sql: string, + otherKind: CompiledValueKind, + ): string => { + if (expression.type !== `val`) return sql + if (valueKind === `date` && otherKind === `date`) return `date(${sql})` + if (valueKind === `datetime` && otherKind === `datetime`) { + return `datetime(${sql})` + } + return sql } - return `(${leftSql} ${operator} ${rightSql})` + + return `(${compileOperand(leftExpression, leftSql, rightKind)} ${operator} ${compileOperand(rightExpression, rightSql, leftKind)})` } function compileRefExpressionSql(jsonPath: string): CompiledSqlFragment { @@ -661,21 +617,64 @@ function stableStringify(value: unknown): string { return serializePersistedRowValue(value) } +function argumentCompilationContext( + parentName: string, + argumentIndex: number, + argument: IR.BasicExpression, + parentContext: SqlExpressionCompilationContext, +): SqlExpressionCompilationContext { + if (parentContext === `index-expression`) return `index-expression` + + switch (parentName) { + case `and`: + case `or`: + case `not`: + return `predicate` + case `eq`: + case `gt`: + case `gte`: + case `lt`: + case `lte`: + case `like`: + case `ilike`: + if (argument.type !== `val`) return `index-expression` + return typeof argument.value === `bigint` + ? `comparison-target` + : `predicate` + case `in`: + return argumentIndex === 0 ? `index-expression` : `predicate` + case `isNull`: + case `isUndefined`: + return `index-expression` + default: + return `index-expression` + } +} + function compileSqlExpression( expression: IR.BasicExpression, + context: SqlExpressionCompilationContext = `predicate`, ): CompiledSqlFragment { if (expression.type === `val`) { const valueKind = getLiteralValueKind(expression.value) + const value = toSqliteParameterValue(expression.value) return { supported: true, - sql: `?`, - params: [toSqliteParameterValue(expression.value)], + sql: + context === `index-expression` + ? toSqliteExpressionLiteral(expression.value) + : context === `comparison-target` + ? expression.value.toString() + : `?`, + params: context === `predicate` ? [value] : [], valueKind, } } if (expression.type === `ref`) { - const jsonPath = createJsonPath(expression.path.map(String)) + const jsonPath = createJsonPath( + IR.getPropRefPropertyPath(expression).map(String), + ) if (!jsonPath) { return { supported: false, @@ -687,7 +686,12 @@ function compileSqlExpression( return compileRefExpressionSql(jsonPath) } - const compiledArgs = expression.args.map((arg) => compileSqlExpression(arg)) + const compiledArgs = expression.args.map((arg, index) => + compileSqlExpression( + arg, + argumentCompilationContext(expression.name, index, arg, context), + ), + ) if (compiledArgs.some((arg) => !arg.supported)) { return { supported: false, @@ -736,9 +740,13 @@ function compileSqlExpression( supported: true, sql: compileComparisonSql( operatorByName[expression.name], + expression.args[0]!, + expression.args[1]!, argSql[0], argSql[1], valueKind, + getCompiledValueKind(compiledArgs[0]), + getCompiledValueKind(compiledArgs[1]), ), params, } @@ -793,13 +801,17 @@ function compileSqlExpression( return { supported: false, sql: ``, params: [] } } + if (context === `index-expression`) { + return { + supported: true, + sql: `(${leftSql} IN (${listValue + .map((value) => toSqliteExpressionLiteral(value)) + .join(`, `)}))`, + params: leftParams, + } + } + if (listValue.length > SQLITE_MAX_IN_BATCH_SIZE) { - const hasBigIntValues = listValue.some( - (value) => typeof value === `bigint`, - ) - const inLeftSql = hasBigIntValues - ? `CAST(${leftSql} AS NUMERIC)` - : leftSql const chunkClauses: Array = [] const batchedParams: Array = [] @@ -812,13 +824,13 @@ function compileSqlExpression( startIndex, startIndex + SQLITE_MAX_IN_BATCH_SIZE, ) - chunkClauses.push( - `(${inLeftSql} IN (${chunkValues.map(() => `?`).join(`, `)}))`, - ) - batchedParams.push(...leftParams) - batchedParams.push( - ...chunkValues.map((value) => toSqliteParameterValue(value)), - ) + const chunkParams: Array = [] + const chunkValueSql = chunkValues.map((value) => { + chunkParams.push(toSqliteParameterValue(value)) + return typeof value === `bigint` ? `CAST(? AS NUMERIC)` : `?` + }) + chunkClauses.push(`(${leftSql} IN (${chunkValueSql.join(`, `)}))`) + batchedParams.push(...leftParams, ...chunkParams) } return { @@ -828,20 +840,15 @@ function compileSqlExpression( } } - const hasBigIntValues = listValue.some( - (value) => typeof value === `bigint`, - ) - const inLeftSql = hasBigIntValues - ? `CAST(${leftSql} AS NUMERIC)` - : leftSql - const listPlaceholders = listValue.map(() => `?`).join(`, `) + const listParams: Array = [] + const listValueSql = listValue.map((value) => { + listParams.push(toSqliteParameterValue(value)) + return typeof value === `bigint` ? `CAST(? AS NUMERIC)` : `?` + }) return { supported: true, - sql: `(${inLeftSql} IN (${listPlaceholders}))`, - params: [ - ...leftParams, - ...listValue.map((value) => toSqliteParameterValue(value)), - ], + sql: `(${leftSql} IN (${listValueSql.join(`, `)}))`, + params: [...leftParams, ...listParams], } } case `like`: @@ -921,7 +928,10 @@ function compileOrderByClauses( const params: Array = [] for (const clause of orderBy) { - const compiledExpression = compileSqlExpression(clause.expression) + const compiledExpression = compileSqlExpression( + clause.expression, + `index-expression`, + ) if (!compiledExpression.supported) { return { supported: false, @@ -957,6 +967,7 @@ function isExpressionLikeShape(value: unknown): value is IR.BasicExpression { path?: unknown name?: unknown args?: unknown + sourceAlias?: unknown } if (candidate.type === `val`) { @@ -964,7 +975,12 @@ function isExpressionLikeShape(value: unknown): value is IR.BasicExpression { } if (candidate.type === `ref`) { - return Array.isArray(candidate.path) + return ( + Array.isArray(candidate.path) && + (candidate.sourceAlias === undefined || + (typeof candidate.sourceAlias === `string` && + candidate.path[0] === candidate.sourceAlias)) + ) } if (candidate.type === `func`) { @@ -994,14 +1010,19 @@ function normalizeIndexSqlFragment(fragment: string): string { // Non-JSON strings are treated as raw SQL fragments below. } - if (hasParsedJson && isExpressionLikeShape(parsedJson)) { - const compiled = compileSqlExpression(parsedJson) + const decodedJson = hasParsedJson + ? decodePersistedJsonValue(parsedJson) + : undefined + if (hasParsedJson && isExpressionLikeShape(decodedJson)) { + const compiled = compileSqlExpression(decodedJson, `index-expression`) if (!compiled.supported) { throw new InvalidPersistedCollectionConfigError( `Persisted index expression is not supported by the SQLite compiler`, ) } - return inlineSqlParams(compiled.sql, compiled.params) + return compiled.params.length === 0 + ? compiled.sql + : inlineSqlParams(compiled.sql, compiled.params) } return sanitizeExpressionSqlFragment(fragment) diff --git a/packages/db-sqlite-persistence-core/src/sqlite-value.ts b/packages/db-sqlite-persistence-core/src/sqlite-value.ts new file mode 100644 index 0000000000..4f76651923 --- /dev/null +++ b/packages/db-sqlite-persistence-core/src/sqlite-value.ts @@ -0,0 +1,30 @@ +import { SQLiteBigIntOutOfRangeError } from './errors' + +export const SQLITE_BIGINT_MIN = -9_223_372_036_854_775_808n +export const SQLITE_BIGINT_MAX = 9_223_372_036_854_775_807n +export const PERSISTED_TYPE_TAG = `__tanstack_db_persisted_type__` +export const PERSISTED_VALUE_TAG = `value` + +export type SerializedSQLiteBigInt = { + [PERSISTED_TYPE_TAG]: `bigint` + [PERSISTED_VALUE_TAG]: string +} + +export function assertSQLiteBigIntInRange(value: bigint): bigint { + if (value < SQLITE_BIGINT_MIN || value > SQLITE_BIGINT_MAX) { + throw new SQLiteBigIntOutOfRangeError( + value, + SQLITE_BIGINT_MIN, + SQLITE_BIGINT_MAX, + ) + } + return value +} + +export function serializeSQLiteBigInt(value: bigint): SerializedSQLiteBigInt { + assertSQLiteBigIntInRange(value) + return { + [PERSISTED_TYPE_TAG]: `bigint`, + [PERSISTED_VALUE_TAG]: value.toString(), + } +} diff --git a/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts b/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts index de10c93e6c..532ef55749 100644 --- a/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts +++ b/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts @@ -1545,11 +1545,11 @@ export function runSQLiteCoreAdapterContractSuite( }) // `meta-field` makes SQL pushdown unsupported, so filter correctness comes - // from the in-memory evaluator. The leading `todos` segment simulates - // alias-qualified refs emitted by higher-level query builders. + // from the in-memory evaluator. The explicit source alias is the only + // signal that the leading `todos` segment is qualification. const rows = await adapter.loadSubset(collectionId, { where: new IR.Func(`eq`, [ - new IR.PropRef([`todos`, `meta-field`]), + new IR.PropRef([`todos`, `meta-field`], `todos`), new IR.Value(`alpha`), ]), }) @@ -1557,6 +1557,43 @@ export function runSQLiteCoreAdapterContractSuite( expect(rows.map((row) => row.key)).toEqual([`1`]) }) + it(`does not guess that a legacy fallback path is an alias`, async () => { + const { driver } = registerContractHarness() + const adapter = new SQLiteCorePersistenceAdapter({ driver }) + const collectionId = `fallback-legacy-nested-ref` + + await adapter.applyCommittedTx(collectionId, { + txId: `seed-legacy-nested-fallback`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: [ + { + type: `insert`, + key: `nested-match`, + value: { + profile: { [`meta-field`]: `alpha` }, + [`meta-field`]: `flat-other`, + }, + }, + { + type: `insert`, + key: `flat-only`, + value: { [`meta-field`]: `alpha` }, + }, + ], + }) + + const rows = await adapter.loadSubset(collectionId, { + where: new IR.Func(`eq`, [ + new IR.PropRef([`profile`, `meta-field`]), + new IR.Value(`alpha`), + ]), + }) + + expect(rows.map((row) => row.key)).toEqual([`nested-match`]) + }) + it(`compiles serialized expression index specs used by phase-2 metadata`, async () => { const { adapter, driver } = registerContractHarness() const collectionId = `serialized-index` diff --git a/packages/db/src/collection/subscription.ts b/packages/db/src/collection/subscription.ts index 5faf36d444..41e36e6834 100644 --- a/packages/db/src/collection/subscription.ts +++ b/packages/db/src/collection/subscription.ts @@ -1,6 +1,6 @@ import { ensureIndexForExpression } from '../indexes/auto-index.js' import { and, eq } from '../query/builder/functions.js' -import { PropRef, Value } from '../query/ir.js' +import { PropRef, Value, getPropRefPropertyPath } from '../query/ir.js' import { EventEmitter } from '../event-emitter.js' import { compileExpression } from '../query/compiler/evaluators.js' import { buildCursor, buildCursorCurrent } from '../utils/cursor.js' @@ -1453,7 +1453,10 @@ export class CollectionSubscription const orderByExpression = orderBy[0]!.expression const valueExtractor = orderByExpression.type === `ref` - ? compileExpression(new PropRef(orderByExpression.path), true) + ? compileExpression( + new PropRef(getPropRefPropertyPath(orderByExpression)), + true, + ) : null while (valuesNeeded() > 0 && !collectionExhausted()) { diff --git a/packages/db/src/query/builder/ref-proxy.ts b/packages/db/src/query/builder/ref-proxy.ts index 3b19330871..d96d0d146c 100644 --- a/packages/db/src/query/builder/ref-proxy.ts +++ b/packages/db/src/query/builder/ref-proxy.ts @@ -9,6 +9,8 @@ export interface RefProxy { /** @internal */ readonly __path: Array /** @internal */ + readonly __sourceAlias?: string + /** @internal */ readonly __type: T } @@ -71,6 +73,7 @@ export function createSingleRowRefProxy< get(target, prop, receiver) { if (prop === `__refProxy`) return true if (prop === `__path`) return path + if (prop === `__sourceAlias`) return undefined if (prop === `__type`) return undefined // Type is only for TypeScript inference if (typeof prop === `symbol`) return Reflect.get(target, prop, receiver) @@ -79,7 +82,12 @@ export function createSingleRowRefProxy< }, has(target, prop) { - if (prop === `__refProxy` || prop === `__path` || prop === `__type`) + if ( + prop === `__refProxy` || + prop === `__path` || + prop === `__sourceAlias` || + prop === `__type` + ) return true return Reflect.has(target, prop) }, @@ -89,7 +97,12 @@ export function createSingleRowRefProxy< }, getOwnPropertyDescriptor(target, prop) { - if (prop === `__refProxy` || prop === `__path` || prop === `__type`) { + if ( + prop === `__refProxy` || + prop === `__path` || + prop === `__sourceAlias` || + prop === `__type` + ) { return { enumerable: false, configurable: true } } return Reflect.getOwnPropertyDescriptor(target, prop) @@ -124,6 +137,7 @@ export function createRefProxy>( get(target, prop, receiver) { if (prop === `__refProxy`) return true if (prop === `__path`) return path + if (prop === `__sourceAlias`) return path[0] if (prop === `__type`) return undefined // Type is only for TypeScript inference if (typeof prop === `symbol`) return Reflect.get(target, prop, receiver) @@ -132,7 +146,12 @@ export function createRefProxy>( }, has(target, prop) { - if (prop === `__refProxy` || prop === `__path` || prop === `__type`) + if ( + prop === `__refProxy` || + prop === `__path` || + prop === `__sourceAlias` || + prop === `__type` + ) return true return Reflect.has(target, prop) }, @@ -151,7 +170,12 @@ export function createRefProxy>( }, getOwnPropertyDescriptor(target, prop) { - if (prop === `__refProxy` || prop === `__path` || prop === `__type`) { + if ( + prop === `__refProxy` || + prop === `__path` || + prop === `__sourceAlias` || + prop === `__type` + ) { return { enumerable: false, configurable: true } } return Reflect.getOwnPropertyDescriptor(target, prop) @@ -167,6 +191,7 @@ export function createRefProxy>( get(target, prop, receiver) { if (prop === `__refProxy`) return true if (prop === `__path`) return [] + if (prop === `__sourceAlias`) return undefined if (prop === `__type`) return undefined // Type is only for TypeScript inference if (typeof prop === `symbol`) return Reflect.get(target, prop, receiver) @@ -179,18 +204,28 @@ export function createRefProxy>( }, has(target, prop) { - if (prop === `__refProxy` || prop === `__path` || prop === `__type`) + if ( + prop === `__refProxy` || + prop === `__path` || + prop === `__sourceAlias` || + prop === `__type` + ) return true if (typeof prop === `string` && aliases.includes(prop)) return true return Reflect.has(target, prop) }, ownKeys(_target) { - return [...aliases, `__refProxy`, `__path`, `__type`] + return [...aliases, `__refProxy`, `__path`, `__sourceAlias`, `__type`] }, getOwnPropertyDescriptor(target, prop) { - if (prop === `__refProxy` || prop === `__path` || prop === `__type`) { + if ( + prop === `__refProxy` || + prop === `__path` || + prop === `__sourceAlias` || + prop === `__type` + ) { return { enumerable: false, configurable: true } } if (typeof prop === `string` && aliases.includes(prop)) { @@ -231,6 +266,7 @@ export function createRefProxyWithSelected>( get(target, prop, receiver) { if (prop === `__refProxy`) return true if (prop === `__path`) return [`$selected`, ...path] + if (prop === `__sourceAlias`) return `$selected` if (prop === `__type`) return undefined if (typeof prop === `symbol`) return Reflect.get(target, prop, receiver) @@ -239,7 +275,12 @@ export function createRefProxyWithSelected>( }, has(target, prop) { - if (prop === `__refProxy` || prop === `__path` || prop === `__type`) + if ( + prop === `__refProxy` || + prop === `__path` || + prop === `__sourceAlias` || + prop === `__type` + ) return true return Reflect.has(target, prop) }, @@ -249,7 +290,12 @@ export function createRefProxyWithSelected>( }, getOwnPropertyDescriptor(target, prop) { - if (prop === `__refProxy` || prop === `__path` || prop === `__type`) { + if ( + prop === `__refProxy` || + prop === `__path` || + prop === `__sourceAlias` || + prop === `__type` + ) { return { enumerable: false, configurable: true } } return Reflect.getOwnPropertyDescriptor(target, prop) @@ -303,7 +349,7 @@ export function toExpression(value: T): BasicExpression export function toExpression(value: RefProxy): BasicExpression export function toExpression(value: any): BasicExpression { if (isRefProxy(value)) { - return new PropRef(value.__path) + return new PropRef(value.__path, value.__sourceAlias) } // toArray(), concat(toArray()), and materialize() must be used as direct // select fields, not inside expressions diff --git a/packages/db/src/query/compiler/evaluators.ts b/packages/db/src/query/compiler/evaluators.ts index 55b8c68f49..322c1ea832 100644 --- a/packages/db/src/query/compiler/evaluators.ts +++ b/packages/db/src/query/compiler/evaluators.ts @@ -10,6 +10,7 @@ import { isUnorderable, normalizeValue, } from '../../utils/comparison.js' +import { getPropRefPropertyPath, getPropRefSourceAlias } from '../ir.js' import type { BasicExpression, Func, PropRef } from '../ir.js' import type { NamespacedRow } from '../../types.js' @@ -144,7 +145,13 @@ function compileExpressionInternal( * Compiles a reference expression into an optimized evaluator */ function compileRef(ref: PropRef): CompiledExpression { - const [namespace, ...propertyPath] = ref.path + const explicitAlias = getPropRefSourceAlias(ref) + const [legacyNamespace, ...legacyPropertyPath] = ref.path + const namespace = explicitAlias ?? legacyNamespace + const propertyPath = + explicitAlias === undefined + ? legacyPropertyPath + : getPropRefPropertyPath(ref) if (!namespace) { throw new EmptyReferencePathError() @@ -221,7 +228,7 @@ function compileRef(ref: PropRef): CompiledExpression { * Compiles a reference expression for single-row evaluation */ function compileSingleRowRef(ref: PropRef): CompiledSingleRowExpression { - const propertyPath = ref.path + const propertyPath = getPropRefPropertyPath(ref) // This function works for all path lengths including empty path return (item) => { diff --git a/packages/db/src/query/compiler/expressions.ts b/packages/db/src/query/compiler/expressions.ts index a52b8d11e5..39a740347c 100644 --- a/packages/db/src/query/compiler/expressions.ts +++ b/packages/db/src/query/compiler/expressions.ts @@ -1,4 +1,10 @@ -import { Func, PropRef, Value } from '../ir.js' +import { + Func, + PropRef, + Value, + getPropRefPropertyPath, + getPropRefSourceAlias, +} from '../ir.js' import type { BasicExpression, OrderBy } from '../ir.js' /** Extracts the source aliases referenced by an expression. */ @@ -6,8 +12,10 @@ export function getSourceAliasesFromExpression( expr: BasicExpression, ): Set { switch (expr.type) { - case `ref`: - return new Set(expr.path[0] ? [expr.path[0]] : []) + case `ref`: { + const sourceAlias = getPropRefSourceAlias(expr) ?? expr.path[0] + return new Set(sourceAlias ? [sourceAlias] : []) + } case `func`: { const sourceAliases = new Set() for (const arg of expr.args) { @@ -48,8 +56,13 @@ export function normalizeExpressionPaths( return new Value(whereClause.value) } else if (tpe === `ref`) { const path = whereClause.path + const sourceAlias = getPropRefSourceAlias(whereClause) if (Array.isArray(path)) { - if (path[0] === collectionAlias && path.length > 1) { + if (sourceAlias === collectionAlias) { + return new PropRef(getPropRefPropertyPath(whereClause)) + } else if (sourceAlias !== undefined) { + return new PropRef(path, sourceAlias) + } else if (path[0] === collectionAlias && path.length > 1) { // Remove the table alias from the path for single-collection queries return new PropRef(path.slice(1)) } else if (path.length === 1 && path[0] !== undefined) { diff --git a/packages/db/src/query/compiler/group-by.ts b/packages/db/src/query/compiler/group-by.ts index 9ce3b2f6a0..9dc604c6aa 100644 --- a/packages/db/src/query/compiler/group-by.ts +++ b/packages/db/src/query/compiler/group-by.ts @@ -670,7 +670,7 @@ export function replaceAggregatesByRefs( for (const [alias, selectExpr] of Object.entries(selectClause)) { if (selectExpr.type === `agg` && aggregatesEqual(aggExpr, selectExpr)) { // Replace with a reference to the computed aggregate - return new PropRef([resultAlias, alias]) + return new PropRef([resultAlias, alias], resultAlias) } } // If no matching aggregate found in SELECT, throw error @@ -801,7 +801,7 @@ function extractAndReplaceAggregates( if (expr.type === `agg`) { const alias = `${aggregatePrefix}${counter.value++}` return { - transformed: new PropRef([`$selected`, alias]), + transformed: new PropRef([`$selected`, alias], `$selected`), extracted: { [alias]: expr }, } } @@ -941,7 +941,7 @@ function replaceGroupByRefsInExpression( ) return groupIndex === -1 ? expr - : new PropRef([`$selected`, groupKeyRefs[groupIndex]!]) + : new PropRef([`$selected`, groupKeyRefs[groupIndex]!], `$selected`) } if (expr.type === `func`) { @@ -988,9 +988,10 @@ function compileGroupedSelectObject( const pathStr = splitIndex >= 0 ? rest.slice(0, splitIndex) : rest const isRefExpr = typeof value === `object` && `type` in value && value.type === `ref` + const path = pathStr.split(`.`) const expression = isRefExpr ? (value as BasicExpression) - : (new PropRef(pathStr.split(`.`)) as BasicExpression) + : (new PropRef(path, path[0]) as BasicExpression) return { key, diff --git a/packages/db/src/query/compiler/lazy-targets.ts b/packages/db/src/query/compiler/lazy-targets.ts index 241ccd7764..c35cc1d153 100644 --- a/packages/db/src/query/compiler/lazy-targets.ts +++ b/packages/db/src/query/compiler/lazy-targets.ts @@ -274,7 +274,11 @@ function toPropRef(expr: unknown): PropRef | undefined { (expr as { type?: string }).type === `ref` && Array.isArray((expr as { path?: unknown }).path) ) { - return new PropRef((expr as unknown as { path: Array }).path) + const ref = expr as unknown as { + path: Array + sourceAlias?: string + } + return new PropRef(ref.path, ref.sourceAlias) } return undefined } diff --git a/packages/db/src/query/compiler/select.ts b/packages/db/src/query/compiler/select.ts index 8b7fb4e127..8ae5c83875 100644 --- a/packages/db/src/query/compiler/select.ts +++ b/packages/db/src/query/compiler/select.ts @@ -302,9 +302,10 @@ function addFromObject( if (pathStr.includes(`.`) || isRefExpr) { // Merge into the current destination (prefixPath) from the referenced source path const targetPath = [...prefixPath] + const path = pathStr.split(`.`) const expr = isRefExpr ? (value as BasicExpression) - : (new PropRef(pathStr.split(`.`)) as BasicExpression) + : (new PropRef(path, path[0]) as BasicExpression) const compiled = compileExpression(expr) ops.push({ kind: `merge`, targetPath, source: compiled }) } else { diff --git a/packages/db/src/query/ir-stable-identity.ts b/packages/db/src/query/ir-stable-identity.ts index 7139942ce2..bc808f9e16 100644 --- a/packages/db/src/query/ir-stable-identity.ts +++ b/packages/db/src/query/ir-stable-identity.ts @@ -3,6 +3,7 @@ import { normalizeValue } from '../utils/comparison.js' import { isRefProxy, toExpression } from './builder/ref-proxy.js' import { getQueryIR } from './builder/query-ir.js' import { getRuntimeReferenceIdentity } from './runtime-reference-identity.js' +import { getPropRefPropertyPath, getPropRefSourceAlias } from './ir.js' import type { Aggregate, BasicExpression, @@ -578,6 +579,38 @@ function canonicalizeExpression( scope?: AliasScope, ): StableIdentityValue { if (expression.type === `ref`) { + const explicitAlias = getPropRefSourceAlias(expression) + if (explicitAlias !== undefined) { + const binding = resolveAliasBinding(scope, explicitAlias) + if (binding !== undefined) { + return { + type: `ref`, + path: [ + [`binding`, ...binding], + ...getPropRefPropertyPath(expression).map((segment, index) => + canonicalizeRuntimeValue( + segment, + `${path}.path[${index + 1}]`, + seen, + ), + ), + ], + } + } + + return { + type: `ref`, + path: expression.path.map((segment, index) => + canonicalizeRuntimeValue(segment, `${path}.path[${index}]`, seen), + ), + sourceAlias: canonicalizeRuntimeValue( + explicitAlias, + `${path}.sourceAlias`, + seen, + ), + } + } + const binding = resolveAliasBinding(scope, expression.path[0] ?? ``) return { type: `ref`, diff --git a/packages/db/src/query/ir.ts b/packages/db/src/query/ir.ts index a04370a90a..872c635805 100644 --- a/packages/db/src/query/ir.ts +++ b/packages/db/src/query/ir.ts @@ -140,13 +140,33 @@ export class UnionAll extends BaseExpression { export class PropRef extends BaseExpression { public type = `ref` as const + declare public readonly sourceAlias?: string constructor( public path: Array, // path to the property in the collection, with the alias as the first element + sourceAlias?: string, ) { super() + if (sourceAlias !== undefined) { + Object.defineProperty(this, `sourceAlias`, { + value: sourceAlias, + enumerable: true, + }) + } } } +/** Returns an explicitly declared source alias without inferring from the path. */ +export function getPropRefSourceAlias(ref: PropRef): string | undefined { + return ref.sourceAlias !== undefined && ref.path[0] === ref.sourceAlias + ? ref.sourceAlias + : undefined +} + +/** Returns the property path after removing only explicit source qualification. */ +export function getPropRefPropertyPath(ref: PropRef): Array { + return getPropRefSourceAlias(ref) === undefined ? ref.path : ref.path.slice(1) +} + export class Value extends BaseExpression { public type = `val` as const constructor( @@ -399,6 +419,24 @@ export function followRef( alias?: string sourceId?: string } | void { + const explicitAlias = getPropRefSourceAlias(ref) + if (explicitAlias !== undefined) { + const aliasRef = getRefFromAlias(query, explicitAlias) + if (!aliasRef) return + + const propertyPath = getPropRefPropertyPath(ref) + if (aliasRef.type === `queryRef`) { + return followRef(aliasRef.query, new PropRef(propertyPath), collection) + } + + return { + collection: aliasRef.collection, + path: propertyPath, + alias: explicitAlias, + sourceId: aliasRef.sourceId, + } + } + if (ref.path.length === 0) { return } diff --git a/packages/db/tests/query/builder/ref-proxy.test.ts b/packages/db/tests/query/builder/ref-proxy.test.ts index e003d34d07..0a10fbc318 100644 --- a/packages/db/tests/query/builder/ref-proxy.test.ts +++ b/packages/db/tests/query/builder/ref-proxy.test.ts @@ -18,6 +18,7 @@ describe(`ref-proxy`, () => { expect(expression).toBeInstanceOf(PropRef) expect((expression as PropRef).path).toEqual([`timestamp`, `seconds`]) + expect((expression as PropRef).sourceAlias).toBeUndefined() }) it(`records built-in method paths only when the type boundary is bypassed`, () => { @@ -195,6 +196,7 @@ describe(`ref-proxy`, () => { expect(expr).toBeInstanceOf(PropRef) expect(expr.type).toBe(`ref`) expect((expr as PropRef).path).toEqual([`users`, `id`]) + expect((expr as PropRef).sourceAlias).toBe(`users`) }) it(`converts literal values to Value expression`, () => { diff --git a/packages/db/tests/query/compiler/evaluators.test.ts b/packages/db/tests/query/compiler/evaluators.test.ts index dac457867a..be801e661e 100644 --- a/packages/db/tests/query/compiler/evaluators.test.ts +++ b/packages/db/tests/query/compiler/evaluators.test.ts @@ -1,6 +1,9 @@ import { describe, expect, it } from 'vitest' import { Temporal } from 'temporal-polyfill' -import { compileExpression } from '../../../src/query/compiler/evaluators.js' +import { + compileExpression, + compileSingleRowExpression, +} from '../../../src/query/compiler/evaluators.js' import { Func, PropRef, Value } from '../../../src/query/ir.js' import type { NamespacedRow } from '../../../src/types.js' @@ -70,6 +73,27 @@ describe(`evaluators`, () => { expect(compiled(row)).toBeUndefined() }) + + it(`uses explicit qualification in namespaced and single-row evaluation`, () => { + const ref = new PropRef([`users`, `profile`, `score`], `users`) + + expect( + compileExpression(ref)({ users: { profile: { score: 7 } } }), + ).toBe(7) + expect(compileSingleRowExpression(ref)({ profile: { score: 7 } })).toBe( + 7, + ) + }) + + it(`keeps unqualified multi-segment refs as nested single-row paths`, () => { + const ref = new PropRef([`profile`, `score`]) + const row = { + score: 3, + profile: { score: 7 }, + } + + expect(compileSingleRowExpression(ref)(row)).toBe(7) + }) }) describe(`function compilation`, () => { diff --git a/packages/db/tests/query/ir-stable-identity.test.ts b/packages/db/tests/query/ir-stable-identity.test.ts index fa1c0748f7..a9d4fe6452 100644 --- a/packages/db/tests/query/ir-stable-identity.test.ts +++ b/packages/db/tests/query/ir-stable-identity.test.ts @@ -627,6 +627,21 @@ describe(`semantic expression identity`, () => { }) describe(`loadSubset demand identity`, () => { + it(`preserves legacy nested-ref identity and distinguishes explicit qualification`, () => { + const nested = new PropRef([`profile`, `score`]) + const qualified = new PropRef([`profile`, `score`], `profile`) + + expect(getLoadSubsetDemandKey({ where: nested })).toBe( + `{"type":"loadSubsetDemand","query":{"type":"loadSubsetQuery","where":{"type":"ref","path":[["string","profile"],["string","score"]]}}}`, + ) + expect(getLoadSubsetDemandKey({ where: qualified })).not.toBe( + getLoadSubsetDemandKey({ where: nested }), + ) + expect(JSON.stringify(nested)).toBe( + `{"path":["profile","score"],"type":"ref"}`, + ) + }) + const id = new PropRef([`id`]) const group = new PropRef([`group`]) const first = new Func(`eq`, [id, new Value(`a`)]) diff --git a/packages/db/tests/query/optimizer.test.ts b/packages/db/tests/query/optimizer.test.ts index 0952ac2ba2..048a435a5b 100644 --- a/packages/db/tests/query/optimizer.test.ts +++ b/packages/db/tests/query/optimizer.test.ts @@ -50,6 +50,28 @@ function createAgg(name: string, ...args: Array) { describe(`Query Optimizer`, () => { describe(`Basic Optimization`, () => { + test(`retains explicit ref qualification while combining predicates`, () => { + const department = new PropRef([`u`, `department_id`], `u`) + const salary = new PropRef([`u`, `salary`], `u`) + const query: QueryIR = { + from: new CollectionRef(mockCollection, `u`), + where: [ + createEq(department, createValue(1)), + createGt(salary, createValue(50_000)), + ], + } + + const { optimizedQuery } = optimizeQuery(query) + const combined = optimizedQuery.where?.[0] as Func + + expect((combined.args[0] as Func).args[0]).toMatchObject({ + sourceAlias: `u`, + }) + expect((combined.args[1] as Func).args[0]).toMatchObject({ + sourceAlias: `u`, + }) + }) + test(`should pass through queries without where clauses`, () => { const query: QueryIR = { from: new CollectionRef(mockCollection, `u`), diff --git a/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts b/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts index 77b5926277..79160fdf78 100644 --- a/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts +++ b/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts @@ -6,56 +6,54 @@ * SQLite requires syntactically matching expressions before an expression * index can satisfy a predicate. * - * History grammar and domain: constructively generated object-rooted SQLite - * JSON paths have a one-to-six-character identifier root and up to three - * identifier/array-index tail segments. Identifiers start with a/m/p/t/x and - * continue with a/b/e/i/n/r/s/0/1; array indices are 0..3. Independent - * equality values are - * integer targets -10_000..10_000 with adjacent distractors, - * `target-'${suffix}` strings with `before-${suffix}`/`after-${suffix}` - * distractors for suffixes 0..10_000, or booleans with duplicated opposite - * distractors. Legal `_root`/`Upper_2` paths, fractional numbers, and empty - * strings are omitted from the generated campaign; their same-path audit probes - * are GREEN, so they are not a permanent matrix. Known omissions: null and - * persisted tagged values remain outside this query-planning law because their - * operators or coercions can change the indexed expression. Each generated - * history inserts rows, creates the serialized ref index, scans, and loads the - * subset. + * History grammar and domain: every generated matrix reaches equality, + * ordinary and 901-value batched IN, range, conjunction, ordering, and + * lower(ref) indexes. Paths have a one-to-six-character identifier root and up + * to three identifier/0..3 tail segments. Numeric targets are + * -10_000..10_000. String and boolean equality values keep independent + * distractors. Fixed cases cover constant-bearing coalesce/strftime/add, + * persisted Date ranges, and BigInt ranges/IN within SQLite's signed-integer + * domain. Explicitly qualified refs lower to the same JSON field expression + * without reinterpreting legacy nested paths. Known omissions: null, arbitrary + * raw SQL, native-host planning, and BigInts outside SQLite's signed range. * - * Independent model: a full adapter scan followed by a small path walker and - * strict scalar equality. It does not call the SQL compiler or reuse its path - * logic. + * Independent model: fixed keys encode the result of each generated relation; + * the equality witness also uses a full adapter scan, a small path walker, and + * strict scalar equality. Neither judgment calls the SQL compiler. * * Production path and checkpoint: the public SQLite-core adapter factory with * the real BetterSqlite3SQLiteDriver. The exact SQL and bindings passed to the - * driver's predicate query are captured, then replayed through EXPLAIN QUERY - * PLAN before cleanup. Result keys and the named expression index in the plan - * are separate observations; result order is outside this law. + * driver's predicate query are captured and executed directly before adapter + * re-filtering, then replayed through EXPLAIN QUERY PLAN. Result keys, ordering + * when promised, and named-index use are separate observations. * - * Reach, challenge, replay, and cleanup: every production case proves the - * index exists, the full-scan row count and target classification match the - * seed, the filter value stays bound, and indexed results equal the independent - * scan classification. Direct SQLite fault controls show that binding the DDL - * path is rejected and binding the predicate path returns the same rows but - * loses the index search. Replay a generated failure with TANSTACK_DB_WS5A_SEED - * and TANSTACK_DB_WS5A_PATH. Teardown retains the semantic failure as - * AggregateError.cause if cleanup also fails. + * Reach, challenge, replay, and cleanup: the property records and asserts every + * declared regime. An overbroad indexed-predicate mutant proves adapter + * re-filtering cannot hide wrong SQL. A production-driver-boundary mutant + * restores the old four path bindings and must lose the named-index search. + * Replay with TANSTACK_DB_WS5A_SEED and TANSTACK_DB_WS5A_PATH. In-memory SQLite + * teardown retains the semantic failure if cleanup also fails. */ -import { mkdtempSync, rmSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' import fc from 'fast-check' import { describe, expect, it } from 'vitest' import { IR } from '@tanstack/db' import { createPersistedTableName, createSQLiteCorePersistenceAdapter, + decodePersistedStorageKey, } from '@tanstack/db-sqlite-persistence-core' import { BetterSqlite3SQLiteDriver } from '../src/node-driver' -import type { SQLiteDriver } from '@tanstack/db-sqlite-persistence-core' +import type { + PersistedTx, + SQLiteDriver, +} from '@tanstack/db-sqlite-persistence-core' const DEFAULT_ORACLE_SEED = 1_659_005 -const DEFAULT_ORACLE_RUNS = 24 +const DEFAULT_ORACLE_RUNS = 8 +const SQLITE_BIGINT_MIN = -9_223_372_036_854_775_808n +const SQLITE_BIGINT_MAX = 9_223_372_036_854_775_807n +const PERSISTED_TYPE_TAG = `__tanstack_db_persisted_type__` +const PERSISTED_VALUE_TAG = `value` type OracleScalar = boolean | number | string @@ -70,6 +68,8 @@ type CapturedQuery = { params: ReadonlyArray } +type QueryTransform = (query: CapturedQuery) => CapturedQuery + type QueryPlanRow = { detail: string } @@ -127,6 +127,9 @@ function oracleRunConfiguration(): { } { const seedText = process.env.TANSTACK_DB_WS5A_SEED const path = process.env.TANSTACK_DB_WS5A_PATH + if (seedText !== undefined && !/^-?\d+$/.test(seedText)) { + throw new Error(`TANSTACK_DB_WS5A_SEED must be an integer`) + } const seed = seedText === undefined ? DEFAULT_ORACLE_SEED : Number(seedText) if (!Number.isSafeInteger(seed)) { @@ -215,15 +218,34 @@ function sqliteScalarParameter(value: OracleScalar): number | string { return typeof value === `boolean` ? (value ? 1 : 0) : value } +function serializeIndexExpression(expression: IR.BasicExpression): string { + return JSON.stringify(expression, (_key, value: unknown) => + typeof value === `bigint` + ? { + [PERSISTED_TYPE_TAG]: `bigint`, + [PERSISTED_VALUE_TAG]: value.toString(), + } + : value, + ) +} + +function bigintRangeError(value: bigint): string { + return `SQLite BigInt value ${value} is outside the signed 64-bit range [${SQLITE_BIGINT_MIN}, ${SQLITE_BIGINT_MAX}]` +} + function createQueryObservingDriver( inner: SQLiteDriver, observe: (query: CapturedQuery) => void, + transform?: QueryTransform, ): SQLiteDriver { const wrap = (driver: SQLiteDriver): SQLiteDriver => ({ exec: (sql) => driver.exec(sql), query: async (sql: string, params: ReadonlyArray = []) => { - observe({ sql, params: [...params] }) - return driver.query(sql, params) + const query = transform + ? transform({ sql, params: [...params] }) + : { sql, params: [...params] } + observe(query) + return driver.query(query.sql, query.params) }, run: (sql, params) => driver.run(sql, params), transaction: (body) => @@ -310,13 +332,437 @@ function sqlitePlanIdentifierPattern(identifier: string): string { return `(?:"${escaped}"|${escaped})` } +type ExpressionIndexScenario = { + label: string + indexExpression: IR.BasicExpression + where?: IR.BasicExpression + orderBy?: IR.OrderBy + preserveResultOrder?: boolean + rows: ReadonlyArray<{ + key: string + value: Record + }> + transformQuery?: QueryTransform +} + +const GENERATED_SCENARIO_KINDS = [ + `eq`, + `in`, + `batched-in`, + `range`, + `and`, + `order-by`, + `wrapped-lower`, +] as const + +type GeneratedScenarioKind = (typeof GENERATED_SCENARIO_KINDS)[number] + +type GeneratedExpressionIndexScenario = ExpressionIndexScenario & { + kind: GeneratedScenarioKind + expectedKeys: Array + expectedPlan: `search` | `ordered-scan` +} + +type ExpressionIndexObservation = { + adapterKeys: Array + directSqlKeys: Array + indexName: string + plan: Array + predicateQuery: CapturedQuery + tableName: string +} + +async function observeExpressionIndexScenario({ + label, + indexExpression, + where, + orderBy, + preserveResultOrder = false, + rows, + transformQuery, +}: ExpressionIndexScenario): Promise { + const baseDriver = new BetterSqlite3SQLiteDriver({ filename: `:memory:` }) + const collectionId = `expression-index-${label}` + const signature = `generated-expression` + const tableName = createPersistedTableName(collectionId, `c`) + let predicateQuery: CapturedQuery | undefined + + const observingDriver = createQueryObservingDriver( + baseDriver, + (query) => { + if ( + query.sql.includes(`SELECT key, value, metadata, row_version`) && + query.sql.includes(`FROM "${tableName}"`) && + (query.sql.includes(` WHERE `) || query.sql.includes(` ORDER BY `)) + ) { + predicateQuery = query + } + }, + transformQuery + ? (query) => + query.sql.includes(`SELECT key, value, metadata, row_version`) && + query.sql.includes(`FROM "${tableName}"`) && + (query.sql.includes(` WHERE `) || query.sql.includes(` ORDER BY `)) + ? transformQuery(query) + : query + : undefined, + ) + const adapter = createSQLiteCorePersistenceAdapter({ + driver: observingDriver, + }) + + try { + await adapter.applyCommittedTx(collectionId, { + txId: `seed-${label}`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: rows.map((row) => ({ + type: `insert` as const, + key: row.key, + value: row.value, + })), + }) + await adapter.ensureIndex(collectionId, signature, { + expressionSql: [serializeIndexExpression(indexExpression)], + }) + + const adapterRows = await adapter.loadSubset(collectionId, { + ...(where ? { where } : {}), + ...(orderBy ? { orderBy } : {}), + }) + if (!predicateQuery) { + throw new Error(`predicate query checkpoint was not reached`) + } + + const directSqlRows = baseDriver + .getDatabase() + .prepare(predicateQuery.sql) + .all(...predicateQuery.params) as Array<{ key: string }> + const registryRow = baseDriver + .getDatabase() + .prepare( + `SELECT index_name FROM persisted_index_registry + WHERE collection_id = ? AND signature = ?`, + ) + .get(collectionId, signature) as { index_name: string } | undefined + if (!registryRow) { + throw new Error(`expression index registry checkpoint was not reached`) + } + + const plan = baseDriver + .getDatabase() + .prepare(`EXPLAIN QUERY PLAN ${predicateQuery.sql}`) + .all(...predicateQuery.params) as Array + + const normalizeKeys = (keys: Array): Array => + preserveResultOrder ? keys : keys.sort() + + return { + adapterKeys: normalizeKeys(adapterRows.map((row) => String(row.key))), + directSqlKeys: normalizeKeys( + directSqlRows.map((row) => String(decodePersistedStorageKey(row.key))), + ), + indexName: registryRow.index_name, + plan, + predicateQuery, + tableName, + } + } finally { + baseDriver.close() + } +} + +function makeOverbroadEqualityMutation(query: CapturedQuery): CapturedQuery { + if (!query.sql.includes(` WHERE `)) return query + const sql = query.sql.replace(/ = \?\)$/, ` >= ?)`) + if (sql === query.sql) { + throw new Error(`overbroad equality mutation did not reach the predicate`) + } + return { sql, params: query.params } +} + +function makeLegacyPathBindingMutation(query: CapturedQuery): CapturedQuery { + if (!query.sql.includes(` WHERE `)) return query + + const pathParams: Array = [] + const sql = query.sql.replace( + /json_extract\(value, ('(?:''|[^'])+')\)/g, + (_match, literal: string) => { + pathParams.push(literal.slice(1, -1).replace(/''/g, `'`)) + return `json_extract(value, ?)` + }, + ) + if (pathParams.length !== 4) { + throw new Error( + `legacy path-binding mutation expected four ref paths, got ${pathParams.length}`, + ) + } + return { sql, params: [...pathParams, ...query.params] } +} + +function planUsesNamedIndexForOrdering( + plan: ReadonlyArray, + tableName: string, + indexName: string, +): boolean { + const tablePattern = sqlitePlanIdentifierPattern(tableName) + const indexPattern = sqlitePlanIdentifierPattern(indexName) + const orderedScanPattern = new RegExp( + `^SCAN(?: TABLE)? ${tablePattern} USING INDEX ${indexPattern}(?:\\s|$)`, + ) + return plan.some(({ detail }) => orderedScanPattern.test(detail)) +} + +const numericIndexCaseArbitrary = fc + .tuple(legalPathArbitrary, fc.integer({ min: -10_000, max: 10_000 })) + .map(([path, target]) => ({ path, target })) + +const stringIndexCaseArbitrary = fc + .tuple(legalPathArbitrary, fc.integer({ min: 0, max: 10_000 })) + .map(([path, suffix]) => ({ path, target: `target-'${suffix}` })) + +const generatedScenarioMatrixArbitrary = fc + .tuple( + expressionIndexCaseArbitrary, + numericIndexCaseArbitrary, + numericIndexCaseArbitrary, + numericIndexCaseArbitrary, + numericIndexCaseArbitrary, + numericIndexCaseArbitrary, + stringIndexCaseArbitrary, + ) + .map( + ([ + eqCase, + inCase, + batchedCase, + rangeCase, + andCase, + orderCase, + lowerCase, + ]) => { + const eqRows = [ + { + key: `eq-match-a`, + value: createNestedRow(eqCase.path, eqCase.target), + }, + { + key: `eq-different`, + value: createNestedRow(eqCase.path, eqCase.distractors[0]), + }, + { + key: `eq-match-b`, + value: createNestedRow(eqCase.path, eqCase.target), + }, + ] + const batchedValues = Array.from( + { length: 901 }, + (_unused, index) => batchedCase.target + index, + ) + const lowerTarget = lowerCase.target.toLowerCase() + + return [ + { + kind: `eq`, + label: `generated-eq`, + indexExpression: new IR.PropRef(eqCase.path), + where: new IR.Func(`eq`, [ + new IR.PropRef(eqCase.path), + new IR.Value(eqCase.target), + ]), + rows: eqRows, + expectedKeys: [`eq-match-a`, `eq-match-b`], + expectedPlan: `search`, + }, + { + kind: `in`, + label: `generated-in`, + indexExpression: new IR.PropRef(inCase.path), + where: new IR.Func(`in`, [ + new IR.PropRef(inCase.path), + new IR.Value([inCase.target - 1, inCase.target]), + ]), + rows: [ + { + key: `in-lower`, + value: createNestedRow(inCase.path, inCase.target - 1), + }, + { + key: `in-match`, + value: createNestedRow(inCase.path, inCase.target), + }, + { + key: `in-higher`, + value: createNestedRow(inCase.path, inCase.target + 1), + }, + ], + expectedKeys: [`in-lower`, `in-match`], + expectedPlan: `search`, + }, + { + kind: `batched-in`, + label: `generated-batched-in`, + indexExpression: new IR.PropRef(batchedCase.path), + where: new IR.Func(`in`, [ + new IR.PropRef(batchedCase.path), + new IR.Value(batchedValues), + ]), + rows: [ + { + key: `batch-first`, + value: createNestedRow(batchedCase.path, batchedCase.target), + }, + { + key: `batch-last`, + value: createNestedRow( + batchedCase.path, + batchedCase.target + 900, + ), + }, + { + key: `batch-outside`, + value: createNestedRow(batchedCase.path, batchedCase.target - 1), + }, + ], + expectedKeys: [`batch-first`, `batch-last`], + expectedPlan: `search`, + }, + { + kind: `range`, + label: `generated-range`, + indexExpression: new IR.PropRef(rangeCase.path), + where: new IR.Func(`gte`, [ + new IR.PropRef(rangeCase.path), + new IR.Value(rangeCase.target), + ]), + rows: [ + { + key: `range-lower`, + value: createNestedRow(rangeCase.path, rangeCase.target - 1), + }, + { + key: `range-match`, + value: createNestedRow(rangeCase.path, rangeCase.target), + }, + { + key: `range-higher`, + value: createNestedRow(rangeCase.path, rangeCase.target + 1), + }, + ], + expectedKeys: [`range-higher`, `range-match`], + expectedPlan: `search`, + }, + { + kind: `and`, + label: `generated-and`, + indexExpression: new IR.PropRef(andCase.path), + where: new IR.Func(`and`, [ + new IR.Func(`eq`, [ + new IR.PropRef(andCase.path), + new IR.Value(andCase.target), + ]), + new IR.Func(`eq`, [ + new IR.PropRef([`status`]), + new IR.Value(`active`), + ]), + ]), + rows: [ + { + key: `and-match`, + value: { + ...createNestedRow(andCase.path, andCase.target), + status: `active`, + }, + }, + { + key: `and-inactive`, + value: { + ...createNestedRow(andCase.path, andCase.target), + status: `inactive`, + }, + }, + { + key: `and-different`, + value: { + ...createNestedRow(andCase.path, andCase.target + 1), + status: `active`, + }, + }, + ], + expectedKeys: [`and-match`], + expectedPlan: `search`, + }, + { + kind: `order-by`, + label: `generated-order-by`, + indexExpression: new IR.PropRef(orderCase.path), + orderBy: [ + { + expression: new IR.PropRef(orderCase.path), + compareOptions: { + direction: `asc`, + nulls: `last`, + }, + }, + ], + preserveResultOrder: true, + rows: [ + { + key: `order-high`, + value: createNestedRow(orderCase.path, orderCase.target + 1), + }, + { + key: `order-low`, + value: createNestedRow(orderCase.path, orderCase.target - 1), + }, + { + key: `order-middle`, + value: createNestedRow(orderCase.path, orderCase.target), + }, + ], + expectedKeys: [`order-low`, `order-middle`, `order-high`], + expectedPlan: `ordered-scan`, + }, + { + kind: `wrapped-lower`, + label: `generated-wrapped-lower`, + indexExpression: new IR.Func(`lower`, [ + new IR.PropRef(lowerCase.path), + ]), + where: new IR.Func(`eq`, [ + new IR.Func(`lower`, [new IR.PropRef(lowerCase.path)]), + new IR.Value(lowerTarget), + ]), + rows: [ + { + key: `lower-upper`, + value: createNestedRow( + lowerCase.path, + lowerCase.target.toUpperCase(), + ), + }, + { + key: `lower-lower`, + value: createNestedRow(lowerCase.path, lowerTarget), + }, + { + key: `lower-other`, + value: createNestedRow(lowerCase.path, `${lowerTarget}-other`), + }, + ], + expectedKeys: [`lower-lower`, `lower-upper`], + expectedPlan: `search`, + }, + ] satisfies Array + }, + ) + async function assertExpressionIndexHistory( testCase: OracleCase, label: string, ): Promise { - const tempDirectory = mkdtempSync(join(tmpdir(), `db-expression-index-`)) - const databasePath = join(tempDirectory, `state.sqlite`) - const baseDriver = new BetterSqlite3SQLiteDriver({ filename: databasePath }) + const baseDriver = new BetterSqlite3SQLiteDriver({ filename: `:memory:` }) const collectionId = `expression-index-${label}` const signature = `generated-ref` const tableName = createPersistedTableName(collectionId, `c`) @@ -405,6 +851,19 @@ async function assertExpressionIndexHistory( `the filter value must remain a bound parameter`, ).toContain(sqliteScalarParameter(testCase.target)) + const directSqlKeys = ( + baseDriver + .getDatabase() + .prepare(predicateQuery.sql) + .all(...predicateQuery.params) as Array<{ key: string }> + ) + .map((row) => String(decodePersistedStorageKey(row.key))) + .sort() + expect( + directSqlKeys, + `captured SQL must produce the independent keys before in-memory filtering`, + ).toEqual(expectedKeys) + const registryRow = baseDriver .getDatabase() .prepare( @@ -447,22 +906,643 @@ async function assertExpressionIndexHistory( ), scannedCollectionTable: planScansTable(plan, tableName), } + const checkpointMessage = `predicate checkpoint must use only the matching expression index:\n${JSON.stringify(checkpoint, null, 2)}` + expect(checkpoint.usedNamedExpressionIndex, checkpointMessage).toBe(true) + expect(checkpoint.scannedCollectionTable, checkpointMessage).toBe(false) + }, [() => baseDriver.close()]) +} + +describe(`SQLite expression-index oracle`, () => { + it(`rejects an explicitly empty replay seed`, () => { + const previousSeed = process.env.TANSTACK_DB_WS5A_SEED + try { + process.env.TANSTACK_DB_WS5A_SEED = `` + expect(() => oracleRunConfiguration()).toThrow( + `TANSTACK_DB_WS5A_SEED must be an integer`, + ) + } finally { + if (previousSeed === undefined) { + delete process.env.TANSTACK_DB_WS5A_SEED + } else { + process.env.TANSTACK_DB_WS5A_SEED = previousSeed + } + } + }) + + it.each([ + { + label: `coalesce-constant`, + indexExpression: new IR.Func(`coalesce`, [ + new IR.PropRef([`nickname`]), + new IR.Value(`none`), + ]), + where: new IR.Func(`eq`, [ + new IR.Func(`coalesce`, [ + new IR.PropRef([`nickname`]), + new IR.Value(`none`), + ]), + new IR.Value(`none`), + ]), + rows: [ + { key: `missing`, value: { nickname: null } }, + { key: `present`, value: { nickname: `Ada` } }, + ], + expectedKeys: [`missing`], + }, + { + label: `strftime-constant`, + indexExpression: new IR.Func(`strftime`, [ + new IR.Value(`%Y-%m-%d`), + new IR.Func(`datetime`, [new IR.PropRef([`createdAt`])]), + ]), + where: new IR.Func(`eq`, [ + new IR.Func(`strftime`, [ + new IR.Value(`%Y-%m-%d`), + new IR.Func(`datetime`, [new IR.PropRef([`createdAt`])]), + ]), + new IR.Value(`2026-04-05`), + ]), + rows: [ + { key: `current`, value: { createdAt: `2026-04-05T00:00:00.000Z` } }, + { key: `past`, value: { createdAt: `2025-04-05T00:00:00.000Z` } }, + ], + expectedKeys: [`current`], + }, + { + label: `add-constant`, + indexExpression: new IR.Func(`add`, [ + new IR.PropRef([`score`]), + new IR.Value(1), + ]), + where: new IR.Func(`eq`, [ + new IR.Func(`add`, [new IR.PropRef([`score`]), new IR.Value(1)]), + new IR.Value(3), + ]), + rows: [ + { key: `matching`, value: { score: 2 } }, + { key: `different`, value: { score: 4 } }, + ], + expectedKeys: [`matching`], + }, + { + label: `bigint-coalesce-constant`, + indexExpression: new IR.Func(`coalesce`, [ + new IR.PropRef([`largeViewCount`]), + new IR.Value(SQLITE_BIGINT_MIN), + ]), + where: new IR.Func(`eq`, [ + new IR.Func(`coalesce`, [ + new IR.PropRef([`largeViewCount`]), + new IR.Value(SQLITE_BIGINT_MIN), + ]), + new IR.Value(SQLITE_BIGINT_MIN), + ]), + rows: [ + { key: `missing`, value: { largeViewCount: null } }, + { key: `minimum`, value: { largeViewCount: SQLITE_BIGINT_MIN } }, + { key: `zero`, value: { largeViewCount: 0n } }, + ], + expectedKeys: [`minimum`, `missing`], + }, + ])( + `uses a constant-bearing $label expression index`, + async ({ expectedKeys, ...scenario }) => { + const observation = await observeExpressionIndexScenario(scenario) + + expect(observation.adapterKeys).toEqual(expectedKeys) + expect(observation.directSqlKeys).toEqual(expectedKeys) + expect( + planUsesNamedIndex( + observation.plan, + observation.tableName, + observation.indexName, + ), + ).toBe(true) + }, + ) + + it.each([ + { + label: `bigint-field-range`, + indexExpression: new IR.PropRef([`largeViewCount`]), + where: new IR.Func(`gt`, [ + new IR.PropRef([`largeViewCount`]), + new IR.Value(BigInt(`9007199254740993`)), + ]), + rows: [ + { + key: `lower`, + value: { largeViewCount: BigInt(`9007199254740992`) }, + }, + { + key: `higher`, + value: { largeViewCount: BigInt(`9007199254740997`) }, + }, + ], + expectedKeys: [`higher`], + expectedQueryParams: [], + }, + { + label: `bigint-field-min-boundary`, + indexExpression: new IR.PropRef([`largeViewCount`]), + where: new IR.Func(`eq`, [ + new IR.PropRef([`largeViewCount`]), + new IR.Value(SQLITE_BIGINT_MIN), + ]), + rows: [ + { + key: `minimum`, + value: { largeViewCount: SQLITE_BIGINT_MIN }, + }, + { + key: `next`, + value: { largeViewCount: SQLITE_BIGINT_MIN + 1n }, + }, + ], + expectedKeys: [`minimum`], + expectedQueryParams: [], + }, + { + label: `bigint-field-max-boundary`, + indexExpression: new IR.PropRef([`largeViewCount`]), + where: new IR.Func(`eq`, [ + new IR.PropRef([`largeViewCount`]), + new IR.Value(SQLITE_BIGINT_MAX), + ]), + rows: [ + { + key: `previous`, + value: { largeViewCount: SQLITE_BIGINT_MAX - 1n }, + }, + { + key: `maximum`, + value: { largeViewCount: SQLITE_BIGINT_MAX }, + }, + ], + expectedKeys: [`maximum`], + expectedQueryParams: [], + }, + { + label: `date-field-range`, + indexExpression: new IR.PropRef([`createdAt`]), + where: new IR.Func(`gt`, [ + new IR.PropRef([`createdAt`]), + new IR.Value(new Date(`2026-01-02T12:00:00.000Z`)), + ]), + rows: [ + { + key: `earlier`, + value: { createdAt: new Date(`2026-01-02T00:00:00.000Z`) }, + }, + { + key: `later`, + value: { createdAt: new Date(`2026-01-03T00:00:00.000Z`) }, + }, + ], + expectedKeys: [`later`], + expectedQueryParams: [`2026-01-02T12:00:00.000Z`], + }, + { + label: `bigint-field-in`, + indexExpression: new IR.PropRef([`largeViewCount`]), + where: new IR.Func(`in`, [ + new IR.PropRef([`largeViewCount`]), + new IR.Value([BigInt(`9007199254740992`), BigInt(`9007199254740997`)]), + ]), + rows: [ + { + key: `included-low`, + value: { largeViewCount: BigInt(`9007199254740992`) }, + }, + { + key: `excluded`, + value: { largeViewCount: BigInt(`9007199254740994`) }, + }, + { + key: `included-high`, + value: { largeViewCount: BigInt(`9007199254740997`) }, + }, + ], + expectedKeys: [`included-high`, `included-low`], + expectedQueryParams: [`9007199254740992`, `9007199254740997`], + }, + { + label: `bigint-field-batched-in`, + indexExpression: new IR.PropRef([`largeViewCount`]), + where: new IR.Func(`in`, [ + new IR.PropRef([`largeViewCount`]), + new IR.Value( + Array.from( + { length: 901 }, + (_unused, index) => BigInt(`9007199254740992`) + BigInt(index), + ), + ), + ]), + rows: [ + { + key: `included-first`, + value: { largeViewCount: BigInt(`9007199254740992`) }, + }, + { + key: `excluded`, + value: { largeViewCount: BigInt(`9007199254740991`) }, + }, + { + key: `included-last`, + value: { largeViewCount: BigInt(`9007199254741892`) }, + }, + ], + expectedKeys: [`included-first`, `included-last`], + expectedQueryParams: Array.from({ length: 901 }, (_unused, index) => + (BigInt(`9007199254740992`) + BigInt(index)).toString(), + ), + }, + ])( + `uses the raw $label field expression index`, + async ({ expectedKeys, expectedQueryParams, ...scenario }) => { + const observation = await observeExpressionIndexScenario(scenario) + const diagnostic = JSON.stringify( + { + sql: observation.predicateQuery.sql, + params: observation.predicateQuery.params, + plan: observation.plan.map((row) => row.detail), + adapterKeys: observation.adapterKeys, + directSqlKeys: observation.directSqlKeys, + }, + null, + 2, + ) + + expect(observation.adapterKeys, diagnostic).toEqual(expectedKeys) + expect(observation.directSqlKeys, diagnostic).toEqual(expectedKeys) + expect(observation.predicateQuery.params, diagnostic).toEqual( + expectedQueryParams, + ) + expect( + planUsesNamedIndex( + observation.plan, + observation.tableName, + observation.indexName, + ), + diagnostic, + ).toBe(true) + }, + ) + + it.each([ + { + label: `nested row value`, + tx: { + txId: `out-of-range-row`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: [ + { + type: `insert`, + key: `row`, + value: { profile: { count: SQLITE_BIGINT_MAX + 1n } }, + }, + ], + } satisfies PersistedTx, + }, + { + label: `row metadata`, + tx: { + txId: `out-of-range-row-metadata`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: [{ type: `insert`, key: `row`, value: { id: `row` } }], + rowMetadataMutations: [ + { + type: `set`, + key: `row`, + value: { count: SQLITE_BIGINT_MIN - 1n }, + }, + ], + } satisfies PersistedTx, + }, + { + label: `collection metadata`, + tx: { + txId: `out-of-range-collection-metadata`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: [{ type: `insert`, key: `row`, value: { id: `row` } }], + collectionMetadataMutations: [ + { + type: `set`, + key: `checkpoint`, + value: { count: SQLITE_BIGINT_MAX + 1n }, + }, + ], + } satisfies PersistedTx, + }, + ])(`rejects an out-of-range BigInt in a persisted $label`, async ({ tx }) => { + const driver = new BetterSqlite3SQLiteDriver({ filename: `:memory:` }) + const adapter = createSQLiteCorePersistenceAdapter({ driver }) + + await withFailurePreservingCleanup(async () => { + await expect( + adapter.applyCommittedTx(`bigint-write-range`, tx), + ).rejects.toThrow( + /SQLite BigInt value .* outside the signed 64-bit range/, + ) + + if (adapter.scanRows) { + expect(await adapter.scanRows(`bigint-write-range`)).toEqual([]) + } + }, [() => driver.close()]) + }) + + it.each([ + { + label: `scalar comparison`, + rejectedValue: SQLITE_BIGINT_MAX + 1n, + where: new IR.Func(`gt`, [ + new IR.PropRef([`largeViewCount`]), + new IR.Value(SQLITE_BIGINT_MAX + 1n), + ]), + }, + { + label: `ordinary IN`, + rejectedValue: SQLITE_BIGINT_MIN - 1n, + where: new IR.Func(`in`, [ + new IR.PropRef([`largeViewCount`]), + new IR.Value([0n, SQLITE_BIGINT_MIN - 1n]), + ]), + }, + { + label: `batched IN`, + rejectedValue: SQLITE_BIGINT_MAX + 1n, + where: new IR.Func(`in`, [ + new IR.PropRef([`largeViewCount`]), + new IR.Value([ + ...Array.from({ length: 900 }, (_unused, index) => BigInt(index)), + SQLITE_BIGINT_MAX + 1n, + ]), + ]), + }, + ])( + `rejects an out-of-range BigInt in a $label before SQLite comparison`, + async ({ rejectedValue, where }) => { + const driver = new BetterSqlite3SQLiteDriver({ filename: `:memory:` }) + const adapter = createSQLiteCorePersistenceAdapter({ driver }) + const collectionId = `bigint-query-range` + + await withFailurePreservingCleanup(async () => { + await adapter.applyCommittedTx(collectionId, { + txId: `seed-bigint-query-range`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: [ + { + type: `insert`, + key: `zero`, + value: { largeViewCount: 0n }, + }, + ], + }) + + await expect( + adapter.loadSubset(collectionId, { where }), + ).rejects.toThrow(bigintRangeError(rejectedValue)) + }, [() => driver.close()]) + }, + ) + + it(`rejects a hostile serialized out-of-range BigInt index constant`, async () => { + const driver = new BetterSqlite3SQLiteDriver({ filename: `:memory:` }) + const adapter = createSQLiteCorePersistenceAdapter({ driver }) + const value = SQLITE_BIGINT_MAX + 1n + + await withFailurePreservingCleanup(async () => { + await expect( + adapter.ensureIndex(`bigint-index-range`, `out-of-range`, { + expressionSql: [ + serializeIndexExpression( + new IR.Func(`add`, [ + new IR.PropRef([`largeViewCount`]), + new IR.Value(value), + ]), + ), + ], + }), + ).rejects.toThrow(bigintRangeError(value)) + }, [() => driver.close()]) + }) + + it(`round-trips generated signed-64-bit BigInts through the real adapter`, async () => { + await fc.assert( + fc.asyncProperty( + fc.bigInt({ min: SQLITE_BIGINT_MIN, max: SQLITE_BIGINT_MAX }), + async (value) => { + const driver = new BetterSqlite3SQLiteDriver({ filename: `:memory:` }) + const adapter = createSQLiteCorePersistenceAdapter({ driver }) + const collectionId = `generated-bigint-in-range` + + await withFailurePreservingCleanup(async () => { + await adapter.applyCommittedTx(collectionId, { + txId: `seed-${value}`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: [ + { + type: `insert`, + key: `match`, + value: { nested: { count: value } }, + }, + ], + }) + + const rows = await adapter.loadSubset(collectionId, { + where: new IR.Func(`eq`, [ + new IR.PropRef([`nested`, `count`]), + new IR.Value(value), + ]), + }) + expect(rows.map((row) => row.key)).toEqual([`match`]) + expect(rows[0]?.value).toEqual({ nested: { count: value } }) + }, [() => driver.close()]) + }, + ), + oracleRunConfiguration(), + ) + }) + + it(`rejects generated BigInts immediately outside the signed range`, async () => { + const outOfRangeBigIntArbitrary = fc.oneof( + fc + .bigInt({ min: 1n, max: 1_000n }) + .map((distance) => SQLITE_BIGINT_MIN - distance), + fc + .bigInt({ min: 1n, max: 1_000n }) + .map((distance) => SQLITE_BIGINT_MAX + distance), + ) + + await fc.assert( + fc.asyncProperty(outOfRangeBigIntArbitrary, async (value) => { + const driver = new BetterSqlite3SQLiteDriver({ filename: `:memory:` }) + const adapter = createSQLiteCorePersistenceAdapter({ driver }) + + await withFailurePreservingCleanup(async () => { + await expect( + adapter.applyCommittedTx(`generated-bigint-out-of-range`, { + txId: `reject-${value}`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: [ + { + type: `insert`, + key: `rejected`, + value: { nested: { count: value } }, + }, + ], + }), + ).rejects.toThrow(bigintRangeError(value)) + }, [() => driver.close()]) + }), + oracleRunConfiguration(), + ) + }) + + it(`does not confuse an alias-qualified field ref with a nested JSON path`, async () => { + const observation = await observeExpressionIndexScenario({ + label: `alias-qualified-field`, + indexExpression: new IR.PropRef([`score`]), + where: new IR.Func(`gt`, [ + new IR.PropRef([`todos`, `score`], `todos`), + new IR.Value(1), + ]), + rows: [ + { key: `matching`, value: { score: 2 } }, + { key: `different`, value: { score: 0 } }, + ], + }) + const diagnostic = JSON.stringify( + { + sql: observation.predicateQuery.sql, + params: observation.predicateQuery.params, + plan: observation.plan.map((row) => row.detail), + adapterKeys: observation.adapterKeys, + directSqlKeys: observation.directSqlKeys, + }, + null, + 2, + ) + + expect(observation.adapterKeys, diagnostic).toEqual([`matching`]) + expect(observation.directSqlKeys, diagnostic).toEqual([`matching`]) + expect( + planUsesNamedIndex( + observation.plan, + observation.tableName, + observation.indexName, + ), + diagnostic, + ).toBe(true) + }) + + it(`does not guess that a legacy nested path is an alias`, async () => { + const baseDriver = new BetterSqlite3SQLiteDriver({ filename: `:memory:` }) + const adapter = createSQLiteCorePersistenceAdapter({ driver: baseDriver }) + const collectionId = `legacy-nested-path` + + await withFailurePreservingCleanup(async () => { + await adapter.applyCommittedTx(collectionId, { + txId: `seed-legacy-nested-path`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: [ + { + type: `insert`, + key: `nested-match`, + value: { + profile: { [`meta-field`]: `alpha` }, + [`meta-field`]: `flat-other`, + }, + }, + { + type: `insert`, + key: `flat-only`, + value: { [`meta-field`]: `alpha` }, + }, + ], + }) + + const rows = await adapter.loadSubset(collectionId, { + where: new IR.Func(`eq`, [ + new IR.PropRef([`profile`, `meta-field`]), + new IR.Value(`alpha`), + ]), + }) + + expect(rows.map((row) => row.key)).toEqual([`nested-match`]) + }, [() => baseDriver.close()]) + }) + + it(`exposes an overbroad indexed predicate hidden by in-memory re-filtering`, async () => { + const observation = await observeExpressionIndexScenario({ + label: `overbroad-indexed-predicate`, + indexExpression: new IR.PropRef([`score`]), + where: new IR.Func(`eq`, [ + new IR.PropRef([`score`]), + new IR.Value(2), + ]), + rows: [ + { key: `lower`, value: { score: 1 } }, + { key: `matching`, value: { score: 2 } }, + { key: `higher`, value: { score: 3 } }, + ], + transformQuery: makeOverbroadEqualityMutation, + }) + + expect(observation.adapterKeys).toEqual([`matching`]) expect( - checkpoint.usedNamedExpressionIndex, - `predicate checkpoint must use the matching expression index:\n${JSON.stringify(checkpoint, null, 2)}`, + planUsesNamedIndex( + observation.plan, + observation.tableName, + observation.indexName, + ), ).toBe(true) + expect(planScansTable(observation.plan, observation.tableName)).toBe(false) + expect(observation.directSqlKeys).toEqual([`higher`, `matching`]) + }) + + it(`kills the former path-binding compiler behavior at the driver boundary`, async () => { + const observation = await observeExpressionIndexScenario({ + label: `legacy-path-binding`, + indexExpression: new IR.PropRef([`score`]), + where: new IR.Func(`eq`, [ + new IR.PropRef([`score`]), + new IR.Value(2), + ]), + rows: [ + { key: `matching`, value: { score: 2 } }, + { key: `different`, value: { score: 3 } }, + ], + transformQuery: makeLegacyPathBindingMutation, + }) + + expect(observation.adapterKeys).toEqual([`matching`]) + expect(observation.directSqlKeys).toEqual([`matching`]) expect( - checkpoint.scannedCollectionTable, - `predicate checkpoint must not scan the collection table:\n${JSON.stringify(checkpoint, null, 2)}`, + planUsesNamedIndex( + observation.plan, + observation.tableName, + observation.indexName, + ), ).toBe(false) - }, [ - () => baseDriver.close(), - () => rmSync(tempDirectory, { recursive: true, force: true }), - ]) -} + expect(planScansTable(observation.plan, observation.tableName)).toBe(true) + }) -describe(`SQLite expression-index oracle`, () => { it(`recognizes equivalent SQLite plan identifier formats without prefix collisions`, () => { const tableName = `rows` const indexName = `literal_ddl` @@ -497,9 +1577,7 @@ describe(`SQLite expression-index oracle`, () => { }) it(`distinguishes rejected DDL path binding from correct predicate rows without index use`, async () => { - const tempDirectory = mkdtempSync(join(tmpdir(), `db-index-controls-`)) - const databasePath = join(tempDirectory, `state.sqlite`) - const driver = new BetterSqlite3SQLiteDriver({ filename: databasePath }) + const driver = new BetterSqlite3SQLiteDriver({ filename: `:memory:` }) const database = driver.getDatabase() const jsonPath = `$.payload.threadId` const target = `thread-'1` @@ -546,10 +1624,7 @@ describe(`SQLite expression-index oracle`, () => { expect(planScansTable(literalPlan, `rows`)).toBe(false) expect(planUsesNamedIndex(boundPlan, `rows`, `literal_ddl`)).toBe(false) expect(planScansTable(boundPlan, `rows`)).toBe(true) - }, [ - () => driver.close(), - () => rmSync(tempDirectory, { recursive: true, force: true }), - ]) + }, [() => driver.close()]) }) it(`uses the expression index for the fixed filter witness`, async () => { @@ -563,15 +1638,67 @@ describe(`SQLite expression-index oracle`, () => { ) }) - it(`uses matching expression indexes for generated legal paths and values`, async () => { + it(`reaches every declared generated expression-index scenario`, async () => { + const reachedScenarios = new Set() + await fc.assert( - fc.asyncProperty(expressionIndexCaseArbitrary, async (testCase) => { - await assertExpressionIndexHistory(testCase, `generated`) + fc.asyncProperty(generatedScenarioMatrixArbitrary, async (scenarios) => { + for (const { + kind, + expectedKeys, + expectedPlan, + ...scenario + } of scenarios) { + reachedScenarios.add(kind) + const observation = await observeExpressionIndexScenario(scenario) + const diagnostic = JSON.stringify( + { + kind, + sql: observation.predicateQuery.sql, + params: observation.predicateQuery.params, + plan: observation.plan.map((row) => row.detail), + adapterKeys: observation.adapterKeys, + directSqlKeys: observation.directSqlKeys, + }, + null, + 2, + ) + + expect(observation.adapterKeys, diagnostic).toEqual(expectedKeys) + expect(observation.directSqlKeys, diagnostic).toEqual(expectedKeys) + if (expectedPlan === `ordered-scan`) { + expect( + planUsesNamedIndexForOrdering( + observation.plan, + observation.tableName, + observation.indexName, + ), + diagnostic, + ).toBe(true) + } else { + expect( + planUsesNamedIndex( + observation.plan, + observation.tableName, + observation.indexName, + ), + diagnostic, + ).toBe(true) + expect( + planScansTable(observation.plan, observation.tableName), + diagnostic, + ).toBe(false) + } + } }), { ...oracleRunConfiguration(), verbose: 2, }, ) + + expect([...reachedScenarios].sort()).toEqual( + [...GENERATED_SCENARIO_KINDS].sort(), + ) }) }) From 29af470acb027e62bf786b6af13e52668780ed13 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 23 Sep 2026 12:35:37 +0100 Subject: [PATCH 6/8] fix(sqlite): rebuild upgraded expression indexes --- .../fix-sqlite-expression-index-planning.md | 2 +- .../src/sqlite-core-adapter.ts | 32 ++++++++++++- .../tests/sqlite-core-adapter.test.ts | 37 +++++++++++++++ .../tests/expression-index-oracle.test.ts | 46 +++++++++++++++++++ 4 files changed, 114 insertions(+), 3 deletions(-) diff --git a/.changeset/fix-sqlite-expression-index-planning.md b/.changeset/fix-sqlite-expression-index-planning.md index d22921dd29..4b57d1b592 100644 --- a/.changeset/fix-sqlite-expression-index-planning.md +++ b/.changeset/fix-sqlite-expression-index-planning.md @@ -3,4 +3,4 @@ '@tanstack/db-sqlite-persistence-core': patch --- -Preserve explicit source aliases without changing legacy property paths. Compile SQLite expression-index queries consistently and reject BigInts outside SQLite's signed 64-bit range. +Preserve explicit source aliases without changing legacy property paths. Compile SQLite expression-index queries consistently, rebuild affected stale physical indexes, and reject BigInts outside SQLite's signed 64-bit range. diff --git a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts index e03d01f91c..fc7dbf5f8a 100644 --- a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts +++ b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts @@ -1457,11 +1457,39 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { normalizeIndexSqlFragment(fragment), ) const expressionSql = normalizedExpressionSql.join(`, `) + const persistedExpressionSql = JSON.stringify(normalizedExpressionSql) const whereSql = spec.whereSql ? normalizeIndexSqlFragment(spec.whereSql) : undefined + const persistedWhereSql = whereSql ?? null await this.runInTransaction(async (transactionDriver) => { + const existingRows = await transactionDriver.query<{ + index_name: string + expression_sql: string + where_sql: string | null + }>( + `SELECT index_name, expression_sql, where_sql + FROM persisted_index_registry + WHERE collection_id = ? AND signature = ? + LIMIT 1`, + [collectionId, signature], + ) + const existing = existingRows[0] + if ( + existing && + (existing.index_name !== indexName || + existing.expression_sql !== persistedExpressionSql || + existing.where_sql !== persistedWhereSql) + ) { + // A compiler upgrade can change normalized SQL without changing the + // logical index signature. Rebuild only that stale physical index so + // the registry and SQLite planner describe the same expression. + await transactionDriver.exec( + `DROP INDEX IF EXISTS ${quoteIdentifier(existing.index_name)}`, + ) + } + await transactionDriver.run( `INSERT INTO persisted_index_registry ( collection_id, @@ -1489,8 +1517,8 @@ export class SQLiteCorePersistenceAdapter implements PersistenceAdapter { collectionId, signature, indexName, - JSON.stringify(normalizedExpressionSql), - whereSql ?? null, + persistedExpressionSql, + persistedWhereSql, ], ) diff --git a/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts b/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts index 532ef55749..03bb927347 100644 --- a/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts +++ b/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts @@ -947,6 +947,43 @@ export function runSQLiteCoreAdapterContractSuite( expect(sqliteMasterAfter).toHaveLength(0) }) + it(`rebuilds a physical index when the normalized spec changes`, async () => { + const { adapter, driver } = registerContractHarness() + const collectionId = `todos` + const signature = `idx-upgraded-expression` + + await adapter.ensureIndex(collectionId, signature, { + expressionSql: [`json_extract(value, '$.title')`], + }) + + const registryRows = await driver.query<{ index_name: string }>( + `SELECT index_name + FROM persisted_index_registry + WHERE collection_id = ? AND signature = ?`, + [collectionId, signature], + ) + const indexName = registryRows[0]?.index_name + expect(indexName).toBeTruthy() + + await adapter.ensureIndex(collectionId, signature, { + expressionSql: [`json_extract(value, '$.score')`], + }) + + const sqliteMasterRows = await driver.query<{ sql: string }>( + `SELECT sql + FROM sqlite_master + WHERE type = 'index' AND name = ?`, + [indexName], + ) + expect(sqliteMasterRows).toHaveLength(1) + expect(sqliteMasterRows[0]?.sql).toContain( + `json_extract(value, '$.score')`, + ) + expect(sqliteMasterRows[0]?.sql).not.toContain( + `json_extract(value, '$.title')`, + ) + }) + it(`enforces schema mismatch policies`, async () => { const baseHarness = registerContractHarness({ schemaVersion: 1, diff --git a/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts b/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts index 79160fdf78..dd0279fdc8 100644 --- a/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts +++ b/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts @@ -342,6 +342,11 @@ type ExpressionIndexScenario = { key: string value: Record }> + preparePreviousIndex?: ( + adapter: ReturnType, + collectionId: string, + signature: string, + ) => Promise transformQuery?: QueryTransform } @@ -379,6 +384,7 @@ async function observeExpressionIndexScenario({ orderBy, preserveResultOrder = false, rows, + preparePreviousIndex, transformQuery, }: ExpressionIndexScenario): Promise { const baseDriver = new BetterSqlite3SQLiteDriver({ filename: `:memory:` }) @@ -423,6 +429,7 @@ async function observeExpressionIndexScenario({ value: row.value, })), }) + await preparePreviousIndex?.(adapter, collectionId, signature) await adapter.ensureIndex(collectionId, signature, { expressionSql: [serializeIndexExpression(indexExpression)], }) @@ -1022,6 +1029,45 @@ describe(`SQLite expression-index oracle`, () => { }, ) + it(`rebuilds a persisted BigInt-constant index when its normalized spec changes`, async () => { + const indexExpression = new IR.Func(`coalesce`, [ + new IR.PropRef([`largeViewCount`]), + new IR.Value(SQLITE_BIGINT_MIN), + ]) + const observation = await observeExpressionIndexScenario({ + label: `bigint-constant-upgrade`, + indexExpression, + where: new IR.Func(`eq`, [ + indexExpression, + new IR.Value(SQLITE_BIGINT_MIN), + ]), + rows: [ + { key: `missing`, value: { largeViewCount: null } }, + { key: `minimum`, value: { largeViewCount: SQLITE_BIGINT_MIN } }, + { key: `zero`, value: { largeViewCount: 0n } }, + ], + preparePreviousIndex: async (adapter, collectionId, signature) => { + await adapter.ensureIndex(collectionId, signature, { + expressionSql: [ + JSON.stringify(indexExpression, (_key, value: unknown) => + typeof value === `bigint` ? value.toString() : value, + ), + ], + }) + }, + }) + + expect(observation.adapterKeys).toEqual([`minimum`, `missing`]) + expect(observation.directSqlKeys).toEqual([`minimum`, `missing`]) + expect( + planUsesNamedIndex( + observation.plan, + observation.tableName, + observation.indexName, + ), + ).toBe(true) + }) + it.each([ { label: `bigint-field-range`, From 836b4e6f7d8770f7b77dc65816e777ed64642aff Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 23 Sep 2026 15:17:50 +0100 Subject: [PATCH 7/8] test(sqlite): strengthen expression-index oracle replay --- docs/contributing/oracle-coverage.md | 9 + .../tests/expression-index-oracle.test.ts | 521 ++++++++++++++---- 2 files changed, 437 insertions(+), 93 deletions(-) diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 1dbfdb5de8..e54674aea0 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -183,6 +183,15 @@ target execution, not reproduction of a particular bug. Local IVM and offline properties have separate environment variables; inspect their test headers. Do not assume the core multiplier reaches them. +The Node expression-index oracle runs matching fixed-seed and seedless-random +campaigns by default. Supplying both replay values selects only the requested +seed and shrink path: + +```sh +TANSTACK_DB_WS5A_SEED=1659005 TANSTACK_DB_WS5A_PATH=0 \ +pnpm --filter @tanstack/node-db-sqlite-persistence test:oracles +``` + Stress runs need an explicit file list, run budget, seed policy, runtime, exit status and cost. For long synchronous campaigns, yield **between complete histories**, never between an action and its synchronous observation. In this diff --git a/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts b/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts index dd0279fdc8..97757fe48f 100644 --- a/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts +++ b/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts @@ -27,15 +27,19 @@ * re-filtering, then replayed through EXPLAIN QUERY PLAN. Result keys, ordering * when promised, and named-index use are separate observations. * - * Reach, challenge, replay, and cleanup: the property records and asserts every - * declared regime. An overbroad indexed-predicate mutant proves adapter - * re-filtering cannot hide wrong SQL. A production-driver-boundary mutant - * restores the old four path bindings and must lose the named-index search. - * Replay with TANSTACK_DB_WS5A_SEED and TANSTACK_DB_WS5A_PATH. In-memory SQLite - * teardown retains the semantic failure if cleanup also fails. + * Reach, challenge, replay, and cleanup: one property, generator matrix, and + * observation path run in retained fixed-seed and seedless-random campaigns. + * Supplying both TANSTACK_DB_WS5A_SEED and TANSTACK_DB_WS5A_PATH selects only + * the exact replay campaign; either value alone rejects. Per-axis grammar + * ablations and same-path SQL/compiler faults must fail that property. The + * retained fixed campaign reconstructs the known valid matrix. The bounds + * above state its range; exact grammar checks reject missing, duplicate, and + * unexpected axes as nearby invalid matrices. The focused overbroad and former + * four-path-binding controls remain independent. In-memory SQLite teardown + * retains the semantic failure if cleanup also fails. */ import fc from 'fast-check' -import { describe, expect, it } from 'vitest' +import { describe, expect, it as vitestIt } from 'vitest' import { IR } from '@tanstack/db' import { createPersistedTableName, @@ -120,19 +124,28 @@ const expressionIndexCaseArbitrary = fc .tuple(legalPathArbitrary, independentValuesArbitrary) .map(([path, values]): OracleCase => ({ path, ...values })) -function oracleRunConfiguration(): { +type OracleReplayConfiguration = { seed: number + path: string + numRuns: 1 +} + +type GeneratedScenarioCampaign = { + name: `fixed` | `random` | `replay` + seed?: number path?: string numRuns: number -} { - const seedText = process.env.TANSTACK_DB_WS5A_SEED - const path = process.env.TANSTACK_DB_WS5A_PATH +} + +function oracleReplayConfiguration( + environment: NodeJS.ProcessEnv = process.env, +): OracleReplayConfiguration | undefined { + const seedText = environment.TANSTACK_DB_WS5A_SEED + const path = environment.TANSTACK_DB_WS5A_PATH if (seedText !== undefined && !/^-?\d+$/.test(seedText)) { throw new Error(`TANSTACK_DB_WS5A_SEED must be an integer`) } - const seed = seedText === undefined ? DEFAULT_ORACLE_SEED : Number(seedText) - - if (!Number.isSafeInteger(seed)) { + if (seedText !== undefined && !Number.isSafeInteger(Number(seedText))) { throw new Error(`TANSTACK_DB_WS5A_SEED must be an integer`) } if (path !== undefined && !/^\d+(?::\d+)*$/.test(path)) { @@ -140,11 +153,52 @@ function oracleRunConfiguration(): { `TANSTACK_DB_WS5A_PATH must contain colon-separated nonnegative integers`, ) } + if (seedText === undefined && path === undefined) return undefined + if (seedText === undefined) { + throw new Error( + `TANSTACK_DB_WS5A_PATH requires TANSTACK_DB_WS5A_SEED for exact replay`, + ) + } + if (path === undefined) { + throw new Error( + `TANSTACK_DB_WS5A_SEED requires TANSTACK_DB_WS5A_PATH for exact replay`, + ) + } + + return { + seed: Number(seedText), + path, + numRuns: 1, + } +} + +const requestedReplay = oracleReplayConfiguration() + +const generatedScenarioCampaigns: Array = + requestedReplay === undefined + ? [ + { + name: `fixed`, + seed: DEFAULT_ORACLE_SEED, + numRuns: DEFAULT_ORACLE_RUNS, + }, + { name: `random`, numRuns: DEFAULT_ORACLE_RUNS }, + ] + : [{ name: `replay`, ...requestedReplay }] +function generatedCampaignParameters(campaign: GeneratedScenarioCampaign) { return { - seed, - ...(path === undefined ? {} : { path }), - numRuns: path === undefined ? DEFAULT_ORACLE_RUNS : 1, + numRuns: campaign.numRuns, + verbose: 2 as const, + ...(campaign.seed === undefined ? {} : { seed: campaign.seed }), + ...(campaign.path === undefined ? {} : { path: campaign.path }), + } +} + +function fixedOracleRunConfiguration() { + return { + seed: DEFAULT_ORACLE_SEED, + numRuns: DEFAULT_ORACLE_RUNS, } } @@ -362,6 +416,19 @@ const GENERATED_SCENARIO_KINDS = [ type GeneratedScenarioKind = (typeof GENERATED_SCENARIO_KINDS)[number] +const GENERATED_SCENARIO_FAULTS = [ + { kind: `eq`, wrongAnswer: `inverted equality operator` }, + { kind: `in`, wrongAnswer: `shifted first IN binding` }, + { kind: `batched-in`, wrongAnswer: `replaced final batch member` }, + { kind: `range`, wrongAnswer: `exclusive lower boundary` }, + { kind: `and`, wrongAnswer: `wrong conjunct binding` }, + { kind: `order-by`, wrongAnswer: `reversed ordering direction` }, + { kind: `wrapped-lower`, wrongAnswer: `wrong expression wrapper` }, +] as const satisfies ReadonlyArray<{ + kind: GeneratedScenarioKind + wrongAnswer: string +}> + type GeneratedExpressionIndexScenario = ExpressionIndexScenario & { kind: GeneratedScenarioKind expectedKeys: Array @@ -765,6 +832,198 @@ const generatedScenarioMatrixArbitrary = fc }, ) +function assertGeneratedScenarioGrammar( + scenarios: ReadonlyArray, +): void { + const actualKinds = scenarios.map(({ kind }) => kind) + const missingKinds = GENERATED_SCENARIO_KINDS.filter( + (kind) => !actualKinds.includes(kind), + ) + const unexpectedKinds = actualKinds.filter( + (kind) => !GENERATED_SCENARIO_KINDS.includes(kind), + ) + const duplicateKinds = actualKinds.filter( + (kind, index) => actualKinds.indexOf(kind) !== index, + ) + + if ( + missingKinds.length > 0 || + unexpectedKinds.length > 0 || + duplicateKinds.length > 0 + ) { + throw new Error( + `generated scenario grammar mismatch: missing=${missingKinds.join(`,`) || `none`}; unexpected=${unexpectedKinds.join(`,`) || `none`}; duplicate=${duplicateKinds.join(`,`) || `none`}`, + ) + } +} + +function mutateGeneratedScenarioQuery( + kind: GeneratedScenarioKind, +): QueryTransform { + const unreached = (detail: string): never => { + throw new Error(`${kind} SQL/compiler fault did not reach ${detail}`) + } + + switch (kind) { + case `eq`: + return (query) => { + const sql = query.sql.replace(/ = \?(\)?)$/, ` != ?$1`) + if (sql === query.sql) return unreached(`the equality operator`) + return { sql, params: query.params } + } + case `in`: + return (query) => { + if (!query.sql.includes(` IN (`) || query.params.length !== 2) { + return unreached(`the ordinary IN bindings`) + } + const first = query.params[0] + if (typeof first !== `number`) { + return unreached(`a numeric ordinary IN binding`) + } + const params = [...query.params] + params[0] = first + 2 + return { sql: query.sql, params } + } + case `batched-in`: + return (query) => { + if (!query.sql.includes(` IN (`) || query.params.length !== 901) { + return unreached(`the 901 batched IN bindings`) + } + const first = query.params[0] + if (typeof first !== `number`) { + return unreached(`a numeric batched IN binding`) + } + const params = [...query.params] + params[params.length - 1] = first - 1 + return { sql: query.sql, params } + } + case `range`: + return (query) => { + const sql = query.sql.replace(/ >= \?/, ` > ?`) + if (sql === query.sql) return unreached(`the inclusive range operator`) + return { sql, params: query.params } + } + case `and`: + return (query) => { + const activeIndex = query.params.indexOf(`active`) + if (activeIndex === -1) return unreached(`the conjunction binding`) + const params = [...query.params] + params[activeIndex] = `inactive` + return { sql: query.sql, params } + } + case `order-by`: + return (query) => { + const sql = query.sql.replace(/\sASC\b/, ` DESC`) + if (sql === query.sql) return unreached(`the ascending order clause`) + return { sql, params: query.params } + } + case `wrapped-lower`: + return (query) => { + const sql = query.sql.replace(/\blower\(/gi, `upper(`) + if (sql === query.sql) return unreached(`the lower wrapper`) + return { sql, params: query.params } + } + } +} + +function assertGeneratedScenarioObservation( + scenario: GeneratedExpressionIndexScenario, + observation: ExpressionIndexObservation, +): void { + const diagnostic = JSON.stringify( + { + kind: scenario.kind, + sql: observation.predicateQuery.sql, + params: observation.predicateQuery.params, + plan: observation.plan.map((row) => row.detail), + adapterKeys: observation.adapterKeys, + directSqlKeys: observation.directSqlKeys, + }, + null, + 2, + ) + + expect(observation.adapterKeys, diagnostic).toEqual(scenario.expectedKeys) + expect(observation.directSqlKeys, diagnostic).toEqual(scenario.expectedKeys) + if (scenario.expectedPlan === `ordered-scan`) { + expect( + planUsesNamedIndexForOrdering( + observation.plan, + observation.tableName, + observation.indexName, + ), + diagnostic, + ).toBe(true) + } else { + expect( + planUsesNamedIndex( + observation.plan, + observation.tableName, + observation.indexName, + ), + diagnostic, + ).toBe(true) + expect( + planScansTable(observation.plan, observation.tableName), + diagnostic, + ).toBe(false) + } +} + +type GeneratedScenarioPropertyOptions = { + ablateKind?: GeneratedScenarioKind + faultKind?: GeneratedScenarioKind + onScenario?: (kind: GeneratedScenarioKind) => void +} + +function generatedScenarioProperty({ + ablateKind, + faultKind, + onScenario, +}: GeneratedScenarioPropertyOptions = {}) { + return fc.asyncProperty(generatedScenarioMatrixArbitrary, async (matrix) => { + const scenarios = + ablateKind === undefined + ? matrix + : matrix.filter(({ kind }) => kind !== ablateKind) + assertGeneratedScenarioGrammar(scenarios) + + for (const scenario of scenarios) { + onScenario?.(scenario.kind) + const observation = await observeExpressionIndexScenario({ + ...scenario, + ...(faultKind === scenario.kind + ? { transformQuery: mutateGeneratedScenarioQuery(scenario.kind) } + : {}), + }) + assertGeneratedScenarioObservation(scenario, observation) + } + }) +} + +async function runGeneratedScenarioProperty( + campaign: GeneratedScenarioCampaign, +) { + const reachedScenarios = new Set() + const details = await fc.check( + generatedScenarioProperty({ + onScenario: (kind) => reachedScenarios.add(kind), + }), + generatedCampaignParameters(campaign), + ) + + if (details.failed) { + throw new Error( + fc.defaultReportMessage(details) ?? + `generated expression-index property failed without a report`, + ) + } + expect([...reachedScenarios].sort()).toEqual( + [...GENERATED_SCENARIO_KINDS].sort(), + ) + return details +} + async function assertExpressionIndexHistory( testCase: OracleCase, label: string, @@ -921,20 +1180,44 @@ async function assertExpressionIndexHistory( } describe(`SQLite expression-index oracle`, () => { - it(`rejects an explicitly empty replay seed`, () => { - const previousSeed = process.env.TANSTACK_DB_WS5A_SEED - try { - process.env.TANSTACK_DB_WS5A_SEED = `` - expect(() => oracleRunConfiguration()).toThrow( - `TANSTACK_DB_WS5A_SEED must be an integer`, - ) - } finally { - if (previousSeed === undefined) { - delete process.env.TANSTACK_DB_WS5A_SEED - } else { - process.env.TANSTACK_DB_WS5A_SEED = previousSeed - } - } + const it = requestedReplay === undefined ? vitestIt : vitestIt.skip + + it.each([ + { + label: `empty seed`, + environment: { TANSTACK_DB_WS5A_SEED: `` }, + message: `TANSTACK_DB_WS5A_SEED must be an integer`, + }, + { + label: `seed without path`, + environment: { TANSTACK_DB_WS5A_SEED: `1659005` }, + message: `TANSTACK_DB_WS5A_SEED requires TANSTACK_DB_WS5A_PATH for exact replay`, + }, + { + label: `path without seed`, + environment: { TANSTACK_DB_WS5A_PATH: `0` }, + message: `TANSTACK_DB_WS5A_PATH requires TANSTACK_DB_WS5A_SEED for exact replay`, + }, + { + label: `invalid shrink path`, + environment: { + TANSTACK_DB_WS5A_SEED: `1659005`, + TANSTACK_DB_WS5A_PATH: `0:`, + }, + message: `TANSTACK_DB_WS5A_PATH must contain colon-separated nonnegative integers`, + }, + ])(`rejects a $label replay configuration`, ({ environment, message }) => { + expect(() => oracleReplayConfiguration(environment)).toThrow(message) + }) + + it(`accepts only a checked seed and shrink-path replay pair`, () => { + expect( + oracleReplayConfiguration({ + TANSTACK_DB_WS5A_SEED: `-1659005`, + TANSTACK_DB_WS5A_PATH: `0:1:2`, + }), + ).toEqual({ seed: -1659005, path: `0:1:2`, numRuns: 1 }) + expect(oracleReplayConfiguration({})).toBeUndefined() }) it.each([ @@ -1417,7 +1700,7 @@ describe(`SQLite expression-index oracle`, () => { }, [() => driver.close()]) }, ), - oracleRunConfiguration(), + fixedOracleRunConfiguration(), ) }) @@ -1454,7 +1737,7 @@ describe(`SQLite expression-index oracle`, () => { ).rejects.toThrow(bigintRangeError(value)) }, [() => driver.close()]) }), - oracleRunConfiguration(), + fixedOracleRunConfiguration(), ) }) @@ -1684,67 +1967,119 @@ describe(`SQLite expression-index oracle`, () => { ) }) - it(`reaches every declared generated expression-index scenario`, async () => { - const reachedScenarios = new Set() + vitestIt.each(generatedScenarioCampaigns)( + `reaches every declared generated expression-index scenario in the $name campaign`, + async (campaign) => { + const details = await runGeneratedScenarioProperty(campaign) + const parameters = generatedCampaignParameters(campaign) + + expect(details.numRuns).toBe(campaign.numRuns) + expect(`seed` in parameters).toBe(campaign.seed !== undefined) + expect(`path` in parameters).toBe(campaign.path !== undefined) + if (campaign.seed !== undefined) { + expect(details.seed).toBe(campaign.seed) + } + if (campaign.path !== undefined) { + expect(details.runConfiguration.path).toBe(campaign.path) + } + expect(details.seed).toEqual(expect.any(Number)) + }, + ) - await fc.assert( - fc.asyncProperty(generatedScenarioMatrixArbitrary, async (scenarios) => { - for (const { - kind, - expectedKeys, - expectedPlan, - ...scenario - } of scenarios) { - reachedScenarios.add(kind) - const observation = await observeExpressionIndexScenario(scenario) - const diagnostic = JSON.stringify( - { - kind, - sql: observation.predicateQuery.sql, - params: observation.predicateQuery.params, - plan: observation.plan.map((row) => row.detail), - adapterKeys: observation.adapterKeys, - directSqlKeys: observation.directSqlKeys, - }, - null, - 2, - ) + const generatedControlTest = + requestedReplay === undefined ? vitestIt : vitestIt.skip - expect(observation.adapterKeys, diagnostic).toEqual(expectedKeys) - expect(observation.directSqlKeys, diagnostic).toEqual(expectedKeys) - if (expectedPlan === `ordered-scan`) { - expect( - planUsesNamedIndexForOrdering( - observation.plan, - observation.tableName, - observation.indexName, - ), - diagnostic, - ).toBe(true) - } else { - expect( - planUsesNamedIndex( - observation.plan, - observation.tableName, - observation.indexName, - ), - diagnostic, - ).toBe(true) - expect( - planScansTable(observation.plan, observation.tableName), - diagnostic, - ).toBe(false) - } - } - }), - { - ...oracleRunConfiguration(), - verbose: 2, - }, - ) + generatedControlTest.each(GENERATED_SCENARIO_KINDS)( + `rejects the generated grammar when the %s axis is ablated`, + async (kind) => { + const details = await fc.check( + generatedScenarioProperty({ ablateKind: kind }), + { + seed: DEFAULT_ORACLE_SEED, + numRuns: 1, + verbose: 2, + }, + ) + const evidence = JSON.stringify({ + seed: details.seed, + path: details.counterexamplePath, + error: details.error, + }) - expect([...reachedScenarios].sort()).toEqual( - [...GENERATED_SCENARIO_KINDS].sort(), - ) - }) + expect(details.failed, evidence).toBe(true) + expect(details.seed, evidence).toBe(DEFAULT_ORACLE_SEED) + expect(details.counterexamplePath, evidence).not.toBeNull() + expect(details.error, evidence).toContain(`missing=${kind}`) + }, + ) + + generatedControlTest( + `reconstructs the retained generated grammar and rejects a nearby duplicate axis`, + () => { + const [matrix] = fc.sample(generatedScenarioMatrixArbitrary, { + seed: DEFAULT_ORACLE_SEED, + numRuns: 1, + }) + if (!matrix) throw new Error(`fixed grammar witness was not generated`) + assertGeneratedScenarioGrammar(matrix) + expect(GENERATED_SCENARIO_FAULTS.map(({ kind }) => kind)).toEqual( + GENERATED_SCENARIO_KINDS, + ) + const equalityScenario = matrix[0] + if (!equalityScenario) { + throw new Error(`fixed grammar witness omitted equality`) + } + expect(() => + assertGeneratedScenarioGrammar([...matrix, equalityScenario]), + ).toThrow(`duplicate=eq`) + }, + ) + + generatedControlTest.each(GENERATED_SCENARIO_FAULTS)( + `rejects the plausible $wrongAnswer for the $kind axis`, + async ({ kind }) => { + const details = await fc.check( + generatedScenarioProperty({ faultKind: kind }), + { + seed: DEFAULT_ORACLE_SEED, + numRuns: 1, + verbose: 2, + }, + ) + const evidence = JSON.stringify({ + seed: details.seed, + path: details.counterexamplePath, + error: details.error, + }) + + expect(details.failed, evidence).toBe(true) + expect(details.seed, evidence).toBe(DEFAULT_ORACLE_SEED) + expect(details.counterexamplePath, evidence).not.toBeNull() + expect(details.error ?? ``, evidence).not.toContain(`did not reach`) + expect(details.error, evidence).toContain(`"kind": "${kind}"`) + + const replayPath = details.counterexamplePath + if (replayPath === null) { + throw new Error(`${kind} wrong-answer control did not shrink to a path`) + } + const replay = await fc.check( + generatedScenarioProperty({ faultKind: kind }), + { + seed: details.seed, + path: replayPath, + numRuns: 1, + verbose: 2, + }, + ) + const replayEvidence = JSON.stringify({ + seed: replay.seed, + requestedPath: replayPath, + replayPath: replay.counterexamplePath, + error: replay.error, + }) + expect(replay.failed, replayEvidence).toBe(true) + expect(replay.error ?? ``, replayEvidence).not.toContain(`did not reach`) + expect(replay.error, replayEvidence).toContain(`"kind": "${kind}"`) + }, + ) }) From bf763eb211b3c3ca2d5990f09fc211d8486cd30b Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 25 Sep 2026 14:27:39 -0600 Subject: [PATCH 8/8] fix(sqlite-persistence): preserve alias wire shape and legacy bigint reads --- docs/contributing/oracle-coverage.md | 2 +- .../src/remote-subset-wire.ts | 27 ++++++++-- .../src/sqlite-core-adapter.ts | 53 ++++++------------- .../tests/persisted.test.ts | 46 ++++++++++++++++ .../tests/expression-index-oracle.test.ts | 53 ++++++++++++++++++- 5 files changed, 136 insertions(+), 45 deletions(-) diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 44fbd98946..d108f9466b 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -112,7 +112,7 @@ comment and the current API/architecture contract before extending its model. | Electric and TrailBase | [Electric histories](https://github.com/TanStack/db/blob/main/packages/electric-db-collection/tests/electric-oracle.property.test.ts), [recovery histories](https://github.com/TanStack/db/blob/main/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts), [held resume snapshots](https://github.com/TanStack/db/blob/main/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts), [PostgreSQL semantics](https://github.com/TanStack/db/blob/main/packages/electric-db-collection/e2e/sql-predicate-semantics.e2e.test.ts), [TrailBase contract](https://github.com/TanStack/db/blob/main/packages/trailbase-db-collection/tests/ORACLE.md) | Installed SDK delivery/framing, independent predicates, exact subscription arguments, restart/reset lineage, held certification and durability races, source-order publication before durability, and late errors. The queued-presence property runs identical fixed/random generators plus isolated seed-and-path replay across insert, update, delete, and truncate callbacks. The recovery fixtures use a mocked ShapeStream; they do not establish live Electric-service framing or native persistence-host behavior. | | PowerSync | [tests](https://github.com/TanStack/db/tree/main/packages/powersync-db-collection/tests), `tests/correctness-oracle.test.ts` | Applied receipt positions crossed with held peers, native SQLite/SDK and cleanup evidence. Run the focused owner with the package's `test:oracles` command. A timeout mutant proves a progress failure, not every value assertion. | | SQLite persistence and native hosts | [persisted histories](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/persisted.test.ts), [reset/resume histories](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter.test.ts), [dual-adapter resume snapshots](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot.test.ts), [Browser composed-owner histories](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts), [Browser coordinator RPC](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/browser-coordinator.test.ts), [driver contracts](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/contracts/sqlite-driver-contract.ts), [Node shared-handle scheduling](https://github.com/TanStack/db/blob/main/packages/node-db-sqlite-persistence/tests/node-driver.test.ts), [OP-SQLite shared-handle scheduling](https://github.com/TanStack/db/blob/main/packages/react-native-db-sqlite-persistence/tests/op-sqlite-driver.test.ts), [browser OPFS lifecycle](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/opfs-page-lifecycle-oracle.test.ts), [worker diagnostics](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/opfs-worker-diagnostics-oracle.test.ts), [Electron IPC and composed owner](https://github.com/TanStack/db/blob/main/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts), [113-law manifest](https://github.com/TanStack/db/blob/main/packages/db-collection-e2e/src/fixtures/persisted-conformance-manifest.ts) | Core cache/remote rejection/peer/reopen histories, atomic reset/resume lineage, key-set evidence, dual-adapter races, and exact driver results. Browser composes public source commits with per-collection elected-owner routing and covers the complete committed-transaction wire partition through deterministic Node transport seams. Remote-subset histories distinguish logical demand, physical acquisitions, exact acquisition leases, and released replay tombstones. Electron composes source commits with a per-collection renderer owner, IPC adapter, real SQLite, and reopen checks. Same-handle Node and OP-SQLite tests cover transaction admission. Controlled OPFS page/worker histories cover ownership and diagnostic-cause retention. The reset/resume owners use sqlite3 CLI and in-memory node:sqlite seams; they do not prove multi-process WAL, mobile/Tauri, or other native-device execution. Distinct database handles rely on SQLite lock admission rather than one in-process queue. React Native hosts without async-context propagation must use the transaction driver supplied to the callback for nested work. Fake workers and synthetic page events do not prove native handle release or real bfcache admission. The Browser composed seams are not real multi-context/OPFS-worker execution; the Electron harness is not an actual Electron process unless its explicit runtime-bridge mode runs. An ownerless elected node suppresses core routing, while a follower may route demand to the elected owner; host coordinators retry only classified transport or admission failures while demand remains retained. The manifest excludes progressive and move suites; registration and shim runs are not device execution. | -| SQLite expression-index planning | [Node expression-index oracle](https://github.com/TanStack/db/blob/main/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts) | RFC #1659 invariant 8 owns identical persisted-index and runtime-expression shapes. Independent expected keys are checked against direct captured SQL, adapter results, and named-index plans. Limits: bounded unqualified JSON paths/scalars, signed-range BigInts, Node BetterSQLite, and no null, arbitrary raw SQL, or native-host planning. Run the package's `test:oracles` campaign. | +| SQLite expression-index planning | [Node expression-index oracle](https://github.com/TanStack/db/blob/main/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts) | RFC #1659 invariant 8 owns identical persisted-index and runtime-expression shapes. Independent expected keys are checked against direct captured SQL, adapter results, and named-index plans. Generated BigInts use SQLite's signed range; one fixed case checks legacy oversized-value reads. Other limits: bounded unqualified JSON paths/scalars, Node BetterSQLite, and no null, arbitrary raw SQL, or native-host planning. Run the package's `test:oracles` campaign. | | Offline execution | [scheduler](https://github.com/TanStack/db/blob/main/packages/offline-transactions/tests/KeyScheduler.property.test.ts), [leadership](https://github.com/TanStack/db/blob/main/packages/offline-transactions/tests/leadership-replay.property.test.ts), [settlement](https://github.com/TanStack/db/blob/main/packages/offline-transactions/tests/transaction-settlement.property.test.ts), [serialization](https://github.com/TanStack/db/blob/main/packages/offline-transactions/tests/transaction-serializer.property.test.ts) | Declarative FIFO eligibility, per-transaction outcomes, durable state and typed wire trees. Issued work may finish after ownership loss, but new work must not start. Exactly-once network execution is not promised. | | Frameworks | [React conformance](https://github.com/TanStack/db/blob/main/packages/react-db/tests/conformance.test.tsx), [React pagination](https://github.com/TanStack/db/blob/main/packages/react-db/tests/infinite-query-conformance.test.tsx), [shared suites](https://github.com/TanStack/db/tree/main/packages/db-collection-e2e/src/suites) | Exact exposed rows/pages and each framework's own lifecycle cuts. A React witness does not prove Vue/Solid/Angular/Svelte scheduling. Preserve their receiving registrations. | | Structural values and ordered primitives | [hash values](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash.property.test.ts), [hash graphs](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash-graph.property.test.ts), [mixed hash graphs](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash-mixed-graph.property.test.ts), [hash retry](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash-failure-retry.property.test.ts), [comparison](https://github.com/TanStack/db/blob/main/packages/db/tests/comparison.property.test.ts), [deep equality](https://github.com/TanStack/db/blob/main/packages/db/tests/utils.property.test.ts), [cursor](https://github.com/TanStack/db/blob/main/packages/db/tests/cursor.property.test.ts), [indexes](https://github.com/TanStack/db/blob/main/packages/db/tests/index-update.property.test.ts), [query identity](https://github.com/TanStack/db/blob/main/packages/db/tests/query/identity-output-shape-oracle.test.ts), [LIKE semantics](https://github.com/TanStack/db/blob/main/packages/db/tests/query/compiler/evaluators.test.ts) | Independent flat values, graph topology, algebraic laws, Map/group/sort recomputation, expression denotation, LIKE wildcard refinement, and compiled output bags. The LIKE owner covers boolean string matching and bounded work, not nullish three-valued logic or a general Unicode collation contract. Hash collision freedom is not promised. Unsupported composite cursors reject. | diff --git a/packages/db-sqlite-persistence-core/src/remote-subset-wire.ts b/packages/db-sqlite-persistence-core/src/remote-subset-wire.ts index 6fd85e76e6..0404009238 100644 --- a/packages/db-sqlite-persistence-core/src/remote-subset-wire.ts +++ b/packages/db-sqlite-persistence-core/src/remote-subset-wire.ts @@ -40,7 +40,7 @@ export type RemoteSubsetWireValue = | RemoteSubsetWireRecord export type RemoteSubsetWireExpression = - | { type: `ref`; path: Array } + | { type: `ref`; path: Array; sourceAlias?: string } | { type: `val`; value: RemoteSubsetWireValue } | { type: `func` @@ -232,11 +232,11 @@ function projectExpression( switch (type.value) { case `ref`: { assertExpressionPrototype(object, path, IR.PropRef.prototype) - assertAllowedProperties(object, path, [`type`, `path`]) - const projected = { + assertAllowedProperties(object, path, [`type`, `path`, `sourceAlias`]) + const projected: Extract = { type: `ref`, - path: [] as Array, - } satisfies RemoteSubsetWireExpression + path: [], + } state.expressions.set(object, projected) const sourcePath = readRequiredDataProperty( object, @@ -244,6 +244,23 @@ function projectExpression( `${path}.path`, ) projected.path = projectStringArray(sourcePath, `${path}.path`, state) + const sourceAlias = readDataProperty( + object, + `sourceAlias`, + `${path}.sourceAlias`, + ) + if (sourceAlias.present) { + if ( + typeof sourceAlias.value !== `string` || + projected.path[0] !== sourceAlias.value + ) { + throw new RemoteSubsetWireValueError( + `${path}.sourceAlias`, + `source alias must match the first path segment`, + ) + } + projected.sourceAlias = sourceAlias.value + } return projected } case `val`: { diff --git a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts index c0a3f79bd8..1d68098752 100644 --- a/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts +++ b/packages/db-sqlite-persistence-core/src/sqlite-core-adapter.ts @@ -44,10 +44,7 @@ type CompiledSqlFragment = { valueKind?: CompiledValueKind } -type SqlExpressionCompilationContext = - | `predicate` - | `index-expression` - | `comparison-target` +type SqlExpressionCompilationContext = `predicate` | `index-expression` type StoredSqliteRow = { key: string @@ -239,7 +236,7 @@ function decodePersistedJsonValue(value: unknown): unknown { if (isPersistedTaggedValue(value)) { switch (value[PERSISTED_TYPE_TAG]) { case `bigint`: - return assertSQLiteBigIntInRange(BigInt(value[PERSISTED_VALUE_TAG])) + return BigInt(value[PERSISTED_VALUE_TAG]) case `date`: { const parsedDate = new Date(value[PERSISTED_VALUE_TAG]) return Number.isNaN(parsedDate.getTime()) ? null : parsedDate @@ -331,29 +328,6 @@ function toSqliteExpressionLiteral(value: unknown): string { return toSqliteLiteral(toSqliteParameterValue(value)) } -function inlineSqlParams( - sql: string, - params: ReadonlyArray, -): string { - // Every question mark in this compiler-owned SQL is a placeholder. Ref-path - // literals cannot contain one because createJsonPath rejects such segments. - // Callers must bypass this helper when SQL already contains other literals. - let index = 0 - const inlinedSql = sql.replace(/\?/g, () => { - const paramValue = params[index] - index++ - return toSqliteLiteral(paramValue ?? null) - }) - - if (index !== params.length) { - throw new InvalidPersistedCollectionConfigError( - `Unable to inline SQL params; placeholder count did not match provided params`, - ) - } - - return inlinedSql -} - type CompiledRowExpressionEvaluator = (row: Record) => unknown function compileRowExpressionEvaluator( @@ -367,7 +341,7 @@ function compileRowExpressionEvaluator( `Unsupported expression for SQLite adapter fallback evaluator: ${(error as Error).message}`, ) } - return (row) => baseEvaluator(row) + return baseEvaluator } function getOrderByObjectId(value: object): number { @@ -640,7 +614,7 @@ function argumentCompilationContext( case `ilike`: if (argument.type !== `val`) return `index-expression` return typeof argument.value === `bigint` - ? `comparison-target` + ? `index-expression` : `predicate` case `in`: return argumentIndex === 0 ? `index-expression` : `predicate` @@ -658,16 +632,16 @@ function compileSqlExpression( ): CompiledSqlFragment { if (expression.type === `val`) { const valueKind = getLiteralValueKind(expression.value) - const value = toSqliteParameterValue(expression.value) return { supported: true, sql: context === `index-expression` ? toSqliteExpressionLiteral(expression.value) - : context === `comparison-target` - ? expression.value.toString() - : `?`, - params: context === `predicate` ? [value] : [], + : `?`, + params: + context === `predicate` + ? [toSqliteParameterValue(expression.value)] + : [], valueKind, } } @@ -1021,9 +995,12 @@ function normalizeIndexSqlFragment(fragment: string): string { `Persisted index expression is not supported by the SQLite compiler`, ) } - return compiled.params.length === 0 - ? compiled.sql - : inlineSqlParams(compiled.sql, compiled.params) + if (compiled.params.length !== 0) { + throw new InvalidPersistedCollectionConfigError( + `Persisted index expression cannot contain bound parameters`, + ) + } + return compiled.sql } return sanitizeExpressionSqlFragment(fragment) diff --git a/packages/db-sqlite-persistence-core/tests/persisted.test.ts b/packages/db-sqlite-persistence-core/tests/persisted.test.ts index f55f5075b6..2e4de00d1a 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted.test.ts @@ -3063,6 +3063,52 @@ describeUnlessOracleReplay(`persistedCollectionOptions`, () => { } }) + it(`preserves an explicit source alias through remote subset projection`, async () => { + const coordinator = new SingleProcessCoordinator(`single-wire-alias`) + const owner = Object.assign(vi.fn(), { + unloadSubset: vi.fn(), + onError: vi.fn(), + }) + const unregisterOwner = coordinator.registerRemoteSubsetOwner( + `todos`, + owner, + ) + const options: LoadSubsetOptions = { + where: new IR.Func(`eq`, [ + new IR.PropRef([`todos`, `status`], `todos`), + new IR.Value(`kept`), + ]), + } + + try { + await coordinator.requestEnsureRemoteSubset(`todos`, options) + expect(owner).toHaveBeenCalledWith({ + where: { + type: `func`, + name: `eq`, + args: [ + { type: `ref`, path: [`todos`, `status`], sourceAlias: `todos` }, + { type: `val`, value: `kept` }, + ], + }, + }) + } finally { + await coordinator.requestReleaseRemoteSubset(`todos`, options) + unregisterOwner() + } + }) + + it(`rejects a remote subset source alias that disagrees with its path`, () => { + expect(() => + toTransportedLoadSubsetOptions({ + where: new IR.Func(`eq`, [ + new IR.PropRef([`todos`, `status`], `other`), + new IR.Value(`kept`), + ]), + }), + ).toThrowError(/options\.where\.args\[0\]\.sourceAlias/) + }) + it(`projects lexical comparison options without locale-only wire fields`, () => { const projected = toTransportedLoadSubsetOptions({ orderBy: [ diff --git a/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts b/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts index 97757fe48f..56e6f581e6 100644 --- a/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts +++ b/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts @@ -15,7 +15,8 @@ * persisted Date ranges, and BigInt ranges/IN within SQLite's signed-integer * domain. Explicitly qualified refs lower to the same JSON field expression * without reinterpreting legacy nested paths. Known omissions: null, arbitrary - * raw SQL, native-host planning, and BigInts outside SQLite's signed range. + * raw SQL, and native-host planning. Generated BigInts stay inside SQLite's + * signed range. A fixed legacy-byte case checks read compatibility beyond it. * * Independent model: fixed keys encode the result of each generated relation; * the equality witness also uses a full adapter scan, a small path walker, and @@ -1587,6 +1588,56 @@ describe(`SQLite expression-index oracle`, () => { }, [() => driver.close()]) }) + it(`reads a legacy persisted BigInt beyond the new write range`, async () => { + const driver = new BetterSqlite3SQLiteDriver({ filename: `:memory:` }) + const adapter = createSQLiteCorePersistenceAdapter({ driver }) + const collectionId = `legacy-bigint-read` + const tableName = createPersistedTableName(collectionId, `c`) + const legacyValue = 10n ** 30n + + await withFailurePreservingCleanup(async () => { + await adapter.applyCommittedTx(collectionId, { + txId: `seed-legacy-bigint-read`, + term: 1, + seq: 1, + rowVersion: 1, + mutations: [ + { type: `insert`, key: `legacy`, value: { id: `legacy`, count: 1n } }, + ], + }) + driver + .getDatabase() + .prepare(`UPDATE "${tableName}" SET value = ?`) + .run( + JSON.stringify({ + id: `legacy`, + count: { + [PERSISTED_TYPE_TAG]: `bigint`, + [PERSISTED_VALUE_TAG]: legacyValue.toString(), + }, + }), + ) + + if (!adapter.scanRows) { + throw new Error(`real SQLite adapter did not expose scanRows`) + } + const scanned = await adapter.scanRows(collectionId) + expect(scanned.map((row) => row.value)).toEqual([ + { id: `legacy`, count: legacyValue }, + ]) + + const subset = await adapter.loadSubset(collectionId, { + where: new IR.Func(`eq`, [ + new IR.PropRef([`id`]), + new IR.Value(`legacy`), + ]), + }) + expect(subset.map((row) => row.value)).toEqual([ + { id: `legacy`, count: legacyValue }, + ]) + }, [() => driver.close()]) + }) + it.each([ { label: `scalar comparison`,