diff --git a/.changeset/skip-write-and-snapshot-fixes.md b/.changeset/skip-write-and-snapshot-fixes.md new file mode 100644 index 0000000..6eb00ef --- /dev/null +++ b/.changeset/skip-write-and-snapshot-fixes.md @@ -0,0 +1,9 @@ +--- +'fetchium': patch +--- + +Fixed stale data in components when a second query loads entities that are already in memory, and streamed updates that were lost on restart. + +- Development builds freeze query results, so mutating one (such as calling `sort()` on a list in render) throws a `TypeError`. Copy the data first, and treat production results as read-only too. +- In development, the stale-data check logs an error instead of throwing and blocking other components' updates. +- Custom stores can add the new optional `QueryStore.mergeEntity`, so an entity built from streamed updates doesn't overwrite stored fields those updates didn't include. Without it, writes work as before. diff --git a/.changeset/upgrade-behavior-changes.md b/.changeset/upgrade-behavior-changes.md new file mode 100644 index 0000000..626d5b9 --- /dev/null +++ b/.changeset/upgrade-behavior-changes.md @@ -0,0 +1,19 @@ +--- +'fetchium': major +--- + +Breaking changes. Most apps need no code changes, but tests that depend on loading frames, render counts or timers may. + +- **Results are frozen in development.** Mutating query data throws a `TypeError`; copy it first (`[...data.items].sort()`). +- **Cached data shows on the first render** with `SyncQueryStore`. Update tests that wait for a loading state or count renders. +- **Fetches start sooner.** The first fetch and refetches without `debounce` start on a microtask, not after `setTimeout(0)`; update tests that advance timers to see them. +- **Identical data doesn't re-render.** A refetch, poll or streamed update with the same data skips the render; read `isFetching` instead of counting renders. +- **Nested values change identity only when they change.** `t.object`, `t.record` and live-collection values are new objects after a change, so memoized children re-render; drop workarounds that forced it. +- **`t.record` and `t.result` fields apply updates** instead of keeping their old value. +- **Changing a Signal param mid-fetch aborts that fetch** and fetches the new params; the old response is dropped. +- **`QueryClient.destroy()` aborts fetches in flight.** Code that awaits a query during teardown gets an `AbortError`; catch it. +- **Awaiting a query can wait for a remount.** If the last component using a query unmounts before its first data arrives, `await fetchQuery(...)` waits for the next mount, `gcTime` collection or `destroy()` instead of rejecting with an `AbortError`. +- **New reserved `QueryClientConfig` names:** `shouldRetry`, `reactivationGraceMs`, `reactivationStaggerMs`, `activity` and `pollResumeJitterMs` are read as options; rename custom context values that use them. +- **`retry: {}` and `retry: { retryDelay }` now retry** like `retry: true`; use `retry: false` for no retries. +- **`AsyncQueryStore` honors `cache.maxCount`.** Check the value if you set one. +- **Custom `QueryStore`s** get new optional methods (`mergeEntity`, `onDelete`, `onPersisted`, `isSettled`, `hasQueuedDeletes`, `hasEntity`); without them nothing changes. diff --git a/docs/src/app/api/fetchium-react/page.md b/docs/src/app/api/fetchium-react/page.md index 98f2c51..3bc22c9 100644 --- a/docs/src/app/api/fetchium-react/page.md +++ b/docs/src/app/api/fetchium-react/page.md @@ -132,7 +132,7 @@ const SearchResults = component(() => { - `useQuery` is a thin wrapper around Signalium v3's `useReactive`, which is deep-by-default. It returns a **structurally-shared snapshot** of the query result, so memoized children that receive subtrees as props keep stable references when the underlying data is unchanged. - Fetchium registers a custom snapshot for entity proxies so the snapshot walks into entities (instead of returning them by reference), giving you correct re-rendering on entity field changes. -- The snapshot is a plain-object copy of the query result to prevent accidental mutation of the entity cache. Use `draft()` from `fetchium` if you need a mutable copy for mutations. +- The snapshot is a plain-object copy of the query result to prevent accidental mutation of the entity cache. Treat it as read-only: in development builds the snapshot and every nested object and array in it are frozen, so an in-place `sort()`, a `push()` or an assignment in render throws a `TypeError` at that line. Production snapshots are not frozen, and a mutation there is carried forward until the entity changes. Sort a copy (`[...items].sort()`) or use `draft()` from `fetchium` if you need a mutable copy for mutations. - `useQuery` never suspends. Reading `.value` while pending returns `undefined`. Use [`useSuspenseQuery`](#usesuspensequery) to suspend on a cold miss. --- diff --git a/docs/src/app/api/fetchium/page.md b/docs/src/app/api/fetchium/page.md index bcf3f80..5120ea2 100644 --- a/docs/src/app/api/fetchium/page.md +++ b/docs/src/app/api/fetchium/page.md @@ -241,14 +241,14 @@ new QueryClient(config: QueryClientConfig) #### Methods -| Method | Signature | Description | -| -------------------- | -------------------------------------------------- | -------------------------------------------------------------------------------------------------------- | -| `getContext` | `(): QueryContext` | Returns the `QueryContext` passed at construction. | -| `applyMutationEvent` | `(event: MutationEvent): void` | Applies an external mutation event (create/update/delete) to the entity store. | -| `invalidateQueries` | `(targets: ReadonlyArray): void` | Marks matching query instances as stale. Accepts query classes and optional param subsets for filtering. | -| `prefetch` | `(QueryClass, params?, { ttl? }?): () => void` | Starts a query now and keeps it active for `ttl` ms (default `DEFAULT_PREFETCH_TTL`, 10 s). See below. | -| `retain` | `(fn: () => unknown, { ttl? }?): () => void` | Keeps the queries `fn` reads active until the returned `release` is called, or for `ttl` ms. See below. | -| `destroy` | `(): void` | Tears down the GC manager, network manager, and all caches, and releases outstanding leases. | +| Method | Signature | Description | +| -------------------- | -------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------- | +| `getContext` | `(): QueryContext` | Returns the `QueryContext` passed at construction. | +| `applyMutationEvent` | `(event: MutationEvent): void` | Applies an external mutation event (create/update/delete) to the entity store. | +| `invalidateQueries` | `(targets: ReadonlyArray): void` | Marks matching query instances as stale. Accepts query classes and optional param subsets for filtering. | +| `prefetch` | `(QueryClass, params?, { ttl? }?): () => void` | Starts a query now and keeps it active for `ttl` ms (default `DEFAULT_PREFETCH_TTL`, 10 s). See below. | +| `retain` | `(fn: () => unknown, { ttl? }?): () => void` | Keeps the queries `fn` reads active until the returned `release` is called, or for `ttl` ms. See below. | +| `destroy` | `(): void` | Tears down the GC manager, network manager, and all caches, releases outstanding leases, and unsubscribes the store listeners it registered. | #### `prefetch` and `retain` @@ -710,8 +710,8 @@ The `t` object provides a declarative type definition DSL for describing query p ### Store types -| Type | Definition | Description | -| -------------------- | ----------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | -| `QueryStore` | Interface | Storage backend interface. See [stores/sync](/api/stores-sync) and [stores/async](/api/stores-async). | -| `CachedQuery` | `{ value: unknown; refIds: Set \| undefined; updatedAt: number; preloadedEntities?: PreloadedEntityMap }` | Cached query data returned by `QueryStore.loadQuery()`. | -| `PreloadedEntityMap` | `Map>` | Pre-loaded entity data for hydration. | +| Type | Definition | Description | +| -------------------- | ----------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `QueryStore` | Interface | Storage backend interface. See [stores/sync](/api/stores-sync) and [stores/async](/api/stores-async). Besides the required methods, a store may implement `onDelete` (the client then skips writing entities whose records are current, and writes one again once the store reports its record dropped), `onPersisted` (for a store that applies writes later), `isSettled`, `hasEntity` (lets a streamed event refresh the record of an entity that is not in memory) and `mergeEntity` (merges the fields streamed events supplied for an entity built from them over its record; without it, such an entity is written whole through `saveEntity`). A store without `onDelete` is written on every apply, as before 0.6; when `hasEntity` is missing or answers `undefined`, an event for an entity that is not in memory and that no live array retains is written only as a `mergeEntity` call with `ifStored`, which writes nothing when the store holds no record, and not at all without `mergeEntity`. | +| `CachedQuery` | `{ value: unknown; refIds: Set \| undefined; updatedAt: number; preloadedEntities?: PreloadedEntityMap }` | Cached query data returned by `QueryStore.loadQuery()`. | +| `PreloadedEntityMap` | `Map>` | Pre-loaded entity data for hydration. | diff --git a/docs/src/app/api/stores-async/page.md b/docs/src/app/api/stores-async/page.md index d6e5f25..ab56842 100644 --- a/docs/src/app/api/stores-async/page.md +++ b/docs/src/app/api/stores-async/page.md @@ -41,14 +41,19 @@ new AsyncQueryStore(config: AsyncQueryStoreConfig) #### Methods -| Method | Signature | Description | -| ------------------- | -------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `loadQuery` | `(queryDef: QueryDefinition, queryKey: number): Promise` | Loads a cached query. Only works when a `delegate` is available (writer mode). Returns `undefined` if no delegate, if the entry has expired, or if it does not exist. Preloads referenced entities. | -| `saveQuery` | `(queryDef: QueryDefinition, queryKey: number, value: unknown, updatedAt: number, refIds?: Set): void` | Dispatches a save-query message. In writer mode, enqueues for serial processing. In reader mode, sends via `sendMessage`. | -| `saveEntity` | `(entityKey: number, value: unknown, refIds?: Set): void` | Dispatches a save-entity message. | -| `activateQuery` | `(queryDef: QueryDefinition, queryKey: number): void` | Dispatches an activate-query message to update the LRU queue. | -| `deleteQuery` | `(queryKey: number): void` | Dispatches a delete-query message. | -| `purgeStaleQueries` | `(): Promise` | Scans all stored query classes and removes expired entries. Only operates when a `delegate` is available. | +| Method | Signature | Description | +| ------------------- | -------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `loadQuery` | `(queryDef: QueryDefinition, queryKey: number): Promise` | Loads a cached query. Only works when a `delegate` is available (writer mode). Returns `undefined` if no delegate, if the entry has expired, or if it does not exist. Preloads referenced entities. | +| `saveQuery` | `(queryDef: QueryDefinition, queryKey: number, value: unknown, updatedAt: number, refIds?: Set): void` | Dispatches a save-query message. In writer mode, enqueues for serial processing. In reader mode, sends via `sendMessage`. | +| `saveEntity` | `(entityKey: number, value: unknown, refIds?: Set): void` | Dispatches a save-entity message. | +| `mergeEntity` | `(entityKey: number, fields: unknown, refIds?: Set, ifStored?: boolean): void` | Dispatches a save-entity message with `merge: true`; the writer merges `fields` over the stored record, if there is one (see `SyncQueryStore.mergeEntity`). With `ifStored` and no stored record, the writer writes nothing and reports the key to `onDelete` listeners. | +| `activateQuery` | `(queryDef: QueryDefinition, queryKey: number): void` | Dispatches an activate-query message to update the LRU queue. A queue persisted with a different `maxCount` is resized by the writer: growing reallocates it, shrinking evicts the keys that no longer fit. | +| `deleteQuery` | `(queryKey: number): void` | Dispatches a delete-query message. | +| `purgeStaleQueries` | `(): Promise` | Scans all stored query classes and removes expired entries. In writer mode it runs through the same serial queue as the messages, after everything queued before it; a reader with its own `delegate` runs it directly. Only operates when a `delegate` is available. | +| `onDelete` | `(listener: (key: number) => void): () => void` | Writer mode only. Registers a listener called with the key of every record the writer drops on its own (an LRU eviction, a reference-count cascade, a purge) or fails to write. Returns an unsubscribe function. | +| `onPersisted` | `(listener: (key: number) => void): () => void` | Writer mode only. Registers a listener called with the key of every entity record a queued write has stored. Returns an unsubscribe function. | +| `isSettled` | `(): boolean` | Whether every operation handed to the writer has been applied (nothing queued, nothing mid-flight). Always `true` in reader mode. | +| `hasEntity` | `(key: number): boolean \| undefined` | Writer mode only. Whether the delegate holds a record for the key, answered from a set the writer reads from the delegate when it starts (alongside its queue, not ahead of it) and keeps current with every write it queues and every deletion it performs. `undefined` for a key it has not written itself until that read lands; after a read that keeps failing, nothing is assumed held. An event for an entity not in memory is sent as an `ifStored` merge while the answer is `undefined`, and always from a reader. | --- @@ -126,6 +131,7 @@ type StoreMessage = value: unknown; updatedAt: number; cacheTime: number; + maxCount?: number; // absent from readers built before 0.6; the writer then keeps the queue's size refIds?: number[]; } | { @@ -133,12 +139,14 @@ type StoreMessage = entityKey: number; value: unknown; refIds?: number[]; + merge?: boolean; // `value` holds only some fields: merge them over the stored record } | { type: 2; // ActivateQuery queryDefId: string; queryKey: number; cacheTime: number; + maxCount?: number; } | { type: 3; // DeleteQuery diff --git a/docs/src/app/api/stores-sync/page.md b/docs/src/app/api/stores-sync/page.md index 6135588..c4f4235 100644 --- a/docs/src/app/api/stores-sync/page.md +++ b/docs/src/app/api/stores-sync/page.md @@ -32,14 +32,17 @@ new SyncQueryStore(kv: SyncPersistentStore) #### Methods -| Method | Signature | Description | -| ------------------- | -------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `loadQuery` | `(queryDef: QueryDefinition, queryKey: number): CachedQuery \| undefined` | Loads a cached query by key. Returns `undefined` if the cache entry has expired (beyond `cacheTime`) or does not exist. Also preloads referenced entities. | -| `saveQuery` | `(queryDef: QueryDefinition, queryKey: number, value: unknown, updatedAt: number, refIds?: Set): void` | Persists a query result, its timestamp, and entity reference IDs. Activates the query in the LRU queue. | -| `saveEntity` | `(entityKey: number, value: unknown, refIds?: Set): void` | Persists an entity's serialized data and its child entity references. Manages reference counts for nested entities. | -| `activateQuery` | `(queryDef: QueryDefinition, queryKey: number): void` | Moves a query to the front of the LRU queue for its query class. If the queue is full, the oldest entry is evicted. | -| `deleteQuery` | `(queryKey: number): void` | Deletes a query's stored value, reference IDs, and decrements reference counts for all referenced entities. | -| `purgeStaleQueries` | `(): void` | Scans all query classes and removes those whose `lastUsedAt` timestamp exceeds their `cacheTime`. Called automatically on `QueryClient` construction. | +| Method | Signature | Description | +| ------------------- | -------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `loadQuery` | `(queryDef: QueryDefinition, queryKey: number): CachedQuery \| undefined` | Loads a cached query by key. Returns `undefined` if the cache entry has expired (beyond `cacheTime`) or does not exist. Also preloads referenced entities. | +| `saveQuery` | `(queryDef: QueryDefinition, queryKey: number, value: unknown, updatedAt: number, refIds?: Set): void` | Persists a query result, its timestamp, and entity reference IDs. Activates the query in the LRU queue. | +| `saveEntity` | `(entityKey: number, value: unknown, refIds?: Set): void` | Persists an entity's serialized data and its child entity references. Manages reference counts for nested entities. | +| `mergeEntity` | `(entityKey: number, fields: unknown, refIds?: Set): void` | Writes the fields streamed events supplied for an entity built from them: merged over the stored record when there is one (the record's references are then derived from the merged value), written as the record otherwise. A record that cannot be merged into is replaced. | +| `activateQuery` | `(queryDef: QueryDefinition, queryKey: number): void` | Moves a query to the front of the LRU queue for its query class. If the queue is full, the oldest entry is evicted. A queue persisted with a different `maxCount` is resized: growing reallocates it, shrinking evicts the keys that no longer fit. | +| `deleteQuery` | `(queryKey: number): void` | Deletes a query's stored value, reference IDs, and decrements reference counts for all referenced entities. | +| `purgeStaleQueries` | `(): void` | Scans all query classes and removes those whose `lastUsedAt` timestamp exceeds their `cacheTime`. Called automatically on `QueryClient` construction. | +| `onDelete` | `(listener: (key: number) => void): () => void` | Registers a listener called with the key of every record the store drops on its own (an LRU eviction, the reference-count cascade that follows it, a purge). Listeners are shared by every store over the same `kv`. Returns an unsubscribe function; `QueryClient.destroy()` calls it. | +| `hasEntity` | `(key: number): boolean` | Whether the store holds a record for the key. The client refreshes the record of an entity a streamed event updates while the entity is not in memory only when this answers `true`. | --- diff --git a/docs/src/app/core/entities/page.md b/docs/src/app/core/entities/page.md index 71187c0..11745ae 100644 --- a/docs/src/app/core/entities/page.md +++ b/docs/src/app/core/entities/page.md @@ -58,7 +58,7 @@ class GetPost extends RESTQuery { ``` {% callout type="warning" %} -Entities are **read-only**. Attempting to set a property on an entity will throw an error in development mode. To update entity data, use mutations or streaming updates. +Entities are **read-only**, and so are the snapshots `useQuery` and `useReactive` take of them. Attempting to set a property on an entity, or on a snapshot in a development build, will throw an error. To update entity data, use mutations or streaming updates. {% /callout %} --- diff --git a/docs/src/app/data/caching/page.md b/docs/src/app/data/caching/page.md index feacbd1..c3582a0 100644 --- a/docs/src/app/data/caching/page.md +++ b/docs/src/app/data/caching/page.md @@ -133,7 +133,7 @@ class GetUser extends RESTQuery { } ``` -`maxCount` limits how many distinct parameter combinations are stored for this query class. When the limit is exceeded, the oldest entries are evicted. The default is `50`. +`maxCount` limits how many distinct parameter combinations are stored for this query class. When the limit is exceeded, the oldest entries are evicted. The default is `50`. Changing `maxCount` between releases resizes the queue already persisted for the class the first time the store touches that class after starting: growing keeps every entry, shrinking evicts the oldest entries that no longer fit. --- diff --git a/docs/src/app/setup/project-setup/page.md b/docs/src/app/setup/project-setup/page.md index d90077d..b7ca2b8 100644 --- a/docs/src/app/setup/project-setup/page.md +++ b/docs/src/app/setup/project-setup/page.md @@ -289,6 +289,8 @@ Fetchium uses a compile-time constant `IS_DEV` to enable additional runtime chec - **Parse validation warnings** --- detailed logs when response data doesn't match your type definitions - **Entity write protection** --- throws when you accidentally try to mutate an entity property directly (entities are read-only) +- **Frozen snapshots** --- the snapshots `useQuery` and `useReactive` hand out are frozen, so mutating one (an in-place `sort()`, an assignment in render) throws a `TypeError` at the offending line instead of silently corrupting what the next render shows +- **Stale-snapshot detection** --- a snapshot taken at an unchanged entity version re-reads only the fields whose values live elsewhere; development builds re-read the rest too and, if one changed, serve the re-read value, log the discrepancy through the client's `log.error`, and raise it once per typename and field as an uncaught error - **Reference validation** --- catches common mistakes like using conditional logic in class field definitions In production builds, all `IS_DEV` code paths are tree-shaken, so there is zero runtime cost. diff --git a/packages/fetchium/src/ConstraintMatcher.ts b/packages/fetchium/src/ConstraintMatcher.ts index 1bb40eb..8058205 100644 --- a/packages/fetchium/src/ConstraintMatcher.ts +++ b/packages/fetchium/src/ConstraintMatcher.ts @@ -201,13 +201,15 @@ export class ConstraintMatcher { } } + /** With `dryRun`, only reports matches through `onMatch`, touching no collection. */ routeEvent( typename: string, entityData: Record, entityKey: number, eventType: 'create' | 'update' | 'delete', - onMatch?: () => void, + onMatch?: (willRetain: boolean) => void, deleteData?: Record, + dryRun: boolean = false, ): void { for (const group of this._groups.values()) { const hash = computeConstraintHash(entityData, group.fieldPaths); @@ -216,7 +218,7 @@ export class ConstraintMatcher { const bindings = group.getMatching(hash); if (bindings !== undefined) { for (const binding of bindings) { - binding.onEvent(typename, entityKey, eventType, onMatch, deleteData); + binding.onEvent(typename, entityKey, eventType, onMatch, deleteData, dryRun); } } } diff --git a/packages/fetchium/src/EntityInstance.ts b/packages/fetchium/src/EntityInstance.ts index 7451a23..041b733 100644 --- a/packages/fetchium/src/EntityInstance.ts +++ b/packages/fetchium/src/EntityInstance.ts @@ -153,6 +153,11 @@ function snapshotCameFrom(snapshotObj: unknown, sourceId: number): boolean { return snapshotOrigins.get(snapshotObj)?.sourceId === sourceId; } +/** Dev-only: a consumer that mutates a snapshot throws at its call site. */ +function freezeInDev(obj: T): T { + return IS_DEV ? Object.freeze(obj) : obj; +} + function asPrevObject(prev: unknown): Record | undefined { return prev !== null && typeof prev === 'object' && !Array.isArray(prev) ? (prev as Record) @@ -185,9 +190,10 @@ function snapshotRawValue(value: unknown, prev: unknown, snap: SnapshotFn): unkn } const next = snapshotRawValue(item, prevItem, snap); result[i] = next; - if (!changed && next !== prevArr![i]) changed = true; + // `Object.is`, so a NaN slot doesn't change identity on every walk. + if (!changed && !Object.is(next, prevArr![i])) changed = true; } - return changed ? result : prevArr!; + return changed ? freezeInDev(result) : prevArr!; } if (Object.getPrototypeOf(value) === ObjectProto) { @@ -200,15 +206,61 @@ function snapshotRawValue(value: unknown, prev: unknown, snap: SnapshotFn): unkn const key = keys[i]; const next = snapshotRawValue(obj[key], prevObj?.[key], snap); result[key] = next; - if (!changed && next !== prevObj![key]) changed = true; + if (!changed && !Object.is(next, prevObj![key])) changed = true; } - return changed ? result : prevObj!; + return changed ? freezeInDev(result) : prevObj!; } return snap(value, prev); } -/** Dev-only: losing the fast path is invisible to behavioral tests. */ +/** Dev guard: tells real drift from a value that only lost identity, like a rebuilt `Set`. */ +function sameSnapshotValue(a: unknown, b: unknown): boolean { + if (Object.is(a, b)) return true; + if (typeof a !== 'object' || typeof b !== 'object' || a === null || b === null) return false; + if (Array.isArray(a) || Array.isArray(b)) { + if (!Array.isArray(a) || !Array.isArray(b) || a.length !== b.length) return false; + for (let i = 0; i < a.length; i++) if (!sameSnapshotValue(a[i], b[i])) return false; + return true; + } + if (a instanceof Set || b instanceof Set) { + if (!(a instanceof Set) || !(b instanceof Set) || a.size !== b.size) return false; + // Pair by position, searching only after the first misalignment. + const bItems = [...b]; + let matched: Set | undefined; + let i = 0; + for (const item of a) { + if (matched === undefined && sameSnapshotValue(item, bItems[i])) { + i++; + continue; + } + if (matched === undefined) { + matched = new Set(); + for (let k = 0; k < i; k++) matched.add(k); + } + const j = bItems.findIndex((other, idx) => !matched!.has(idx) && sameSnapshotValue(item, other)); + if (j === -1) return false; + matched.add(j); + i++; + } + return true; + } + if (a instanceof Map || b instanceof Map) { + if (!(a instanceof Map) || !(b instanceof Map) || a.size !== b.size) return false; + for (const [k, v] of a) if (!b.has(k) || !sameSnapshotValue(v, b.get(k))) return false; + return true; + } + if (Object.getPrototypeOf(a) !== ObjectProto || Object.getPrototypeOf(b) !== ObjectProto) return false; + const aKeys = Object.keys(a); + if (aKeys.length !== Object.keys(b).length) return false; + for (const key of aKeys) { + if (!Object.hasOwn(b, key)) return false; + if (!sameSnapshotValue((a as Record)[key], (b as Record)[key])) return false; + } + return true; +} + +/** Dev-only counters: losing the fast path is invisible to behavioral tests. */ export let __debug_snapshotFieldReads = 0; export let __debug_snapshotFullWalks = 0; export function __debug_resetSnapshotCounters(): void { @@ -232,10 +284,33 @@ function walkFields( } } +/** For proxies from a previous instance of this module (hot reload), absent from `snapshotSources`. */ +function snapshotProxyByWalking(current: object, prev: unknown, snap: SnapshotFn): unknown { + const obj = current as Record; + const keys = Object.keys(obj); + const prevObj = asPrevObject(prev); + let changed = prevObj === undefined || Object.keys(prevObj).length !== keys.length; + const result: Record = {}; + for (let i = 0; i < keys.length; i++) { + const key = keys[i]; + const val = obj[key]; + const next = typeof val === 'function' ? val : snap(val, prevObj?.[key]); + result[key] = next; + if (!changed && !Object.is(next, prevObj![key])) changed = true; + } + return changed ? freezeInDev(result) : prevObj; +} + const snapshotEntity = (current: object, prev: unknown, snap: SnapshotFn): unknown => { const source = snapshotSources.get(current); - // Not built by `createProxy`, so there are no fields to read. - if (source === undefined) return current; + if (source === undefined) { + // Hot-reloaded proxy. Returning it as-is would stop React re-rendering on its changes. + if (PROXY_ID.has(current) || typeof (current as { toJSON?: unknown }).toJSON === 'function') { + return snapshotProxyByWalking(current, prev, snap); + } + // A bare `Entity` has no fields to read, only the shape's type defs. + return current; + } // eslint-disable-next-line @typescript-eslint/no-unused-expressions source.relay?.value; @@ -261,14 +336,29 @@ const snapshotEntity = (current: object, prev: unknown, snap: SnapshotFn): unkno let result: Record | undefined; for (let i = 0; i < keys.dynamic.length; i++) { const key = keys.dynamic[i]; - if (patch[key] !== before[key]) { + if (!Object.is(patch[key], before[key])) { if (result === undefined) result = { ...before }; result[key] = patch[key]; } } - if (IS_DEV) assertStaticFieldsUnchanged(source, keys, before, snap); - return rememberSnapshot(source.id, result ?? before, version, keys); + if (IS_DEV) { + // Report drift, don't throw: Signalium's watcher flush has no catch. + const drifted = verifyStaticFields(source, keys, before, snap); + if (drifted !== undefined) { + if (result === undefined) result = { ...before }; + for (const key of Object.keys(drifted)) result[key] = drifted[key]; + } + // A consumer altered the snapshot's keys. Frozen ones can't have been. + if (!Object.isFrozen(before) && !sameKeyList(Object.keys(before), keys.enumerable)) { + const source_ = result ?? before; + const rebuilt: Record = {}; + for (const key of keys.enumerable) rebuilt[key] = source_[key]; + result = rebuilt; + reportDrift(source, '(keys)', 'its key set was altered'); + } + } + return rememberSnapshot(source.id, result !== undefined ? freezeInDev(result) : before, version, keys); } if (IS_DEV) __debug_snapshotFullWalks++; @@ -279,37 +369,83 @@ const snapshotEntity = (current: object, prev: unknown, snap: SnapshotFn): unkno let changed = prevObj === undefined || Object.keys(prevObj).length !== enumerable.length; if (!changed) { for (let i = 0; i < enumerable.length; i++) { - if (result[enumerable[i]] !== prevObj![enumerable[i]]) { + if (!Object.is(result[enumerable[i]], prevObj![enumerable[i]])) { changed = true; break; } } } - return rememberSnapshot(source.id, changed ? result : prevObj!, version, keys); + return rememberSnapshot(source.id, changed ? freezeInDev(result) : prevObj!, version, keys); +}; + +function sameKeyList(a: string[], b: string[]): boolean { + if (a.length !== b.length) return false; + for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false; + return true; +} + +/** Dev-only: drifts already raised, keyed `typename.field`. */ +const reportedDrift = new Set(); + +export type SnapshotDriftHandler = (error: Error, queryClient: QueryClient) => void; + +/** Rethrows in a microtask so the error surfaces as uncaught, outside Signalium's watcher flush. */ +const defaultDriftHandler: SnapshotDriftHandler = (error, queryClient) => { + queryClient.getContext().log?.error?.(error.message, error); + queueMicrotask(() => { + throw error; + }); }; -/** Dev check: a static field must not change while `version` stays put. */ -function assertStaticFieldsUnchanged( +let snapshotDriftHandler: SnapshotDriftHandler = defaultDriftHandler; + +/** Dev-only test hook. Also clears the raised set. `undefined` restores the default. */ +export function __setSnapshotDriftHandler(handler: SnapshotDriftHandler | undefined): void { + snapshotDriftHandler = handler ?? defaultDriftHandler; + reportedDrift.clear(); +} + +function reportDrift(source: EntitySnapshotSource, key: string, what: string): void { + const tag = `${source.instance.typename}.${key}`; + if (reportedDrift.has(tag)) return; + reportedDrift.add(tag); + const error = new Error( + `[fetchium] stale entity snapshot: ${source.instance.typename}:${source.instance.id} ${what} while version ` + + `stayed at ${source.instance.version}. The re-read value is being served, but a production build would keep ` + + `the stale one. Either the field was mutated without notify(), the shape's static/dynamic split is wrong ` + + `for it, or a consumer mutated a snapshot it received (dev snapshots are frozen so that fails at the call ` + + `site; in production they are not). Raised once per field.`, + ); + snapshotDriftHandler(error, source.instance._queryClient); +} + +/** + * Checks the fast path's assumptions: `data` only changes through a `notify()` + * that bumps `version`, and every field outside `keys.dynamic` is a pure + * function of `data`. Returns the static fields that drifted, if any. + * Dynamic fields are skipped, since re-reading them is not identity-stable. + * + * The re-read cancels the fast path's savings, so the speedup only shows in + * production builds. + */ +function verifyStaticFields( source: EntitySnapshotSource, keys: EntityKeys, before: Record, snap: SnapshotFn, -): void { +): Record | undefined { const verified: Record = {}; const readsBefore = __debug_snapshotFieldReads; walkFields(source, keys.static, before, snap, verified); __debug_snapshotFieldReads = readsBefore; + let drifted: Record | undefined; for (const key of keys.static) { - if (verified[key] !== before[key]) { - throw new Error( - `[fetchium] stale entity snapshot: ${source.instance.typename}:${source.instance.id} field '${key}' ` + - `changed while version stayed at ${source.instance.version}. Either the field was mutated without ` + - `notify(), or the shape's static/dynamic split is wrong for it. This throws from inside a snapshot, ` + - `so the reactive computation holding it stays in an error state and will rethrow on every read — ` + - `reload rather than chasing the later throws, which are all this one.`, - ); - } + if (Object.is(verified[key], before[key]) || sameSnapshotValue(verified[key], before[key])) continue; + if (drifted === undefined) drifted = {}; + drifted[key] = verified[key]; + reportDrift(source, key, `field '${key}' changed`); } + return drifted; } // Register once on the `Entity` base class. Signalium 3.0.1+ resolves custom @@ -335,6 +471,19 @@ export class EntityInstance { data: Record; refCount: number = 0; _persisted: boolean = false; + /** Writes sent to an acknowledging store, not yet acknowledged. */ + _pendingWrites: number = 0; + /** An apply is still reifying this instance's fields. */ + _applying: boolean = false; + /** Data came only from events, so a write merges over the stored record. */ + _partial: boolean = false; + /** With `_partial`: the fields a write merges. */ + _partialKeys: Set | undefined = undefined; + /** A write waiting for a child's apply to finish. */ + _deferredWrite: boolean = false; + /** The store is known to hold a record of this entity. */ + _recorded: boolean = false; + private _saving: boolean = false; entityRefs: Map | undefined; liveCollections: LiveCollectionBinding[] = []; satisfiedDefs: WeakSet> = new WeakSet(); @@ -406,13 +555,24 @@ export class EntityInstance { } if (oldRefs !== undefined && oldRefs.size > 0) { for (const child of oldRefs.keys()) { - if (newRefs === undefined || !newRefs.has(child)) child.release(); + if (newRefs === undefined || !newRefs.has(child)) { + child.release(); + if (persist) this.writeDropsRef(child); + } } } this.entityRefs = newRefs; if (persist) this.save(); } + /** + * The write may delete the child's record after the current apply, so it is + * rewritten rather than trusted. Sync stores report deletes via `onDelete`. + */ + private writeDropsRef(child: EntityInstance): void { + if (this._queryClient.storeAcksWrites) child.recordDropped(); + } + addChildRef(child: EntityInstance, persist: boolean = true): void { if (this.entityRefs === undefined) this.entityRefs = new Map(); const count = this.entityRefs.get(child) ?? 0; @@ -428,6 +588,7 @@ export class EntityInstance { if (count <= 1) { this.entityRefs.delete(child); child.release(); + if (persist) this.writeDropsRef(child); } else { this.entityRefs.set(child, count - 1); } @@ -457,9 +618,84 @@ export class EntityInstance { return false; } - save(): void { - this._persisted = true; - this._queryClient.entityMap.save(this); + /** + * `storeHolds`: the store's `hasEntity` answer, if the caller already has it. + * `ifStored`: a merge only refreshes a stored record, never creates one. + */ + save(storeHolds?: boolean, ifStored?: boolean): void { + const client = this._queryClient; + if (this._saving) return; + this._saving = true; + try { + // References must point at existing records, so unrecorded children are + // written first. A child still being reified defers both writes. + const refs = this.entityRefs; + if (refs !== undefined) { + let deferred = false; + for (const child of refs.keys()) { + if (child._recorded || child._persisted || child._pendingWrites > 0 || child._saving) continue; + if (client.entityMap.getEntity(child.key) !== child) continue; + if (child._applying) { + client.deferWrite(child); + deferred = true; + } else { + try { + child.save(); + } catch (e) { + this.markUnwritten(); + throw e; + } + if (child._deferredWrite) deferred = true; + } + } + if (deferred) { + this._deferredWrite = true; + client.deferWrite(this); + return; + } + } + this._deferredWrite = false; + // No record to merge over, so the fields held are the whole record. + if (this._partial && !this._recorded && (storeHolds ?? client.store.hasEntity?.(this.key)) === false) { + this._partial = false; + this._partialKeys = undefined; + } + // Counted before the call: a store may acknowledge synchronously. + if (client.storeAcksWrites) this._pendingWrites++; + try { + client.entityMap.save(this, this._partial ? this._partialKeys : undefined, ifStored); + } catch (e) { + this.markUnwritten(); + throw e; + } + // After the call, so a throwing store leaves it unset. + if (!client.storeAcksWrites) this._persisted = this._recorded = true; + } finally { + this._saving = false; + } + } + + /** A live field of this entity gained or lost a member outside an apply. */ + liveFieldChanged(fieldKey: string): void { + if (this._partial) this._partialKeys?.add(fieldKey); + } + + /** Ignores acks for writes a previous instance of this entity dispatched. */ + acknowledgeWrite(): void { + if (this._pendingWrites === 0) return; + if (--this._pendingWrites === 0) this._persisted = this._recorded = true; + } + + /** The store dropped (or failed to write) this entity's record. */ + recordDropped(): void { + this._recorded = false; + this.markUnwritten(); + } + + /** Forces the next apply to write. Queued writes hold older data, so their acks are dropped. */ + markUnwritten(): void { + this._persisted = false; + this._pendingWrites = 0; } notify(): void { @@ -478,18 +714,31 @@ function sameMembers(a: unknown[], b: unknown[]): boolean { return true; } -function filterEntityArray(array: unknown[], innerDef: ValidatorDef, queryClient: QueryClient): unknown[] { - const result: unknown[] = []; +interface NarrowedArray { + source: unknown[]; + filtered: unknown[]; + /** Members not (yet) satisfying the def, the only ones whose membership can change. */ + excluded: EntityInstance[]; + parseId: number; +} + +function filterEntityArray( + array: unknown[], + innerDef: ValidatorDef, + queryClient: QueryClient, +): { filtered: unknown[]; excluded: EntityInstance[] } { + const filtered: unknown[] = []; + const excluded: EntityInstance[] = []; for (const item of array) { if (typeof item !== 'object' || item === null) continue; const entityKey = PROXY_ID.get(item); if (entityKey === undefined) continue; const entityInstance = queryClient.entityMap.getEntity(entityKey); - if (entityInstance !== undefined && entityInstance.satisfiesDef(innerDef)) { - result.push(item); - } + if (entityInstance === undefined) continue; + if (entityInstance.satisfiesDef(innerDef)) filtered.push(item); + else excluded.push(entityInstance); } - return result; + return { filtered, excluded }; } // Fields that can change without the entity notifying (formats may read other signals). @@ -533,8 +782,15 @@ function computeIsStaticFieldDef(def: ValidatorDef): boolean { return true; } -/** Key lists a proxy reports, shared per shape. */ -interface EntityKeys { +// ====================================================== +// Per-shape key metadata +// ====================================================== + +/** + * The key lists a proxy reports, shared per shape. `own` is `ownKeys`; + * `enumerable` drops the non-enumerable entity methods. + */ +export interface EntityKeys { own: string[]; enumerable: string[]; dynamic: string[]; @@ -542,14 +798,22 @@ interface EntityKeys { enumerableSet: Set; } -const shapeKeyCache = new WeakMap, EntityKeys>(); +/** No class sharing the typename can put an entity under a shapeless field. */ +function isShapelessFieldDef(def: unknown): boolean { + if (typeof def === 'number' || typeof def === 'string') return true; + if (def instanceof Set) return true; + return def instanceof ValidatorDef && (def.shape === undefined || def.shape === null); +} +/** Cached per client: the split depends on which classes share the typename. */ function shapeKeys( validatorDef: ValidatorDef, shapeFields: Record, methods: Record unknown> | undefined, + queryClient: QueryClient, ): EntityKeys { - let keys = shapeKeyCache.get(validatorDef); + const cache = queryClient.shapeKeyCache; + let keys = cache.get(validatorDef); if (keys !== undefined) return keys; const own = Object.keys(shapeFields); @@ -560,7 +824,14 @@ function shapeKeys( if (!own.includes(methodKey)) own.push(methodKey); } } - const dynamic = enumerable.filter(key => key !== '__typename' && !isStaticFieldDef(shapeFields[key])); + // Another class sharing the typename may write an entity into a nested object. + const typename = validatorDef.typenameValue; + const shared = typename !== undefined && (queryClient.getEntityDefsForTypename(typename)?.length ?? 0) > 1; + const dynamic = enumerable.filter( + key => + key !== '__typename' && + (!isStaticFieldDef(shapeFields[key]) || (shared && !isShapelessFieldDef(shapeFields[key]))), + ); const dynamicSet = new Set(dynamic); keys = { @@ -570,7 +841,7 @@ function shapeKeys( static: enumerable.filter(key => !dynamicSet.has(key)), enumerableSet: new Set(enumerable), }; - shapeKeyCache.set(validatorDef, keys); + cache.set(validatorDef, keys); return keys; } @@ -617,12 +888,17 @@ function bindMethod( return bound; } -/** Narrows a shared-typename array to this field's def. */ +/** An excluded member that gains the def's fields must recompute the consumer. */ +function consumeExcluded(excluded: EntityInstance[]): void { + for (let i = 0; i < excluded.length; i++) excluded[i].consume(); +} + +/** Narrow a shared-typename array to this field's def, cached on array identity. */ function narrowEntityArray( prop: string, value: unknown[], shapeFields: Record, - filterCache: Map, + filterCache: Map, queryClient: QueryClient, ): unknown[] { const fieldDef = shapeFields[prop]; @@ -637,11 +913,14 @@ function narrowEntityArray( const parseId = queryClient.currentParseId; const cached = filterCache.get(prop); if (cached !== undefined && cached.source === value && cached.parseId === parseId) { + consumeExcluded(cached.excluded); return cached.filtered; } - let filtered = filterEntityArray(value, innerDef, queryClient); + const narrowed = filterEntityArray(value, innerDef, queryClient); + let filtered = narrowed.filtered; if (cached !== undefined && sameMembers(cached.filtered, filtered)) filtered = cached.filtered; - filterCache.set(prop, { source: value, filtered, parseId }); + filterCache.set(prop, { source: value, filtered, excluded: narrowed.excluded, parseId }); + consumeExcluded(narrowed.excluded); return filtered; } } @@ -670,7 +949,7 @@ function createProxy( const typenameField = shape.typenameField; const wrappedMethods = new Map unknown>(); - const filterCache = new Map(); + const filterCache = new Map(); const toJSON = () => ({ __entityRef: key }); @@ -696,21 +975,28 @@ function createProxy( throw new Error(`typenameField "${typenameField}" must be declared in the entity shape`); } - const baseKeys = shapeKeys(validatorDef, shapeFields, methods); + // Shared per shape; only a query's root entity allocates its own lists. + let cachedBaseCache: QueryClient['shapeKeyCache'] | undefined; let cachedMethods: Record | undefined; let cachedGetters: Record | undefined; - let keys = baseKeys; + let keys: EntityKeys | undefined; function entityKeys(): EntityKeys { + // The client replaces its key cache when a class registers for this typename. + const baseCache = queryClient.shapeKeyCache; const methodsNow = instance._extraMethods; const gettersNow = instance._extraGetters; - if (methodsNow !== cachedMethods || gettersNow !== cachedGetters) { + if ( + keys === undefined || + baseCache !== cachedBaseCache || + methodsNow !== cachedMethods || + gettersNow !== cachedGetters + ) { + const base = shapeKeys(validatorDef, shapeFields, methods, queryClient); + cachedBaseCache = baseCache; cachedMethods = methodsNow; cachedGetters = gettersNow; - keys = - methodsNow === undefined && gettersNow === undefined - ? baseKeys - : withExtraKeys(baseKeys, methodsNow, gettersNow); + keys = methodsNow === undefined && gettersNow === undefined ? base : withExtraKeys(base, methodsNow, gettersNow); } return keys; } diff --git a/packages/fetchium/src/EntityStore.ts b/packages/fetchium/src/EntityStore.ts index 89e0b31..32f8522 100644 --- a/packages/fetchium/src/EntityStore.ts +++ b/packages/fetchium/src/EntityStore.ts @@ -3,11 +3,20 @@ import type { QueryClient } from './QueryClient.js'; import { EntityInstance } from './EntityInstance.js'; import { ValidatorDef } from './typeDefs.js'; +type PersistEntity = ( + key: number, + data: Record, + refKeys?: Set, + merge?: boolean, + ifStored?: boolean, +) => void; + export class EntityStore { private instances = new Map(); - private persistEntity: (key: number, data: Record, refKeys?: Set) => void; + private persistEntity: PersistEntity; + mergesEntities: boolean = false; - constructor(persistEntity: (key: number, data: Record, refKeys?: Set) => void) { + constructor(persistEntity: PersistEntity) { this.persistEntity = persistEntity; } @@ -58,12 +67,20 @@ export class EntityStore { this.instances.clear(); } - save(instance: EntityInstance): void { + /** With `mergeKeys`, only those fields are sent, merged over the stored record. */ + save(instance: EntityInstance, mergeKeys?: Set, ifStored?: boolean): void { let refKeys: Set | undefined; if (instance.entityRefs) { refKeys = new Set(); - for (const e of instance.entityRefs.keys()) refKeys.add(e.key); + // A child evicted from memory has no record to reference. + for (const e of instance.entityRefs.keys()) if (this.instances.get(e.key) === e) refKeys.add(e.key); + } + let value = instance.data; + const merge = mergeKeys !== undefined && this.mergesEntities; + if (merge) { + value = {}; + for (const k of mergeKeys) value[k] = instance.data[k]; } - this.persistEntity(instance.key, instance.data, refKeys); + this.persistEntity(instance.key, value, refKeys, merge, ifStored); } } diff --git a/packages/fetchium/src/LiveCollection.ts b/packages/fetchium/src/LiveCollection.ts index 36bd814..f635de0 100644 --- a/packages/fetchium/src/LiveCollection.ts +++ b/packages/fetchium/src/LiveCollection.ts @@ -65,6 +65,8 @@ export interface LiveCollectionParent { addChildRef(child: EntityInstance, persist?: boolean): void; removeChildRef(child: EntityInstance, persist?: boolean): void; save(): void; + /** A live field's membership changed outside an apply. */ + liveFieldChanged?(fieldKey: string): void; } // ====================================================== @@ -152,8 +154,9 @@ export class LiveCollectionBinding { typename: string, entityKey: number, eventType: 'create' | 'update' | 'delete', - onMatch?: () => void, + onMatch?: (willRetain: boolean) => void, deleteData?: Record, + dryRun: boolean = false, ): void { const defs = this._entityDefsByTypename.get(typename); if (defs === undefined) return; @@ -174,8 +177,8 @@ export class LiveCollectionBinding { } const entity = entityInstance !== undefined ? entityInstance.getProxy(def as unknown as EntityDef) : deleteData; if (entity !== undefined) { - this.instance.onEvent(entityKey, entity, deleteData ?? entityInstance?.data ?? {}, 'delete'); - onMatch?.(); + if (!dryRun) this.instance.onEvent(entityKey, entity, deleteData ?? entityInstance?.data ?? {}, 'delete'); + onMatch?.(false); } return; } @@ -188,7 +191,9 @@ export class LiveCollectionBinding { if (def === undefined) return; if (!entityInstance.satisfiesDef(def as unknown as ValidatorDef)) return; - onMatch?.(); + // A live array retains what it adds; a live value only reduces over it. + onMatch?.(this.instance instanceof LiveArrayInstance); + if (dryRun) return; const proxy = entityInstance.getProxy(def as unknown as EntityDef); this.instance.onEvent(entityKey, proxy, entityInstance.data, eventType); } @@ -213,6 +218,8 @@ export class LiveArrayInstance { _queryClient: QueryClient; _parent: LiveCollectionParent; + _fieldKey: string | undefined; + constructor( queryClient: QueryClient, parent: LiveCollectionParent, @@ -220,12 +227,14 @@ export class LiveArrayInstance { constraintFieldPaths?: FieldPath[], constraintHash?: number, sort?: (a: unknown, b: unknown) => number, + fieldKey?: string, ) { this._notifier = notifier(); this._items = items; this._keys = buildKeySet(items); this._queryClient = queryClient; this._parent = parent; + this._fieldKey = fieldKey; const needsFilter = constraintFieldPaths !== undefined && constraintHash !== undefined; const needsSort = sort !== undefined; @@ -308,6 +317,7 @@ export class LiveArrayInstance { this._keys.add(key); this._items.push(proxy); + if (this._fieldKey !== undefined) this._parent.liveFieldChanged?.(this._fieldKey); const child = this._queryClient.entityMap.getEntity(key); if (child !== undefined) { // The child's record is already current. @@ -324,6 +334,7 @@ export class LiveArrayInstance { const idx = this._findIndex(key); if (idx !== -1) this._items.splice(idx, 1); + if (this._fieldKey !== undefined) this._parent.liveFieldChanged?.(this._fieldKey); const child = this._queryClient.entityMap.getEntity(key); if (child !== undefined) { this._parent.removeChildRef(child); @@ -490,6 +501,7 @@ export function createLiveCollection( parent: LiveCollectionParent, parentData: Record, queryClient: QueryClient, + fieldKey?: string, ): LiveCollectionBinding { let constraintFieldRefs = config.constraintFieldRefs; @@ -532,6 +544,7 @@ export function createLiveCollection( arrayConstraintFieldPaths, arrayConstraintHash, config.sort, + fieldKey, ); } else { inner = new LiveValueInstance( @@ -589,7 +602,7 @@ export function initializeLiveFields( } data[fieldName] = existing; } else { - data[fieldName] = createLiveCollection(config, data[fieldName], parent, data, queryClient); + data[fieldName] = createLiveCollection(config, data[fieldName], parent, data, queryClient, fieldName); } } else if ( (fieldDef.mask & Mask.OBJECT) !== 0 && diff --git a/packages/fetchium/src/QueryClient.ts b/packages/fetchium/src/QueryClient.ts index 56c2571..f6cbb9f 100644 --- a/packages/fetchium/src/QueryClient.ts +++ b/packages/fetchium/src/QueryClient.ts @@ -11,7 +11,7 @@ import { } from './types.js'; import { PROXY_ID } from './proxyId.js'; import { EntityStore } from './EntityStore.js'; -import { EntityInstance } from './EntityInstance.js'; +import { EntityInstance, type EntityKeys } from './EntityInstance.js'; import { NetworkManager, NoOpNetworkManager } from './NetworkManager.js'; import { QueryInstance } from './QueryResult.js'; import { MutationResultImpl } from './MutationResult.js'; @@ -133,6 +133,10 @@ export class QueryClient { currentParseId: number = 0; /** Without `store.onDelete`, `_persisted` cannot be trusted. */ storeReportsDeletes: boolean = false; + /** With `store.onPersisted`, `_persisted` is set on acknowledgement, not by `save()`. */ + storeAcksWrites: boolean = false; + /** Replaced when a typename gains a class, since the static/dynamic split depends on it. */ + shapeKeyCache = new WeakMap, EntityKeys>(); readonly reactivationGraceMs: number; readonly reactivationStaggerMs: number; @@ -157,6 +161,7 @@ export class QueryClient { private mergedDefCache = new Map>(); private adapters = new Map(); private networkUnsubscribe: (() => void) | undefined; + private storeUnsubscribes: Array<() => void> = []; private mutationParseContext = new ParseContext(); constructor(config: QueryClientConfig = {}) { @@ -183,13 +188,22 @@ export class QueryClient { config.gcManager ?? (this.isServer ? new NoOpGcManager() : new GcManager(this.handleEviction, evictionMultiplier)); this.networkManager = config.networkManager ?? new NetworkManager(); - this.entityMap = new EntityStore((key, data, refs) => this.store.saveEntity(key, data, refs)); + this.entityMap = new EntityStore((key, data, refs, merge, ifStored) => { + if (merge) this.store.mergeEntity!(key, data, refs, ifStored); + else this.store.saveEntity(key, data, refs); + }); + this.entityMap.mergesEntities = typeof this.store.mergeEntity === 'function'; // A record the store drops must be written again by the next apply. this.storeReportsDeletes = typeof this.store.onDelete === 'function'; - this.store.onDelete?.(key => { - const entity = this.entityMap.getEntity(key); - if (entity !== undefined) entity._persisted = false; + const offDelete = this.store.onDelete?.(key => { + this.entityMap.getEntity(key)?.recordDropped(); + }); + if (typeof offDelete === 'function') this.storeUnsubscribes.push(offDelete); + this.storeAcksWrites = typeof this.store.onPersisted === 'function'; + const offPersisted = this.store.onPersisted?.(key => { + this.entityMap.getEntity(key)?.acknowledgeWrite(); }); + if (typeof offPersisted === 'function') this.storeUnsubscribes.push(offPersisted); // Register user-supplied adapters for (const adapter of config.adapters ?? []) { @@ -276,6 +290,13 @@ export class QueryClient { // Typename Registry (per-client) // ====================================================== + /** No queued store deletion can undo a skipped write. */ + storeIsSettled(): boolean { + const store = this.store; + if (store.hasQueuedDeletes !== undefined) return !store.hasQueuedDeletes(); + return store.isSettled?.() ?? true; + } + private registerEntityDef(def: ValidatorDef): void { const typename = def.typenameValue; if (typename === undefined) return; @@ -289,6 +310,7 @@ export class QueryClient { existing.push(def); this.mergedDefCache.delete(typename); this.getMergedDef(typename); + this.shapeKeyCache = new WeakMap(); } else { this.typenameRegistry.set(typename, [def]); } @@ -586,11 +608,18 @@ export class QueryClient { /** * Parse data: validates, formats, produces parsed entity data objects. * Does NOT touch the entity store. Call applyRefs() after to commit entities. + * `copyInput: false` copies a nested object only where a parsed value differs. */ - parseData(obj: unknown, shape: InternalTypeDef, preloadedEntities?: PreloadedEntityMap): ParseResult { + parseData( + obj: unknown, + shape: InternalTypeDef, + preloadedEntities?: PreloadedEntityMap, + copyInput: boolean = true, + ): ParseResult { const warn = this.context.log?.warn ?? (() => {}); const ctx = new ParseContext(); ctx.reset(this, preloadedEntities, warn); + ctx.copyInput = copyInput; const data = parseEntities(obj, shape as unknown as ComplexTypeDef, ctx); return { data, ctx }; } @@ -626,10 +655,11 @@ export class QueryClient { obj !== null && !('__entityRef' in (obj as Record)) ) { - (obj as Record)[QUERY_ID] = queryId; + // On a copy: the payload belongs to the adapter (it may be frozen). + obj = { ...(obj as Record), [QUERY_ID]: queryId }; } - const parseResult = this.parseData(obj, rootEntityShape as unknown as InternalTypeDef, preloadedEntities); + const parseResult = this.parseData(obj, rootEntityShape as unknown as InternalTypeDef, preloadedEntities, false); const result = applyEntityRefs(parseResult.ctx, parseResult.data, persist, appendMode); // Discover the root entity from the returned proxy @@ -637,8 +667,17 @@ export class QueryClient { return this.entityMap.getEntity(proxyKey!)!; } - prepareEntity(key: number, obj: Record, shape: EntityDef): EntityInstance { + prepareEntity( + key: number, + obj: Record, + shape: EntityDef, + existing?: EntityInstance, + ): EntityInstance { this.registerEntityDef(shape as unknown as ValidatorDef); + if (existing !== undefined) { + existing.parseId = this.currentParseId; + return existing; + } return this.entityMap.getOrCreateEntity(key, obj, shape, this); } @@ -677,13 +716,14 @@ export class QueryClient { return; } + // Entities the event creates are written only once something retains them. + const created = existing === undefined ? new Set() : undefined; try { const warn = this.context.log?.warn ?? (() => {}); const parseCtx = this.mutationParseContext; parseCtx.reset(this, undefined, warn, /* isPartialEvent */ true); const parsedData = parseEntity(data, mergedDef as unknown as EntityDef, parseCtx); - // A new entity is written only once something routes it. - applyEntityRefs(parseCtx, parsedData, /* persist */ existing !== undefined); + applyEntityRefs(parseCtx, parsedData, existing !== undefined ? true : 'existing', false, created); } catch (e) { // Unknown required-union variant: surface as an error (not a silent warn) // so the dropped update is visible. Optional unions degrade during parse. @@ -693,8 +733,8 @@ export class QueryClient { this.context.log?.warn?.('Failed to apply mutation event', e); } if (existing === undefined) { - const created = this.entityMap.getEntity(key); - if (created !== undefined) created.evict(); + const createdRoot = this.entityMap.getEntity(key); + if (createdRoot !== undefined) this.evictCreated(createdRoot, created!); } return; } @@ -702,19 +742,109 @@ export class QueryClient { const entity = this.entityMap.getEntity(key); if (entity === undefined) return; - const wasNew = existing === undefined; + if (existing !== undefined) { + this.routeEvent(typename, entity.data, key, type, eventSource); + return; + } + + // Write a created root if a live array will retain it, else only refresh a + // record the store may hold. Before routing, so a failed write routes nothing. let matched = false; + let retains = false; + this.routeEvent( + typename, + entity.data, + key, + type, + eventSource, + willRetain => { + matched = true; + if (willRetain) retains = true; + }, + undefined, + /* dryRun */ true, + ); + let held: boolean | undefined; + if (!entity._persisted && entity._pendingWrites === 0) { + if (!retains) held = this.store.hasEntity?.(key); + // Unknown: entities created under the root would be written even if the root isn't. + const refresh = + held === true || + (held === undefined && entity._partial && created!.size === 1 && this.entityMap.mergesEntities); + if (retains || refresh) { + try { + entity.save(held, !retains); + } catch (e) { + this.context.log?.warn?.('Failed to apply mutation event', e); + this.evictUnlessAdopted(entity, created!); + return; + } + } + } - this.routeEvent(typename, entity.data, key, type, eventSource, () => { - matched = true; - }); + if (matched) this.routeEvent(typename, entity.data, key, type, eventSource); + else this.evictUnlessAdopted(entity, created!); + } - if (wasNew) { - if (matched) { - // The parent persisted a ref to it, so its record must exist. - persistUnwritten(entity); - } else { - entity.evict(); + /** References from entities the same event created don't count as adoption. */ + private evictUnlessAdopted(root: EntityInstance, created: Set): void { + let createdHolders = 0; + for (const c of created) if (c.entityRefs?.has(root)) createdHolders++; + if (root.refCount <= createdHolders) this.evictCreated(root, created); + } + + /** Also evicts unreferenced entities the event created, so a later event can't write them as orphans. */ + private evictCreated(root: EntityInstance, created: Set): void { + root.evict(); + let evicted = true; + while (evicted) { + evicted = false; + for (const c of created) { + if (c !== root && c.refCount === 0 && this.entityMap.getEntity(c.key) === c) { + c.evict(); + evicted = true; + } + } + } + } + + private deferredWrites: Set | undefined; + + /** @internal */ + deferWrite(entity: EntityInstance): void { + (this.deferredWrites ??= new Set()).add(entity); + } + + /** Children were deferred before parents, so insertion order is write order. */ + /** @internal */ + flushDeferredWrites(): void { + const deferred = this.deferredWrites; + if (deferred === undefined || deferred.size === 0) return; + this.deferredWrites = undefined; + try { + for (const entity of deferred) { + if (entity._applying || this.entityMap.getEntity(entity.key) !== entity) continue; + if (entity._deferredWrite || (!entity._persisted && entity._pendingWrites === 0)) entity.save(); + } + } finally { + for (const entity of deferred) { + if (entity._deferredWrite) { + entity._deferredWrite = false; + entity.markUnwritten(); + } + } + } + } + + /** @internal */ + discardDeferredWrites(): void { + const deferred = this.deferredWrites; + if (deferred === undefined) return; + this.deferredWrites = undefined; + for (const entity of deferred) { + if (entity._deferredWrite) { + entity._deferredWrite = false; + entity.markUnwritten(); } } } @@ -836,14 +966,15 @@ export class QueryClient { entityKey: number, eventType: 'create' | 'update' | 'delete', eventSource: number | undefined, - onMatch?: () => void, + onMatch?: (willRetain: boolean) => void, deleteData?: Record, + dryRun: boolean = false, ): void { const matcher = this.constraintRegistry.get(typename); if (matcher === undefined) return; const data = eventSource !== undefined ? { ...entityData, [EVENT_SOURCE_FIELD]: eventSource } : entityData; - matcher.routeEvent(typename, data, entityKey, eventType, onMatch, deleteData); + matcher.routeEvent(typename, data, entityKey, eventType, onMatch, deleteData, dryRun); } destroy(): void { @@ -853,6 +984,8 @@ export class QueryClient { this.staggerTimer = undefined; this.staggerQueue.clear(); this.networkUnsubscribe?.(); + for (const off of this.storeUnsubscribes) off(); + this.storeUnsubscribes = []; this.gcManager.destroy(); this.networkManager.destroy(); for (const adapter of this.adapters.values()) { @@ -913,12 +1046,3 @@ function paramsMatch(instanceParams: Record | undefined, subset } return true; } - -function persistUnwritten(entity: EntityInstance): void { - if (!entity._persisted) entity.save(); - const refs = entity.entityRefs; - if (refs === undefined) return; - for (const child of refs.keys()) { - if (!child._persisted) persistUnwritten(child); - } -} diff --git a/packages/fetchium/src/QueryResult.ts b/packages/fetchium/src/QueryResult.ts index f28527a..3abec88 100644 --- a/packages/fetchium/src/QueryResult.ts +++ b/packages/fetchium/src/QueryResult.ts @@ -290,6 +290,8 @@ export class QueryInstance { __hasNext: () => this.hasNext, __isFetchingNext: () => this._fetchNextPromise !== undefined, }; + // Deliberately not notified: this query's consumers snapshot fresh via the + // relay, and other consumers of a shared entity don't need these extras. } return this.rootEntity.getProxy(def.statics.shape as unknown as EntityDef) as QueryResult; diff --git a/packages/fetchium/src/__tests__/entity-snapshot.test.ts b/packages/fetchium/src/__tests__/entity-snapshot.test.ts index 2839dfc..475f64c 100644 --- a/packages/fetchium/src/__tests__/entity-snapshot.test.ts +++ b/packages/fetchium/src/__tests__/entity-snapshot.test.ts @@ -2,6 +2,7 @@ import { describe, it, expect } from 'vitest'; import { signal, watcher, withContexts } from 'signalium'; import { hashValue, registerCustomSnapshot, snapshot } from 'signalium/utils'; import { + __setSnapshotDriftHandler, __debug_resetSnapshotCounters, __debug_snapshotFieldReads, __debug_snapshotFullWalks, @@ -627,7 +628,7 @@ describe('Entity Snapshots', () => { expect(after.board.slots[1]).toBe(before.board.slots[2]); }); - it('throws in dev when a field changes without the version moving', async () => { + it('raises in dev when a field changes without the version moving, and serves the re-read value', async () => { const { client, mockFetch } = getClient(); mockFetch.get('/portfolio', portfolio(2)); @@ -639,14 +640,44 @@ describe('Entity Snapshots', () => { const instance = client.entityMap.getEntity(hashValue(['Token', 'tok-0']))!; const original = instance.data.symbol; instance.data.symbol = 'MUTATED'; - client.applyMutationEvent({ type: 'update', typename: 'Token', data: { id: 'tok-1', price: 5 } }); + const raised: Error[] = []; + __setSnapshotDriftHandler(error => raised.push(error)); + try { + client.applyMutationEvent({ type: 'update', typename: 'Token', data: { id: 'tok-1', price: 5 } }); + + const snap = read() as unknown as { tokens: { symbol: string }[] }; + expect(snap.tokens[0].symbol).toBe('MUTATED'); + expect(raised).toHaveLength(1); + expect(raised[0].message).toMatch(/stale entity snapshot: Token:tok-0 field 'symbol'/); + + // Raised once per field. + client.applyMutationEvent({ type: 'update', typename: 'Token', data: { id: 'tok-1', price: 6 } }); + read(); + expect(raised).toHaveLength(1); + } finally { + __setSnapshotDriftHandler(undefined); + instance.data.symbol = original; + instance.notify(); + read(); + } + }); - expect(() => read()).toThrow(/stale entity snapshot/); + it('hands out frozen snapshots in dev, so a consumer mutation fails at its call site', async () => { + const { client, mockFetch } = getClient(); + mockFetch.get('/portfolio', portfolio(2)); - // Restore and notify so the watcher doesn't rethrow after the test. - instance.data.symbol = original; - instance.notify(); - read(); + const { query, read } = snapshotHarness(client, () => fetchQuery(GetPortfolio)); + await query; + const snap = read() as unknown as { tokens: { symbol: string }[] }; + + expect(Object.isFrozen(snap)).toBe(true); + expect(Object.isFrozen(snap.tokens)).toBe(true); + expect(Object.isFrozen(snap.tokens[0])).toBe(true); + expect(() => snap.tokens.sort()).toThrow(TypeError); + expect(() => { + snap.tokens[0].symbol = 'X'; + }).toThrow(TypeError); + expect(snap.tokens[0].symbol).not.toBe('X'); }); it('re-snapshots a union field whose only entity-bearing member is an array', async () => { diff --git a/packages/fetchium/src/__tests__/entity-write-counts.test.ts b/packages/fetchium/src/__tests__/entity-write-counts.test.ts index a602ebd..5ff74f3 100644 --- a/packages/fetchium/src/__tests__/entity-write-counts.test.ts +++ b/packages/fetchium/src/__tests__/entity-write-counts.test.ts @@ -31,8 +31,18 @@ class GetList extends RESTQuery { result = { list: t.entity(List) }; } -function itemSaves(spy: ReturnType, typename = 'Item') { - return spy.mock.calls.filter(([, value]) => (value as Record)?.__typename === typename); +function spyEntityWrites(store: { saveEntity: (...args: any[]) => void; mergeEntity: (...args: any[]) => void }) { + const save = vi.spyOn(store, 'saveEntity'); + const merge = vi.spyOn(store, 'mergeEntity'); + return { + get calls() { + return [...save.mock.calls, ...merge.mock.calls]; + }, + }; +} + +function itemSaves(spy: { calls: unknown[][] }, typename = 'Item') { + return spy.calls.filter(([, value]) => (value as Record)?.__typename === typename); } describe('Entity Write Counts', () => { @@ -47,7 +57,7 @@ describe('Entity Write Counts', () => { await query; }); - const saveEntity = vi.spyOn(store, 'saveEntity'); + const saveEntity = spyEntityWrites(store); client.applyMutationEvent({ type: 'update', typename: 'Item', data: { id: 'i-1', name: 'B' } }); expect(itemSaves(saveEntity)).toHaveLength(1); @@ -63,7 +73,7 @@ describe('Entity Write Counts', () => { await query; }); - const saveEntity = vi.spyOn(store, 'saveEntity'); + const saveEntity = spyEntityWrites(store); for (let i = 0; i < 10; i++) { client.applyMutationEvent({ type: 'update', typename: 'Item', data: { id: 'i-1', name: 'A' } }); @@ -84,7 +94,7 @@ describe('Entity Write Counts', () => { await query; }); - const saveEntity = vi.spyOn(store, 'saveEntity'); + const saveEntity = spyEntityWrites(store); client.applyMutationEvent({ type: 'create', typename: 'Item', @@ -112,7 +122,7 @@ describe('Entity Write Counts', () => { await query; }); - const saveEntity = vi.spyOn(store, 'saveEntity'); + const saveEntity = spyEntityWrites(store); client.applyMutationEvent({ type: 'create', typename: 'Item', @@ -138,7 +148,8 @@ describe('Entity Write Counts', () => { listQuery = query as unknown as typeof listQuery; }); - const saveEntity = vi.spyOn(store, 'saveEntity'); + const saveEntity = spyEntityWrites(store); + // i-1 is already current in the store, so the apply skips its write. The insert must not add one. client.applyMutationEvent({ type: 'update', typename: 'Item', diff --git a/packages/fetchium/src/__tests__/event-persistence.test.ts b/packages/fetchium/src/__tests__/event-persistence.test.ts new file mode 100644 index 0000000..eafeeaf --- /dev/null +++ b/packages/fetchium/src/__tests__/event-persistence.test.ts @@ -0,0 +1,1959 @@ +import { describe, it, expect, vi, afterEach } from 'vitest'; +import { watcher, withContexts } from 'signalium'; +import { hashValue } from 'signalium/utils'; +import { t } from '../typeDefs.js'; +import { Entity } from '../proxy.js'; +import { RESTQuery } from '../rest/index.js'; +import { RESTQueryAdapter } from '../rest/RESTQueryAdapter.js'; +import { fetchQuery } from '../query.js'; +import { QueryClient, QueryClientContext } from '../QueryClient.js'; +import { SyncQueryStore, MemoryPersistentStore } from '../stores/sync.js'; +import { AsyncQueryStore, type AsyncPersistentStore, type StoreMessage } from '../stores/async.js'; +import { valueKeyFor, refCountKeyFor, refIdsKeyFor, queueKeyFor, updatedAtKeyFor } from '../stores/shared.js'; +import { createMockFetch, setupTestClient, sleep } from './utils.js'; + +// What a streamed event leaves in the store, and whether every reference on disk resolves. + +function getDoc(kv: MemoryPersistentStore, key: number): Record | undefined { + const value = kv.getString(valueKeyFor(key)); + return value ? (JSON.parse(value) as Record) : undefined; +} +function refIds(kv: MemoryPersistentStore, key: number): number[] { + return Array.from(kv.getBuffer(refIdsKeyFor(key)) ?? []); +} + +/** Every non-query record must be referenced, and every reference must resolve. */ +function audit(kv: MemoryPersistentStore): { orphans: number[]; dangling: string[] } { + const ids = new Set(); + for (const k of kv.getAllKeys()) { + const m = /^sq:doc:(value|refCount|refIds|updatedAt):(\d+)$/.exec(k); + if (m) ids.add(Number(m[2])); + } + const orphans: number[] = []; + const dangling: string[] = []; + for (const id of ids) { + const hasValue = kv.getString(valueKeyFor(id)) !== undefined; + const isQuery = kv.getNumber(updatedAtKeyFor(id)) !== undefined; + if (hasValue && !isQuery && kv.getNumber(refCountKeyFor(id)) === undefined) orphans.push(id); + for (const ref of kv.getBuffer(refIdsKeyFor(id)) ?? []) { + if (kv.getString(valueKeyFor(ref)) === undefined) dangling.push(`${id} -> ${ref}`); + } + } + return { orphans, dangling }; +} + +function holdQuery(client: QueryClient, start: () => T): T { + return withContexts([[QueryClientContext, client]], () => { + const query = start(); + const w = watcher(() => (query as unknown as { value: unknown }).value); + w.addListener(() => {}); + return query; + }); +} + +const K = (typename: string, id: string | number) => hashValue([typename, id]); + +function spyWrites(store: SyncQueryStore): () => number[] { + const saves = vi.spyOn(store, 'saveEntity'); + const merges = vi.spyOn(store, 'mergeEntity'); + return () => [...saves.mock.calls.map(c => c[0]), ...merges.mock.calls.map(c => c[0])]; +} + +function failWritesOf(store: SyncQueryStore, key: number, shouldFail: () => boolean = () => true): void { + const save = store.saveEntity.bind(store); + const merge = store.mergeEntity.bind(store); + vi.spyOn(store, 'saveEntity').mockImplementation((k, value, refs) => { + if (shouldFail() && k === key) throw new Error('quota exceeded'); + save(k, value, refs); + }); + vi.spyOn(store, 'mergeEntity').mockImplementation((k, fields, refs) => { + if (shouldFail() && k === key) throw new Error('quota exceeded'); + merge(k, fields, refs); + }); +} + +describe('records a streamed event writes', () => { + class Badge extends Entity { + __typename = t.typename('Badge'); + id = t.id; + label = t.string; + } + class User extends Entity { + __typename = t.typename('User'); + id = t.id; + name = t.string; + karma = t.number; + badge = t.optional(t.entity(Badge)); + } + class Comment extends Entity { + __typename = t.typename('Comment'); + id = t.id; + postId = t.string; + body = t.string; + author = t.entity(User); + mentions = t.optional(t.array(t.entity(User))); + } + class Post extends Entity { + __typename = t.typename('Post'); + id = t.id; + title = t.string; + comments = t.liveArray(Comment, { constraints: { postId: (this as any).id } }); + commentCount = t.liveValue(t.number, Comment, { + constraints: { postId: (this as any).id }, + onCreate: (count: number) => count + 1, + onUpdate: (count: number) => count, + onDelete: (count: number) => count - 1, + }); + } + class Thread extends Entity { + __typename = t.typename('Thread'); + id = t.id; + latest = t.liveValue(t.optional(t.string), Comment, { + constraints: { postId: (this as any).id }, + onCreate: (_prev: string | undefined, comment: Comment) => comment.body, + onUpdate: (prev: string | undefined) => prev, + onDelete: (prev: string | undefined) => prev, + }); + } + class GetPost extends RESTQuery { + params = { id: t.id }; + path = `/post/${this.params.id}`; + result = { post: t.entity(Post) }; + } + class GetThread extends RESTQuery { + params = { id: t.id }; + path = `/thread/${this.params.id}`; + result = { thread: t.entity(Thread) }; + } + class GetUser extends RESTQuery { + params = { id: t.id }; + path = `/user/${this.params.id}`; + result = { user: t.entity(User) }; + } + + const getClient = setupTestClient(); + const newAuthor = () => ({ + __typename: 'User', + id: 'u7', + name: 'Newbie', + karma: 1, + badge: { __typename: 'Badge', id: 'b7', label: 'new' }, + }); + + it('the same new entity twice in one payload is referenced, counted and written once', async () => { + const { client, mockFetch, kv, store } = getClient(); + mockFetch.get('/post/p1', { post: { __typename: 'Post', id: 'p1', title: 'T', comments: [], commentCount: 0 } }); + const postQ = holdQuery(client, () => fetchQuery(GetPost, { id: 'p1' })); + await postQ; + const writes = spyWrites(store); + + client.applyMutationEvent({ + type: 'create', + typename: 'Comment', + data: { __typename: 'Comment', id: 'c9', postId: 'p1', body: 'hi', author: newAuthor(), mentions: [newAuthor()] }, + }); + await sleep(5); + + const u7 = client.entityMap.getEntity(K('User', 'u7'))!; + // u7 is referenced twice (author and mentions[0]). + expect(client.entityMap.getEntity(K('Comment', 'c9'))!.entityRefs!.get(u7)).toBe(2); + expect([...u7.entityRefs!.keys()].map(e => e.key)).toEqual([K('Badge', 'b7')]); + expect(writes().filter(k => k === K('User', 'u7'))).toHaveLength(1); + expect(refIds(kv, K('User', 'u7'))).toEqual([K('Badge', 'b7')]); + expect(getDoc(kv, K('Badge', 'b7'))).toMatchObject({ label: 'new' }); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + expect((postQ.value as any).post.comments[0].mentions[0].badge.label).toBe('new'); + }); + + it('the same entity twice in a fetched payload keeps its child references', async () => { + const { client, mockFetch, kv } = getClient(); + class GetComments extends RESTQuery { + path = '/comments'; + result = { comments: t.array(t.entity(Comment)) }; + } + const author = () => ({ + __typename: 'User', + id: 'u1', + name: 'Alice', + karma: 3, + badge: { __typename: 'Badge', id: 'b1', label: 'gold' }, + }); + const payload = () => ({ + comments: [ + { __typename: 'Comment', id: 'c1', postId: 'p1', body: 'a', author: author() }, + { __typename: 'Comment', id: 'c2', postId: 'p1', body: 'b', author: author() }, + ], + }); + mockFetch.get('/comments', payload()); + const q = holdQuery(client, () => fetchQuery(GetComments)); + await q; + + const u1 = client.entityMap.getEntity(K('User', 'u1'))!; + expect(u1.refCount).toBe(2); + expect([...u1.entityRefs!.keys()].map(e => e.key)).toEqual([K('Badge', 'b1')]); + expect(client.entityMap.getEntity(K('Badge', 'b1'))!.refCount).toBe(1); + expect(refIds(kv, K('User', 'u1'))).toEqual([K('Badge', 'b1')]); + expect(getDoc(kv, K('Badge', 'b1'))).toMatchObject({ label: 'gold' }); + expect((q.value as any).comments.map((c: any) => c.author.badge.label)).toEqual(['gold', 'gold']); + + mockFetch.get('/comments', payload()); + await (q.value as any).__refetch(); + expect(u1.refCount).toBe(2); + expect(client.entityMap.getEntity(K('Badge', 'b1'))!.refCount).toBe(1); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + }); + + it('a create matched only by a liveValue reducer writes nothing, so no orphan record is left', async () => { + const { client, mockFetch, kv, store } = getClient(); + mockFetch.get('/thread/p1', { thread: { __typename: 'Thread', id: 'p1', latest: null } }); + const threadQ = holdQuery(client, () => fetchQuery(GetThread, { id: 'p1' })); + await threadQ; + const writes = spyWrites(store); + + client.applyMutationEvent({ + type: 'create', + typename: 'Comment', + data: { __typename: 'Comment', id: 'c9', postId: 'p1', body: 'newest', author: newAuthor() }, + }); + await sleep(5); + + expect((threadQ.value as any).thread.latest).toBe('newest'); + expect(writes()).toEqual([]); + expect(getDoc(kv, K('User', 'u7'))).toBeUndefined(); + expect(getDoc(kv, K('Badge', 'b7'))).toBeUndefined(); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + }); + + it('a partial update for an entity on disk but not in memory merges into its record', async () => { + const { client, mockFetch, kv } = getClient(); + mockFetch.get('/user/u1', { + user: { + __typename: 'User', + id: 'u1', + name: 'Alice', + karma: 10, + badge: { __typename: 'Badge', id: 'b1', label: 'gold' }, + }, + }); + await holdQuery(client, () => fetchQuery(GetUser, { id: 'u1' })); + client.entityMap.getEntity(K('User', 'u1'))!.evict(); + expect(client.entityMap.getEntity(K('User', 'u1'))).toBeUndefined(); + + // Required fields only: an event for an entity not in memory parses as a full record. + client.applyMutationEvent({ + type: 'update', + typename: 'User', + data: { __typename: 'User', id: 'u1', name: 'Alicia', karma: 10 }, + }); + await sleep(5); + + expect(getDoc(kv, K('User', 'u1'))).toMatchObject({ name: 'Alicia', karma: 10 }); + expect(refIds(kv, K('User', 'u1'))).toEqual([K('Badge', 'b1')]); + expect(getDoc(kv, K('Badge', 'b1'))).toMatchObject({ label: 'gold' }); + expect(kv.getNumber(refCountKeyFor(K('Badge', 'b1')))).toBe(1); + expect(client.entityMap.getEntity(K('User', 'u1'))).toBeUndefined(); + + client.applyMutationEvent({ + type: 'update', + typename: 'User', + data: { + __typename: 'User', + id: 'u1', + name: 'Alicia', + karma: 10, + badge: { __typename: 'Badge', id: 'b9', label: 'platinum' }, + }, + }); + await sleep(5); + expect(getDoc(kv, K('User', 'u1'))).toMatchObject({ name: 'Alicia', karma: 10 }); + expect(refIds(kv, K('User', 'u1'))).toEqual([K('Badge', 'b9')]); + expect(getDoc(kv, K('Badge', 'b9'))).toMatchObject({ label: 'platinum' }); + expect(getDoc(kv, K('Badge', 'b1'))).toBeUndefined(); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + + client.destroy(); + const warn = vi.fn(); + const second = new QueryClient({ + store: new SyncQueryStore(kv), + adapters: [new RESTQueryAdapter({ fetch: mockFetch as never, baseUrl: 'http://localhost' })], + log: { warn, error: () => {} }, + } as never); + mockFetch.get('/user/u1', { user: { __typename: 'User', id: 'u1', name: 'FRESH', karma: 0 } }, { delay: 5000 }); + const q = holdQuery(second, () => fetchQuery(GetUser, { id: 'u1' })); + await sleep(10); + expect((q.value as any).user.name).toBe('Alicia'); + expect((q.value as any).user.karma).toBe(10); + expect((q.value as any).user.badge.label).toBe('platinum'); + expect(warn).not.toHaveBeenCalled(); + second.destroy(); + }); + + it('a store write that fails after routing is reverted, and the failure is logged rather than thrown', async () => { + const { client, mockFetch, kv, store } = getClient(); + mockFetch.get('/post/p1', { post: { __typename: 'Post', id: 'p1', title: 'T', comments: [], commentCount: 0 } }); + const postQ = holdQuery(client, () => fetchQuery(GetPost, { id: 'p1' })); + await postQ; + const warn = vi + .spyOn(client.getContext().log as { warn: (...args: unknown[]) => void }, 'warn') + .mockImplementation(() => {}); + failWritesOf(store, K('Comment', 'c9')); + + expect(() => + client.applyMutationEvent({ + type: 'create', + typename: 'Comment', + data: { __typename: 'Comment', id: 'c9', postId: 'p1', body: 'hi', author: newAuthor() }, + }), + ).not.toThrow(); + await sleep(5); + + expect(warn).toHaveBeenCalledWith('Failed to apply mutation event', expect.any(Error)); + // The author written before the failing write stays behind unreferenced. + expect((postQ.value as any).post.comments).toEqual([]); + expect((postQ.value as any).post.commentCount).toBe(0); + expect(client.entityMap.getEntity(K('Comment', 'c9'))).toBeUndefined(); + expect(refIds(kv, K('Post', 'p1'))).toEqual([]); + expect(getDoc(kv, K('Comment', 'c9'))).toBeUndefined(); + expect(audit(kv).dangling).toEqual([]); + }); + + it('a store write that fails for an unrouted root does not leak the root', async () => { + const { client, mockFetch, store } = getClient(); + mockFetch.get('/user/u1', { user: { __typename: 'User', id: 'u1', name: 'Alice', karma: 10 } }); + await holdQuery(client, () => fetchQuery(GetUser, { id: 'u1' })); + client.entityMap.getEntity(K('User', 'u1'))!.evict(); + const warn = vi + .spyOn(client.getContext().log as { warn: (...args: unknown[]) => void }, 'warn') + .mockImplementation(() => {}); + failWritesOf(store, K('User', 'u1')); + + client.applyMutationEvent({ + type: 'update', + typename: 'User', + data: { __typename: 'User', id: 'u1', name: 'Alicia', karma: 10 }, + }); + await sleep(5); + + expect(warn).toHaveBeenCalledWith('Failed to apply mutation event', expect.any(Error)); + expect(client.entityMap.getEntity(K('User', 'u1'))).toBeUndefined(); + }); + + it('a failed create delivered again is routed once: the array and the reducer agree', async () => { + const { client, mockFetch, kv, store } = getClient(); + mockFetch.get('/post/p1', { post: { __typename: 'Post', id: 'p1', title: 'T', comments: [], commentCount: 0 } }); + const postQ = holdQuery(client, () => fetchQuery(GetPost, { id: 'p1' })); + await postQ; + vi.spyOn(client.getContext().log as { warn: (...args: unknown[]) => void }, 'warn').mockImplementation(() => {}); + let fail = true; + failWritesOf(store, K('Comment', 'c9'), () => fail); + const event = () => ({ + type: 'create' as const, + typename: 'Comment', + data: { __typename: 'Comment', id: 'c9', postId: 'p1', body: 'hi', author: newAuthor() }, + }); + + client.applyMutationEvent(event()); + await sleep(5); + expect((postQ.value as any).post.comments).toEqual([]); + expect((postQ.value as any).post.commentCount).toBe(0); + + fail = false; + client.applyMutationEvent(event()); + await sleep(5); + expect((postQ.value as any).post.comments.map((c: any) => c.id)).toEqual(['c9']); + expect((postQ.value as any).post.commentCount).toBe(1); + expect(getDoc(kv, K('Comment', 'c9'))).toMatchObject({ body: 'hi' }); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + }); + + it('an update matched only by a liveValue reducer refreshes the record of an entity on disk but not in memory', async () => { + const { client, mockFetch, kv } = getClient(); + class GetComments extends RESTQuery { + path = '/comments'; + result = { comments: t.array(t.entity(Comment)) }; + } + mockFetch.get('/thread/p1', { thread: { __typename: 'Thread', id: 'p1', latest: null } }); + mockFetch.get('/comments', { + comments: [ + { + __typename: 'Comment', + id: 'c1', + postId: 'p1', + body: 'orig', + author: { __typename: 'User', id: 'u1', name: 'Alice', karma: 1 }, + }, + ], + }); + const threadQ = holdQuery(client, () => fetchQuery(GetThread, { id: 'p1' })); + await threadQ; + await holdQuery(client, () => fetchQuery(GetComments)); + // Collected from memory. Its record stays on disk. + client.entityMap.getEntity(K('Comment', 'c1'))!.evict(); + + client.applyMutationEvent({ + type: 'update', + typename: 'Comment', + data: { + __typename: 'Comment', + id: 'c1', + postId: 'p1', + body: 'edited', + author: { __typename: 'User', id: 'u1', name: 'Alice', karma: 1 }, + }, + }); + await sleep(5); + + expect((threadQ.value as any).thread.latest).toBeUndefined(); + expect(getDoc(kv, K('Comment', 'c1'))).toMatchObject({ body: 'edited' }); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + }); + + it('a create adopted by an existing entity in its own payload is kept and written, even when nothing routes it', async () => { + const { client, mockFetch, kv } = getClient(); + class Note extends Entity { + __typename = t.typename('Note'); + id = t.id; + body = t.string; + article = t.optional(t.entity(ArticleRef)); + } + class NoteRef extends Entity { + __typename = t.typename('Note'); + id = t.id; + body = t.string; + } + class ArticleRef extends Entity { + __typename = t.typename('Article'); + id = t.id; + latestNote = t.optional(t.entity(NoteRef)); + } + class Article extends Entity { + __typename = t.typename('Article'); + id = t.id; + title = t.string; + latestNote = t.optional(t.entity(NoteRef)); + notes = t.array(t.entity(Note)); + } + class GetArticle extends RESTQuery { + params = { id: t.id }; + path = `/article/${this.params.id}`; + result = { article: t.entity(Article) }; + } + mockFetch.get('/article/a1', { + article: { + __typename: 'Article', + id: 'a1', + title: 'A', + notes: [{ __typename: 'Note', id: 'n1', body: 'first' }], + }, + }); + const artQ = holdQuery(client, () => fetchQuery(GetArticle, { id: 'a1' })); + await artQ; + + // The payload's article points back at the note. + client.applyMutationEvent({ + type: 'create', + typename: 'Note', + data: { + __typename: 'Note', + id: 'n9', + body: 'hi', + article: { __typename: 'Article', id: 'a1', latestNote: { __typename: 'Note', id: 'n9', body: 'hi' } }, + }, + }); + await sleep(5); + + const n9 = client.entityMap.getEntity(K('Note', 'n9'))!; + expect(n9).toBeDefined(); + expect(n9.refCount).toBe(1); + expect((artQ.value as any).article.latestNote.body).toBe('hi'); + expect(refIds(kv, K('Article', 'a1')).sort()).toEqual([K('Note', 'n1'), K('Note', 'n9')].sort()); + expect(getDoc(kv, K('Note', 'n9'))).toMatchObject({ body: 'hi' }); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + + client.applyMutationEvent({ type: 'update', typename: 'Note', data: { id: 'n9', body: 'edited' } }); + expect((artQ.value as any).article.latestNote.body).toBe('edited'); + expect(getDoc(kv, K('Note', 'n9'))).toMatchObject({ body: 'edited' }); + + client.destroy(); + const second = new QueryClient({ + store: new SyncQueryStore(kv), + adapters: [new RESTQueryAdapter({ fetch: mockFetch as never, baseUrl: 'http://localhost' })], + log: { warn: () => {}, error: () => {} }, + } as never); + mockFetch.get( + '/article/a1', + { article: { __typename: 'Article', id: 'a1', title: 'FRESH', notes: [] } }, + { delay: 5000 }, + ); + const q = holdQuery(second, () => fetchQuery(GetArticle, { id: 'a1' })); + await sleep(10); + expect((q.value as any).article.latestNote.body).toBe('edited'); + second.destroy(); + }); + + it('created entities that only reference each other are evicted with the unrouted root', async () => { + const { client, mockFetch, kv } = getClient(); + class Author extends Entity { + __typename = t.typename('User'); + id = t.id; + name = t.string; + karma = t.number; + bestComment = t.optional(t.entity(CommentRef)); + } + class CommentRef extends Entity { + __typename = t.typename('Comment'); + id = t.id; + body = t.string; + } + class Remark extends Entity { + __typename = t.typename('Comment'); + id = t.id; + postId = t.string; + body = t.string; + author = t.entity(Author); + } + class GetRemark extends RESTQuery { + params = { id: t.id }; + path = `/remark/${this.params.id}`; + result = { remark: t.entity(Remark) }; + } + mockFetch.get('/remark/c0', { + remark: { + __typename: 'Comment', + id: 'c0', + postId: 'p0', + body: 'seed', + author: { __typename: 'User', id: 'u0', name: 'Zero', karma: 0 }, + }, + }); + await holdQuery(client, () => fetchQuery(GetRemark, { id: 'c0' })); + + client.applyMutationEvent({ + type: 'create', + typename: 'Comment', + data: { + __typename: 'Comment', + id: 'c9', + postId: 'nowhere', + body: 'hi', + author: { + __typename: 'User', + id: 'u7', + name: 'Newbie', + karma: 1, + bestComment: { __typename: 'Comment', id: 'c9', body: 'hi' }, + }, + }, + }); + await sleep(5); + + expect(client.entityMap.getEntity(K('Comment', 'c9'))).toBeUndefined(); + expect(client.entityMap.getEntity(K('User', 'u7'))).toBeUndefined(); + expect(getDoc(kv, K('Comment', 'c9'))).toBeUndefined(); + expect(getDoc(kv, K('User', 'u7'))).toBeUndefined(); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + }); + + it('a root write that fails while an existing entity adopted the root leaves no dangling reference', async () => { + const { client, mockFetch, kv, store } = getClient(); + class PostRef extends Entity { + __typename = t.typename('Post'); + id = t.id; + latestComment = t.optional(t.entity(CommentRef)); + } + class CommentRef extends Entity { + __typename = t.typename('Comment'); + id = t.id; + body = t.string; + } + class Reply extends Entity { + __typename = t.typename('Comment'); + id = t.id; + postId = t.string; + body = t.string; + author = t.entity(User); + post = t.optional(t.entity(PostRef)); + } + class Topic extends Entity { + __typename = t.typename('Post'); + id = t.id; + title = t.string; + latestComment = t.optional(t.entity(CommentRef)); + replies = t.liveArray(Reply, { constraints: { postId: (this as any).id } }); + } + class GetTopic extends RESTQuery { + params = { id: t.id }; + path = `/topic/${this.params.id}`; + result = { topic: t.entity(Topic) }; + } + mockFetch.get('/topic/t1', { topic: { __typename: 'Post', id: 't1', title: 'T', replies: [] } }); + const topicQ = holdQuery(client, () => fetchQuery(GetTopic, { id: 't1' })); + await topicQ; + vi.spyOn(client.getContext().log as { warn: (...args: unknown[]) => void }, 'warn').mockImplementation(() => {}); + failWritesOf(store, K('Comment', 'c9')); + + client.applyMutationEvent({ + type: 'create', + typename: 'Comment', + data: { + __typename: 'Comment', + id: 'c9', + postId: 't1', + body: 'hi', + author: newAuthor(), + post: { __typename: 'Post', id: 't1', latestComment: { __typename: 'Comment', id: 'c9', body: 'hi' } }, + }, + }); + await sleep(5); + + // Memory holds the adoption and the next write heals the record. + expect(audit(kv).dangling).toEqual([]); + expect((topicQ.value as any).topic.replies).toEqual([]); + expect((topicQ.value as any).topic.latestComment.body).toBe('hi'); + expect(refIds(kv, K('Post', 't1'))).toEqual([]); + }); + + it('a payload that links back to its parent is applied, persisted and hydrated', async () => { + const { client, mockFetch, kv } = getClient(); + class Folder extends Entity { + __typename = t.typename('Folder'); + id = t.id; + name = t.string; + files = t.array(t.entity(File)); + } + class FolderRef extends Entity { + __typename = t.typename('Folder'); + id = t.id; + name = t.string; + } + class File extends Entity { + __typename = t.typename('File'); + id = t.id; + name = t.string; + folder = t.entity(FolderRef); + } + class GetFolder extends RESTQuery { + params = { id: t.id }; + path = `/folder/${this.params.id}`; + result = { folder: t.entity(Folder) }; + } + mockFetch.get('/folder/f1', { + folder: { + __typename: 'Folder', + id: 'f1', + name: 'root', + files: [ + { __typename: 'File', id: 'x1', name: 'a.txt', folder: { __typename: 'Folder', id: 'f1', name: 'root' } }, + ], + }, + }); + const q = holdQuery(client, () => fetchQuery(GetFolder, { id: 'f1' })); + await q; + expect((q.value as any).folder.files[0].folder.name).toBe('root'); + expect(refIds(kv, K('Folder', 'f1'))).toEqual([K('File', 'x1')]); + expect(refIds(kv, K('File', 'x1'))).toEqual([K('Folder', 'f1')]); + + client.destroy(); + const warn = vi.fn(); + const second = new QueryClient({ + store: new SyncQueryStore(kv), + adapters: [new RESTQueryAdapter({ fetch: mockFetch as never, baseUrl: 'http://localhost' })], + log: { warn, error: () => {} }, + } as never); + mockFetch.get( + '/folder/f1', + { folder: { __typename: 'Folder', id: 'f1', name: 'FRESH', files: [] } }, + { delay: 5000 }, + ); + const q2 = holdQuery(second, () => fetchQuery(GetFolder, { id: 'f1' })); + await sleep(10); + expect(warn).not.toHaveBeenCalled(); + expect((q2.value as any).folder.files.map((f: any) => f.folder.name)).toEqual(['root']); + second.destroy(); + }); + + it('merging keeps the fields an event did not carry, including a parse result, and merges created descendants too', async () => { + const { client, mockFetch, kv } = getClient(); + class Icon extends Entity { + __typename = t.typename('Icon'); + id = t.id; + url = t.string; + } + class Award extends Entity { + __typename = t.typename('Badge'); + id = t.id; + label = t.string; + icon = t.optional(t.entity(Icon)); + } + class Profile extends Entity { + __typename = t.typename('Profile'); + id = t.id; + name = t.string; + bio = t.optional(t.string); + badge = t.optional(t.entity(Award)); + parsed = t.optional(t.result(t.entity(Award))); + } + class GetProfile extends RESTQuery { + params = { id: t.id }; + path = `/profile/${this.params.id}`; + result = { profile: t.entity(Profile) }; + } + mockFetch.get('/profile/pr1', { + profile: { + __typename: 'Profile', + id: 'pr1', + name: 'Alice', + bio: 'hello', + badge: { + __typename: 'Badge', + id: 'b1', + label: 'gold', + icon: { __typename: 'Icon', id: 'i1', url: '/gold.png' }, + }, + parsed: { __typename: 'Badge', id: 'b5', label: 'parsed' }, + }, + }); + await holdQuery(client, () => fetchQuery(GetProfile, { id: 'pr1' })); + for (const key of ['pr1']) client.entityMap.getEntity(K('Profile', key))!.evict(); + client.entityMap.getEntity(K('Badge', 'b1'))?.evict(); + client.entityMap.getEntity(K('Icon', 'i1'))?.evict(); + + // A created badge with a record on disk: its merge keeps the icon. + client.applyMutationEvent({ + type: 'update', + typename: 'Profile', + data: { + __typename: 'Profile', + id: 'pr1', + name: 'Alicia', + badge: { __typename: 'Badge', id: 'b1', label: 'gold+' }, + }, + }); + await sleep(5); + + const doc = getDoc(kv, K('Profile', 'pr1'))!; + expect(doc).toMatchObject({ name: 'Alicia', bio: 'hello' }); + expect(doc.parsed).toEqual({ success: true, value: { __entityRef: K('Badge', 'b5') } }); + expect(getDoc(kv, K('Badge', 'b1'))).toMatchObject({ label: 'gold+', icon: { __entityRef: K('Icon', 'i1') } }); + expect(refIds(kv, K('Badge', 'b1'))).toEqual([K('Icon', 'i1')]); + expect(getDoc(kv, K('Icon', 'i1'))).toMatchObject({ url: '/gold.png' }); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + }); + + it('a write after a cold start does not rewrite the hydrated entities under it', async () => { + const { client, mockFetch, kv } = getClient(); + const comments = Array.from({ length: 5 }, (_, i) => ({ + __typename: 'Comment', + id: `c${i}`, + postId: 'p1', + body: `b${i}`, + author: { + __typename: 'User', + id: `u${i}`, + name: `n${i}`, + karma: i, + badge: { __typename: 'Badge', id: `b${i}`, label: `l${i}` }, + }, + })); + mockFetch.get('/post/p1', { post: { __typename: 'Post', id: 'p1', title: 'T', comments, commentCount: 5 } }); + await holdQuery(client, () => fetchQuery(GetPost, { id: 'p1' })); + client.destroy(); + + const store = new SyncQueryStore(kv); + const second = new QueryClient({ + store, + adapters: [new RESTQueryAdapter({ fetch: mockFetch as never, baseUrl: 'http://localhost' })], + log: { warn: () => {}, error: () => {} }, + } as never); + mockFetch.get( + '/post/p1', + { post: { __typename: 'Post', id: 'p1', title: 'FRESH', comments: [], commentCount: 0 } }, + { delay: 5000 }, + ); + const q = holdQuery(second, () => fetchQuery(GetPost, { id: 'p1' })); + await sleep(10); + expect((q.value as any).post.comments).toHaveLength(5); + const writes = spyWrites(store); + + second.applyMutationEvent({ type: 'update', typename: 'Post', data: { id: 'p1', title: 'T2' } }); + await sleep(5); + expect(writes()).toEqual([K('Post', 'p1')]); + expect(getDoc(kv, K('Post', 'p1'))).toMatchObject({ title: 'T2' }); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + second.destroy(); + }); + + it('a member routed into the live array of an entity built from events reaches its record', async () => { + const { client, mockFetch, kv } = getClient(); + // Without the live fields, a second class makes them optional in events. + class PostPreview extends Entity { + __typename = t.typename('Post'); + id = t.id; + title = t.string; + blogId = t.optional(t.string); + } + class Blog extends Entity { + __typename = t.typename('Blog'); + id = t.id; + posts = t.liveArray(PostPreview, { constraints: { blogId: (this as any).id } }); + } + class GetBlog extends RESTQuery { + params = { id: t.id }; + path = `/blog/${this.params.id}`; + result = { blog: t.entity(Blog) }; + } + mockFetch.get('/post/p9', { + post: { + __typename: 'Post', + id: 'p9', + title: 'T', + comments: [ + { + __typename: 'Comment', + id: 'c1', + postId: 'p9', + body: 'first', + author: { __typename: 'User', id: 'u1', name: 'A', karma: 1 }, + }, + ], + commentCount: 1, + }, + }); + mockFetch.get('/blog/b1', { blog: { __typename: 'Blog', id: 'b1', posts: [] } }); + const postQ = holdQuery(client, () => fetchQuery(GetPost, { id: 'p9' })); + await postQ; + const blogQ = holdQuery(client, () => fetchQuery(GetBlog, { id: 'b1' })); + await blogQ; + expect(refIds(kv, K('Post', 'p9'))).toEqual([K('Comment', 'c1')]); + client.entityMap.getEntity(K('Post', 'p9'))!.evict(); + + // Rebuilt from the event alone: no comments, no count. + client.applyMutationEvent({ + type: 'update', + typename: 'Post', + data: { __typename: 'Post', id: 'p9', title: 'T2', blogId: 'b1', commentCount: 1 }, + }); + await sleep(5); + expect((blogQ.value as any).blog.posts.map((p: any) => p.title)).toEqual(['T2']); + const p9 = client.entityMap.getEntity(K('Post', 'p9'))!; + expect(p9._partial).toBe(true); + expect(getDoc(kv, K('Post', 'p9'))).toMatchObject({ title: 'T2', comments: [{ __entityRef: K('Comment', 'c1') }] }); + + client.applyMutationEvent({ + type: 'create', + typename: 'Comment', + data: { __typename: 'Comment', id: 'c9', postId: 'p9', body: 'second', author: newAuthor() }, + }); + await sleep(5); + expect(getDoc(kv, K('Post', 'p9'))).toMatchObject({ title: 'T2', comments: [{ __entityRef: K('Comment', 'c9') }] }); + expect(refIds(kv, K('Post', 'p9'))).toEqual([K('Comment', 'c9')]); + expect(getDoc(kv, K('Comment', 'c9'))).toMatchObject({ body: 'second' }); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + }); + + it('an unrouted root that references itself is evicted', async () => { + const { client, mockFetch, kv } = getClient(); + class MessageRef extends Entity { + __typename = t.typename('Message'); + id = t.id; + body = t.string; + } + class Message extends Entity { + __typename = t.typename('Message'); + id = t.id; + roomId = t.string; + body = t.string; + threadRoot = t.optional(t.entity(MessageRef)); + } + class Room extends Entity { + __typename = t.typename('Room'); + id = t.id; + messages = t.liveArray(Message, { constraints: { roomId: (this as any).id } }); + } + class GetRoom extends RESTQuery { + params = { id: t.id }; + path = `/room/${this.params.id}`; + result = { room: t.entity(Room) }; + } + mockFetch.get('/room/r1', { room: { __typename: 'Room', id: 'r1', messages: [] } }); + await holdQuery(client, () => fetchQuery(GetRoom, { id: 'r1' })); + + client.applyMutationEvent({ + type: 'create', + typename: 'Message', + data: { + __typename: 'Message', + id: 'm9', + roomId: 'OTHER', + body: 'hi', + threadRoot: { __typename: 'Message', id: 'm9', body: 'hi' }, + }, + }); + await sleep(5); + expect(client.entityMap.getEntity(K('Message', 'm9'))).toBeUndefined(); + expect(getDoc(kv, K('Message', 'm9'))).toBeUndefined(); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + }); + + it('an entity an event creates under a root in memory merges over its record too', async () => { + const { client, mockFetch, kv } = getClient(); + mockFetch.get('/post/p1', { post: { __typename: 'Post', id: 'p1', title: 'T', comments: [], commentCount: 0 } }); + mockFetch.get('/user/u7', { user: { ...newAuthor(), name: 'Old' } }); + const postQ = holdQuery(client, () => fetchQuery(GetPost, { id: 'p1' })); + await postQ; + await holdQuery(client, () => fetchQuery(GetUser, { id: 'u7' })); + client.entityMap.getEntity(K('User', 'u7'))!.evict(); + client.entityMap.getEntity(K('Badge', 'b7'))?.evict(); + + client.applyMutationEvent({ + type: 'create', + typename: 'Comment', + data: { + __typename: 'Comment', + id: 'c9', + postId: 'p1', + body: 'hi', + author: { __typename: 'User', id: 'u7', name: 'Newbie', karma: 2 }, + }, + }); + await sleep(5); + expect((postQ.value as any).post.comments.map((c: any) => c.author.name)).toEqual(['Newbie']); + expect(getDoc(kv, K('User', 'u7'))).toMatchObject({ + name: 'Newbie', + karma: 2, + badge: { __entityRef: K('Badge', 'b7') }, + }); + expect(getDoc(kv, K('Badge', 'b7'))).toMatchObject({ label: 'new' }); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + }); +}); + +describe('records a streamed event writes when released entities linger (gcTime)', () => { + class Badge extends Entity { + static cache = { gcTime: 1 }; + __typename = t.typename('Badge'); + id = t.id; + label = t.string; + } + class User extends Entity { + static cache = { gcTime: 1 }; + __typename = t.typename('User'); + id = t.id; + name = t.string; + badge = t.optional(t.entity(Badge)); + } + class Comment extends Entity { + static cache = { gcTime: 1 }; + __typename = t.typename('Comment'); + id = t.id; + postId = t.string; + body = t.string; + author = t.entity(User); + } + class Post extends Entity { + __typename = t.typename('Post'); + id = t.id; + title = t.string; + comments = t.liveArray(Comment, { constraints: { postId: (this as any).id } }); + } + class GetPost extends RESTQuery { + params = { id: t.id }; + path = `/post/${this.params.id}`; + result = { post: t.entity(Post) }; + } + const getClient = setupTestClient({ evictionMultiplier: 0.001 }); + const newAuthor = () => ({ + __typename: 'User', + id: 'u7', + name: 'Newbie', + badge: { __typename: 'Badge', id: 'b7', label: 'new' }, + }); + + it('an unrouted create evicts everything it created at once, gcTime or not, and writes none of it', async () => { + const { client, mockFetch, kv, store } = getClient(); + mockFetch.get('/post/p1', { post: { __typename: 'Post', id: 'p1', title: 'T', comments: [] } }); + const postQ = holdQuery(client, () => fetchQuery(GetPost, { id: 'p1' })); + await postQ; + const writes = spyWrites(store); + + client.applyMutationEvent({ + type: 'create', + typename: 'Comment', + data: { __typename: 'Comment', id: 'c9', postId: 'OTHER', body: 'x', author: newAuthor() }, + }); + await sleep(5); + + // Had the author lingered until gcTime, a later event could write it as an orphan. + expect(client.entityMap.getEntity(K('Comment', 'c9'))).toBeUndefined(); + expect(client.entityMap.getEntity(K('User', 'u7'))).toBeUndefined(); + expect(client.entityMap.getEntity(K('Badge', 'b7'))).toBeUndefined(); + expect(writes()).toEqual([]); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + + client.applyMutationEvent({ + type: 'create', + typename: 'Comment', + data: { __typename: 'Comment', id: 'c9b', postId: 'OTHER', body: 'x', author: newAuthor() }, + }); + await sleep(5); + expect(writes()).toEqual([]); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + + client.applyMutationEvent({ + type: 'create', + typename: 'Comment', + data: { __typename: 'Comment', id: 'c10', postId: 'p1', body: 'y', author: newAuthor() }, + }); + await sleep(5); + expect((postQ.value as any).post.comments.map((c: any) => c.author.badge.label)).toEqual(['new']); + expect(getDoc(kv, K('Comment', 'c10'))).toMatchObject({ body: 'y' }); + expect(getDoc(kv, K('User', 'u7'))).toMatchObject({ name: 'Newbie' }); + expect(getDoc(kv, K('Badge', 'b7'))).toMatchObject({ label: 'new' }); + expect(refIds(kv, K('User', 'u7'))).toEqual([K('Badge', 'b7')]); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + + await sleep(300); + expect(client.entityMap.getEntity(K('User', 'u7'))).toBeDefined(); + expect(client.entityMap.getEntity(K('Badge', 'b7'))).toBeDefined(); + }); + + it('an unrouted create followed by GC leaves nothing in memory or on disk', async () => { + const { client, mockFetch, kv } = getClient(); + mockFetch.get('/post/p1', { post: { __typename: 'Post', id: 'p1', title: 'T', comments: [] } }); + await holdQuery(client, () => fetchQuery(GetPost, { id: 'p1' })); + + client.applyMutationEvent({ + type: 'create', + typename: 'Comment', + data: { __typename: 'Comment', id: 'c9', postId: 'OTHER', body: 'x', author: newAuthor() }, + }); + await sleep(300); + + expect(client.entityMap.getEntity(K('User', 'u7'))).toBeUndefined(); + expect(client.entityMap.getEntity(K('Badge', 'b7'))).toBeUndefined(); + expect(getDoc(kv, K('User', 'u7'))).toBeUndefined(); + expect(getDoc(kv, K('Badge', 'b7'))).toBeUndefined(); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + }); +}); + +// AsyncQueryStore writer + +class AsyncDelegate implements AsyncPersistentStore { + kv: Record = {}; + constructor(public delayMs: number = 1) {} + private tick() { + return new Promise(resolve => setTimeout(resolve, this.delayMs)); + } + async has(key: string) { + await this.tick(); + return key in this.kv; + } + async getString(key: string) { + await this.tick(); + return this.kv[key] as string | undefined; + } + async setString(key: string, value: string) { + await this.tick(); + this.kv[key] = value; + } + async getNumber(key: string) { + await this.tick(); + return this.kv[key] as number | undefined; + } + async setNumber(key: string, value: number) { + await this.tick(); + this.kv[key] = value; + } + async getBuffer(key: string) { + await this.tick(); + return this.kv[key] as Uint32Array | undefined; + } + async setBuffer(key: string, value: Uint32Array) { + await this.tick(); + this.kv[key] = value; + } + async delete(key: string) { + await this.tick(); + delete this.kv[key]; + } + async getAllKeys() { + await this.tick(); + return Object.keys(this.kv); + } +} + +function writerStore(delegate: AsyncPersistentStore): AsyncQueryStore { + return new AsyncQueryStore({ + isWriter: true, + delegate, + connect: handleMessage => ({ sendMessage: msg => handleMessage(msg) }), + }); +} + +/** Waits for the writer's read of the keys it holds. */ +async function scanned(store: AsyncQueryStore): Promise { + for (let i = 0; i < 5000; i++) { + if (store.hasEntity!(0) !== undefined) return; + await sleep(2); + } + throw new Error('writer never read its keys'); +} + +async function drain(store: AsyncQueryStore): Promise { + for (let i = 0; i < 5000; i++) { + if (store.isSettled()) return; + await sleep(2); + } + throw new Error('writer queue did not drain'); +} + +function asyncDoc(delegate: AsyncDelegate, key: number): Record | undefined { + const value = delegate.kv[valueKeyFor(key)] as string | undefined; + return value ? (JSON.parse(value) as Record) : undefined; +} + +describe('AsyncQueryStore writer', () => { + const clients: QueryClient[] = []; + afterEach(() => { + for (const client of clients.splice(0)) client.destroy(); + }); + + class Badge extends Entity { + __typename = t.typename('Badge'); + id = t.id; + label = t.string; + } + class User extends Entity { + __typename = t.typename('User'); + id = t.id; + name = t.string; + karma = t.number; + badge = t.optional(t.entity(Badge)); + } + class GetUser extends RESTQuery { + params = { id: t.id }; + path = `/user/${this.params.id}`; + result = { user: t.entity(User) }; + } + + function makeClient(store: AsyncQueryStore, mockFetch: ReturnType) { + const client = new QueryClient({ + store, + adapters: [new RESTQueryAdapter({ fetch: mockFetch as never, baseUrl: 'http://localhost' })], + log: { warn: () => {}, error: () => {} }, + } as never); + clients.push(client); + return client; + } + + it('answers hasEntity from the records it holds, seeded from the delegate', async () => { + const delegate = new AsyncDelegate(); + delegate.kv[valueKeyFor(42)] = JSON.stringify({ name: 'on disk' }); + const store = writerStore(delegate); + await scanned(store); + expect(store.hasEntity!(42)).toBe(true); + expect(store.hasEntity!(43)).toBe(false); + + store.saveEntity(43, { name: 'written' }); + await drain(store); + expect(store.hasEntity!(43)).toBe(true); + + // A query whose record references 43 is deleted: the cascade drops it. + store.saveQuery( + { statics: { id: 'def', cache: {} } } as never, + 7, + { user: { __entityRef: 43 } }, + Date.now(), + new Set([43]), + ); + await drain(store); + store.deleteQuery(7); + await drain(store); + expect(store.hasEntity!(43)).toBe(false); + expect(store.hasEntity!(42)).toBe(true); + }); + + it('refreshes the record of an entity on disk but not in memory by merging the event into it', async () => { + const delegate = new AsyncDelegate(); + const store = writerStore(delegate); + const mockFetch = createMockFetch(); + mockFetch.get('/user/u1', { + user: { + __typename: 'User', + id: 'u1', + name: 'Alice', + karma: 10, + badge: { __typename: 'Badge', id: 'b1', label: 'gold' }, + }, + }); + const client = makeClient(store, mockFetch); + await holdQuery(client, () => fetchQuery(GetUser, { id: 'u1' })); + await drain(store); + await scanned(store); + client.entityMap.getEntity(K('User', 'u1'))!.evict(); + + client.applyMutationEvent({ + type: 'update', + typename: 'User', + data: { __typename: 'User', id: 'u1', name: 'Alicia', karma: 10 }, + }); + await drain(store); + + expect(asyncDoc(delegate, K('User', 'u1'))).toMatchObject({ name: 'Alicia', karma: 10 }); + expect(Array.from(delegate.kv[refIdsKeyFor(K('User', 'u1'))] as Uint32Array)).toEqual([K('Badge', 'b1')]); + expect(asyncDoc(delegate, K('Badge', 'b1'))).toMatchObject({ label: 'gold' }); + expect(client.entityMap.getEntity(K('User', 'u1'))).toBeUndefined(); + + client.applyMutationEvent({ + type: 'update', + typename: 'User', + data: { __typename: 'User', id: 'u404', name: 'Nobody', karma: 0 }, + }); + await drain(store); + expect(asyncDoc(delegate, K('User', 'u404'))).toBeUndefined(); + }); + + it('refreshes a record it holds from an event that lands before it has read its keys', async () => { + const delegate = new AsyncDelegate(1); + const mockFetch = createMockFetch(); + mockFetch.get('/user/u1', { + user: { + __typename: 'User', + id: 'u1', + name: 'Alice', + karma: 10, + badge: { __typename: 'Badge', id: 'b1', label: 'gold' }, + }, + }); + mockFetch.get('/user/u2', { user: { __typename: 'User', id: 'u2', name: 'Bob', karma: 1 } }); + const first = writerStore(delegate); + const client1 = makeClient(first, mockFetch); + await holdQuery(client1, () => fetchQuery(GetUser, { id: 'u1' })); + await drain(first); + client1.destroy(); + + // A fresh writer has not read its keys yet when the event lands. + let releaseScan = () => {}; + const scanGate = new Promise(resolve => (releaseScan = resolve)); + const originalGetAllKeys = delegate.getAllKeys.bind(delegate); + delegate.getAllKeys = async () => { + await scanGate; + return originalGetAllKeys(); + }; + const store = writerStore(delegate); + const client = makeClient(store, mockFetch); + await holdQuery(client, () => fetchQuery(GetUser, { id: 'u2' })); + expect(store.hasEntity!(K('User', 'u1'))).toBeUndefined(); + expect(store.hasEntity!(K('User', 'u2'))).toBe(true); + const update = (data: Record) => + client.applyMutationEvent({ type: 'update', typename: 'User', data: { __typename: 'User', karma: 10, ...data } }); + // Not written: the badge it creates would be, whether or not a record of u1 exists. + update({ id: 'u1', name: 'Al', badge: { __typename: 'Badge', id: 'b2', label: 'new' } }); + update({ id: 'u1', name: 'Alicia' }); + update({ id: 'u404', name: 'Nobody' }); + expect(client.entityMap.getEntity(K('User', 'u1'))).toBeUndefined(); + releaseScan(); + await scanned(store); + await drain(store); + + expect(asyncDoc(delegate, K('User', 'u1'))).toMatchObject({ + name: 'Alicia', + karma: 10, + badge: { __entityRef: K('Badge', 'b1') }, + }); + expect(asyncDoc(delegate, K('Badge', 'b1'))).toMatchObject({ label: 'gold' }); + expect(asyncDoc(delegate, K('User', 'u404'))).toBeUndefined(); + expect(asyncDoc(delegate, K('Badge', 'b2'))).toBeUndefined(); + expect(store.hasEntity!(K('User', 'u404'))).toBe(false); + }); + + it('a reader sends an event for an entity on disk but not in memory to the writer as a merge', async () => { + const delegate = new AsyncDelegate(); + let toWriter: (msg: StoreMessage) => void = () => {}; + const writer = new AsyncQueryStore({ + isWriter: true, + delegate, + connect: handle => ((toWriter = handle), { sendMessage: () => {} }), + }); + const reader = new AsyncQueryStore({ isWriter: false, connect: () => ({ sendMessage: msg => toWriter(msg) }) }); + const mockFetch = createMockFetch(); + mockFetch.get('/user/u1', { + user: { + __typename: 'User', + id: 'u1', + name: 'Alice', + karma: 10, + badge: { __typename: 'Badge', id: 'b1', label: 'gold' }, + }, + }); + const client = makeClient(reader, mockFetch); + await holdQuery(client, () => fetchQuery(GetUser, { id: 'u1' })); + await drain(writer); + client.entityMap.getEntity(K('User', 'u1'))!.evict(); + + const update = (id: string, name: string) => + client.applyMutationEvent({ type: 'update', typename: 'User', data: { __typename: 'User', id, name, karma: 3 } }); + update('u1', 'Alicia'); + update('u404', 'Nobody'); + await drain(writer); + + expect(asyncDoc(delegate, K('User', 'u1'))).toMatchObject({ + name: 'Alicia', + karma: 3, + badge: { __entityRef: K('Badge', 'b1') }, + }); + expect(asyncDoc(delegate, K('User', 'u404'))).toBeUndefined(); + }); + + it("a reader's write of an entity created under one in memory is written whole, so the reference resolves", async () => { + const delegate = new AsyncDelegate(); + let toWriter: (msg: StoreMessage) => void = () => {}; + const writer = new AsyncQueryStore({ + isWriter: true, + delegate, + connect: handle => ((toWriter = handle), { sendMessage: () => {} }), + }); + const reader = new AsyncQueryStore({ isWriter: false, connect: () => ({ sendMessage: msg => toWriter(msg) }) }); + const mockFetch = createMockFetch(); + mockFetch.get('/user/u1', { user: { __typename: 'User', id: 'u1', name: 'Alice', karma: 10 } }); + const client = makeClient(reader, mockFetch); + await holdQuery(client, () => fetchQuery(GetUser, { id: 'u1' })); + await drain(writer); + + client.applyMutationEvent({ + type: 'update', + typename: 'User', + data: { + __typename: 'User', + id: 'u1', + name: 'Alice', + karma: 10, + badge: { __typename: 'Badge', id: 'b9', label: 'x' }, + }, + }); + await drain(writer); + + expect(asyncDoc(delegate, K('User', 'u1'))).toMatchObject({ badge: { __entityRef: K('Badge', 'b9') } }); + expect(asyncDoc(delegate, K('Badge', 'b9'))).toEqual({ __typename: 'Badge', id: 'b9', label: 'x' }); + }); + + it('ignores what it does not understand on the channel and keeps processing', async () => { + const delegate = new AsyncDelegate(); + const store = writerStore(delegate); + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {}); + try { + const handle = (store as unknown as { handleMessage(msg: unknown): void }).handleMessage.bind(store); + handle(null); + handle(undefined); + handle('hello'); + handle({ type: -1 }); + handle({ type: 9, entityKey: 1 }); + handle({ type: 1.5 }); + store.saveEntity(7, { name: 'still written' }); + await drain(store); + expect(asyncDoc(delegate, 7)).toMatchObject({ name: 'still written' }); + expect(store.isSettled()).toBe(true); + expect(consoleError).toHaveBeenCalled(); + } finally { + consoleError.mockRestore(); + } + }); + + it('writes the fields as the record when the stored record cannot be merged into', async () => { + const delegate = new AsyncDelegate(); + delegate.kv[valueKeyFor(5)] = '{not json'; + delegate.kv[valueKeyFor(6)] = JSON.stringify(['an', 'array']); + const store = writerStore(delegate); + store.mergeEntity(5, { name: 'healed' }); + store.mergeEntity(6, { name: 'healed too' }); + await drain(store); + expect(asyncDoc(delegate, 5)).toEqual({ name: 'healed' }); + expect(asyncDoc(delegate, 6)).toEqual({ name: 'healed too' }); + + const kv = new MemoryPersistentStore(); + kv.setString(valueKeyFor(5), '{not json'); + new SyncQueryStore(kv).mergeEntity(5, { name: 'healed' }); + expect(getDoc(kv, 5)).toEqual({ name: 'healed' }); + }); + + it('counts a queued write as held, so an event for it is refreshed before the write lands', async () => { + const delegate = new AsyncDelegate(5); + const store = writerStore(delegate); + await scanned(store); + const mockFetch = createMockFetch(); + mockFetch.get('/user/u1', { user: { __typename: 'User', id: 'u1', name: 'Alice', karma: 10 } }); + const client = makeClient(store, mockFetch); + await holdQuery(client, () => fetchQuery(GetUser, { id: 'u1' })); + // The write is queued, not processed; the entity is collected meanwhile. + expect(store.isSettled()).toBe(false); + expect(store.hasEntity!(K('User', 'u1'))).toBe(true); + client.entityMap.getEntity(K('User', 'u1'))!.evict(); + client.applyMutationEvent({ + type: 'update', + typename: 'User', + data: { __typename: 'User', id: 'u1', name: 'Alicia', karma: 10 }, + }); + await drain(store); + expect(asyncDoc(delegate, K('User', 'u1'))).toMatchObject({ name: 'Alicia' }); + }); + + it('a reader with its own delegate still purges stale queries directly', async () => { + const delegate = new AsyncDelegate(); + delegate.kv['sq:doc:lastUsed:old-def'] = 1; + delegate.kv['sq:doc:cacheTime:old-def'] = 1; + delegate.kv[queueKeyFor('old-def')] = new Uint32Array([9, 0]); + delegate.kv[valueKeyFor(9)] = JSON.stringify({ stale: true }); + delegate.kv[updatedAtKeyFor(9)] = 1; + const reader = new AsyncQueryStore({ + isWriter: false, + delegate, + connect: () => ({ sendMessage: () => {} }), + }); + await reader.purgeStaleQueries(); + expect(delegate.kv[valueKeyFor(9)]).toBeUndefined(); + expect(delegate.kv[queueKeyFor('old-def')]).toBeUndefined(); + }); + + it('a store without mergeEntity gets the whole in-memory data of an event-built entity', async () => { + const kv = new MemoryPersistentStore(); + const inner = new SyncQueryStore(kv); + // A custom store with only the pre-0.6 surface. + const custom = { + loadQuery: inner.loadQuery.bind(inner), + saveQuery: inner.saveQuery.bind(inner), + saveEntity: (key: number, value: unknown, refIds?: Set) => inner.saveEntity(key, value, refIds), + activateQuery: inner.activateQuery.bind(inner), + deleteQuery: inner.deleteQuery.bind(inner), + hasEntity: inner.hasEntity.bind(inner), + }; + const mockFetch = createMockFetch(); + mockFetch.get('/user/u1', { + user: { + __typename: 'User', + id: 'u1', + name: 'Alice', + karma: 10, + badge: { __typename: 'Badge', id: 'b1', label: 'gold' }, + }, + }); + const client = new QueryClient({ + store: custom, + adapters: [new RESTQueryAdapter({ fetch: mockFetch as never, baseUrl: 'http://localhost' })], + log: { warn: () => {}, error: () => {} }, + } as never); + clients.push(client); + await holdQuery(client, () => fetchQuery(GetUser, { id: 'u1' })); + client.entityMap.getEntity(K('User', 'u1'))!.evict(); + client.applyMutationEvent({ + type: 'update', + typename: 'User', + data: { __typename: 'User', id: 'u1', name: 'Alicia', karma: 10 }, + }); + await sleep(5); + // Written whole, never as a record of only the event's fields. + expect(getDoc(kv, K('User', 'u1'))).toMatchObject({ __typename: 'User', id: 'u1', name: 'Alicia', karma: 10 }); + expect(getDoc(kv, K('User', 'u1'))!.badge).toBeUndefined(); + }); + + it('runs purgeStaleQueries through its queue, after the writes queued before it', async () => { + const delegate = new AsyncDelegate(); + // A definition last used long ago, with one query whose record references entity 5. + delegate.kv['sq:doc:lastUsed:old-def'] = 1; + delegate.kv['sq:doc:cacheTime:old-def'] = 1; + delegate.kv[queueKeyFor('old-def')] = new Uint32Array([9, 0, 0]); + delegate.kv[valueKeyFor(9)] = JSON.stringify({ user: { __entityRef: 5 } }); + delegate.kv[updatedAtKeyFor(9)] = 1; + delegate.kv[refIdsKeyFor(9)] = new Uint32Array([5]); + delegate.kv[valueKeyFor(5)] = JSON.stringify({ name: 'old' }); + delegate.kv[refCountKeyFor(5)] = 1; + const store = writerStore(delegate); + const log: string[] = []; + store.onPersisted!(key => log.push(`ack ${key}`)); + store.onDelete!(key => log.push(`delete ${key}`)); + + store.saveEntity(5, { name: 'new' }); + const purged = store.purgeStaleQueries(); + expect(store.isSettled()).toBe(false); + await purged; + + // Acknowledged before the purge dropped it, so the record ends up marked missing. + expect(log).toEqual(['ack 5', 'delete 9', 'delete 5']); + expect(delegate.kv[valueKeyFor(5)]).toBeUndefined(); + expect(store.hasEntity!(5)).toBe(false); + }); + + it('shrinking the LRU while activating a key among the dropped slots keeps that key', async () => { + const delegate = new AsyncDelegate(); + const defId = 'shrink-def'; + const def = (maxCount: number) => ({ statics: { id: defId, cache: { maxCount } } }) as never; + delegate.kv[queueKeyFor(defId)] = new Uint32Array([4, 3, 2, 1]); + for (const key of [4, 3, 2, 1]) { + delegate.kv[valueKeyFor(key)] = JSON.stringify({ __entityRef: key }); + delegate.kv[updatedAtKeyFor(key)] = 1; + } + const store = writerStore(delegate); + store.activateQuery(def(3), 1); + await drain(store); + + expect(Array.from(delegate.kv[queueKeyFor(defId)] as Uint32Array)).toEqual([1, 4, 3]); + expect(delegate.kv[valueKeyFor(1)]).toBeDefined(); + expect(delegate.kv[valueKeyFor(2)]).toBeUndefined(); + + // The same for SyncQueryStore. + const kv = new MemoryPersistentStore(); + kv.setBuffer(queueKeyFor(defId), new Uint32Array([4, 3, 2, 1])); + for (const key of [4, 3, 2, 1]) { + kv.setString(valueKeyFor(key), JSON.stringify({ __entityRef: key })); + kv.setNumber(updatedAtKeyFor(key), 1); + } + new SyncQueryStore(kv).activateQuery(def(3), 1); + expect(Array.from(kv.getBuffer(queueKeyFor(defId))!)).toEqual([1, 4, 3]); + expect(kv.getString(valueKeyFor(1))).toBeDefined(); + expect(kv.getString(valueKeyFor(2))).toBeUndefined(); + }); + + it('a message without maxCount keeps the persisted queue at its size', async () => { + const delegate = new AsyncDelegate(); + delegate.kv[queueKeyFor('legacy-def')] = new Uint32Array([4, 3, 2, 1]); + for (const key of [4, 3, 2, 1]) { + delegate.kv[valueKeyFor(key)] = JSON.stringify({ __entityRef: key }); + delegate.kv[updatedAtKeyFor(key)] = 1; + } + delegate.kv[valueKeyFor(7)] = JSON.stringify({ __entityRef: 7 }); + const store = writerStore(delegate); + (store as unknown as { handleMessage(msg: unknown): void }).handleMessage({ + type: 2, + queryDefId: 'legacy-def', + queryKey: 7, + cacheTime: 1440, + }); + await drain(store); + + const queue = delegate.kv[queueKeyFor('legacy-def')] as Uint32Array; + expect(Array.from(queue)).toEqual([7, 4, 3, 2]); + expect(delegate.kv[valueKeyFor(1)]).toBeUndefined(); + expect(delegate.kv[valueKeyFor(2)]).toBeDefined(); + }); + + it('an acknowledgement only counts against a write the instance dispatched itself', async () => { + const delegate = new AsyncDelegate(); + const store = writerStore(delegate); + const mockFetch = createMockFetch(); + mockFetch.get('/user/u1', { user: { __typename: 'User', id: 'u1', name: 'Alice', karma: 10 } }); + const client = makeClient(store, mockFetch); + await holdQuery(client, () => fetchQuery(GetUser, { id: 'u1' })); + const first = client.entityMap.getEntity(K('User', 'u1'))!; + expect(first._pendingWrites).toBe(1); + expect(first._persisted).toBe(false); + await drain(store); + expect(first._pendingWrites).toBe(0); + expect(first._persisted).toBe(true); + + // An instance that never wrote (hydrated from the store) ignores a stray acknowledgement. + first.recordDropped(); + first.acknowledgeWrite(); + expect(first._persisted).toBe(false); + + first.save(); + first.save(); + expect(first._pendingWrites).toBe(2); + await drain(store); + expect(first._pendingWrites).toBe(0); + expect(first._persisted).toBe(true); + }); + + it('does not write a parent referencing a created child before the child is written', async () => { + class Item extends Entity { + __typename = t.typename('Item'); + id = t.id; + listId = t.string; + title = t.string; + } + class List extends Entity { + __typename = t.typename('List'); + id = t.id; + items = t.liveArray(Item, { constraints: { listId: (this as any).id } }); + } + class GetList extends RESTQuery { + path = '/list'; + result = { list: t.entity(List) }; + } + + class GatedDelegate extends AsyncDelegate { + gatedKey: string | undefined; + gate: Promise | undefined; + override async setString(key: string, value: string) { + if (key === this.gatedKey) await this.gate; + return super.setString(key, value); + } + } + + const delegate = new GatedDelegate(); + const store = writerStore(delegate); + const mockFetch = createMockFetch(); + mockFetch.get('/list', { list: { __typename: 'List', id: 'l1', items: [] } }); + const client = makeClient(store, mockFetch); + await holdQuery(client, () => fetchQuery(GetList)); + await drain(store); + + const listKey = K('List', 'l1'); + const itemKey = K('Item', 'i1'); + let release = () => {}; + delegate.gate = new Promise(resolve => (release = resolve)); + delegate.gatedKey = valueKeyFor(itemKey); + + client.applyMutationEvent({ + type: 'create', + typename: 'Item', + data: { __typename: 'Item', id: 'i1', listId: 'l1', title: 'new' }, + }); + await sleep(50); + + // The child's write is held, so a cache read now must not find the parent pointing at it. + expect(asyncDoc(delegate, itemKey)).toBeUndefined(); + expect(asyncDoc(delegate, listKey)).toMatchObject({ items: [] }); + expect(Array.from((delegate.kv[refIdsKeyFor(listKey)] as Uint32Array | undefined) ?? [])).toEqual([]); + + release(); + await drain(store); + expect(asyncDoc(delegate, itemKey)).toMatchObject({ title: 'new' }); + expect(asyncDoc(delegate, listKey)).toMatchObject({ items: [{ __entityRef: itemKey }] }); + }); +}); + +describe('store work per streamed event', () => { + class Author extends Entity { + __typename = t.typename('Author'); + id = t.id; + name = t.string; + } + class Note extends Entity { + __typename = t.typename('Note'); + id = t.id; + boardId = t.string; + body = t.string; + pinned = t.optional(t.boolean); + author = t.optional(t.entity(Author)); + } + class Board extends Entity { + __typename = t.typename('Board'); + id = t.id; + notes = t.liveArray(Note, { constraints: { boardId: (this as any).id } }); + } + class GetBoard extends RESTQuery { + params = { id: t.id }; + path = `/board/${this.params.id}`; + result = { board: t.entity(Board) }; + } + + const getClient = setupTestClient(); + + it('an entity an event built, of which the store held no record, is written whole afterwards', async () => { + const { client, mockFetch, kv, store } = getClient(); + mockFetch.get('/board/b1', { board: { __typename: 'Board', id: 'b1', notes: [] } }); + const boardQ = holdQuery(client, () => fetchQuery(GetBoard, { id: 'b1' })); + await boardQ; + const merges = vi.spyOn(store, 'mergeEntity'); + const reads = vi.spyOn(kv, 'getString'); + + client.applyMutationEvent({ + type: 'create', + typename: 'Note', + data: { __typename: 'Note', id: 'n1', boardId: 'b1', body: 'first' }, + }); + for (let i = 0; i < 3; i++) { + client.applyMutationEvent({ type: 'update', typename: 'Note', data: { id: 'n1', body: `edit ${i}` } }); + } + await sleep(5); + + expect(merges).not.toHaveBeenCalled(); + expect(reads.mock.calls.filter(([key]) => key === valueKeyFor(K('Note', 'n1')))).toEqual([]); + expect(getDoc(kv, K('Note', 'n1'))).toEqual({ __typename: 'Note', id: 'n1', boardId: 'b1', body: 'edit 2' }); + expect((boardQ.value as any).board.notes.map((n: any) => n.body)).toEqual(['edit 2']); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + }); + + it('an entity an event built over a record the store holds keeps merging into it', async () => { + const { client, mockFetch, kv, store } = getClient(); + mockFetch.get('/board/b1', { + board: { + __typename: 'Board', + id: 'b1', + notes: [{ __typename: 'Note', id: 'n1', boardId: 'b1', body: 'a', pinned: true }], + }, + }); + const boardQ = holdQuery(client, () => fetchQuery(GetBoard, { id: 'b1' })); + await boardQ; + // The note leaves memory but its record stays (the board's cache holds it). + client.applyMutationEvent({ type: 'delete', typename: 'Note', data: 'n1' }); + await sleep(5); + kv.setString( + valueKeyFor(K('Note', 'n1')), + JSON.stringify({ __typename: 'Note', id: 'n1', boardId: 'b1', body: 'a', pinned: true }), + ); + const merges = vi.spyOn(store, 'mergeEntity'); + + client.applyMutationEvent({ + type: 'create', + typename: 'Note', + data: { __typename: 'Note', id: 'n1', boardId: 'b1', body: 'b' }, + }); + await sleep(5); + + expect(merges.mock.calls.map(c => c[0])).toEqual([K('Note', 'n1')]); + expect(getDoc(kv, K('Note', 'n1'))).toMatchObject({ body: 'b', pinned: true }); + }); +}); + +describe('AsyncQueryStore writer: write skipping while writes are queued', () => { + const clients: QueryClient[] = []; + afterEach(() => { + for (const client of clients.splice(0)) client.destroy(); + }); + + class Badge extends Entity { + // Lingers once released, so a later apply finds the same instance. + static cache = { gcTime: 60 }; + __typename = t.typename('Badge'); + id = t.id; + label = t.string; + } + class User extends Entity { + __typename = t.typename('User'); + id = t.id; + name = t.string; + karma = t.number; + badge = t.optional(t.entity(Badge)); + } + class GetUser extends RESTQuery { + params = { id: t.id }; + path = `/user/${this.params.id}`; + result = { user: t.entity(User) }; + } + + async function setup(users: Record[]) { + const delegate = new AsyncDelegate(3); + const store = writerStore(delegate); + const mockFetch = createMockFetch(); + for (const user of users) mockFetch.get(`/user/${user.id as string}`, { user }); + const client = new QueryClient({ + store, + adapters: [new RESTQueryAdapter({ fetch: mockFetch as never, baseUrl: 'http://localhost' })], + log: { warn: () => {}, error: () => {} }, + } as never); + clients.push(client); + for (const user of users) await holdQuery(client, () => fetchQuery(GetUser, { id: user.id as string })); + await drain(store); + await scanned(store); + const saves = vi.spyOn(store, 'saveEntity'); + const merges = vi.spyOn(store, 'mergeEntity'); + const writes = () => [...saves.mock.calls.map(c => c[0]), ...merges.mock.calls.map(c => c[0])]; + return { delegate, store, client, writes }; + } + + const user = (id: string, karma: number, badge?: Record) => ({ + __typename: 'User', + id, + name: id, + karma, + ...(badge !== undefined ? { badge } : {}), + }); + + it("one entity's queued write leaves skipping on for identical events of other entities", async () => { + const { store, client, writes } = await setup([user('u1', 1), user('u2', 2), user('u3', 3)]); + + client.applyMutationEvent({ type: 'update', typename: 'User', data: { id: 'u1', karma: 10 } }); + client.applyMutationEvent({ type: 'update', typename: 'User', data: { id: 'u2', karma: 2 } }); + client.applyMutationEvent({ type: 'update', typename: 'User', data: { id: 'u3', karma: 3 } }); + // Identical to the write still queued for u1. + client.applyMutationEvent({ type: 'update', typename: 'User', data: { id: 'u1', karma: 10 } }); + expect(store.isSettled()).toBe(false); + await drain(store); + + expect(writes()).toEqual([K('User', 'u1')]); + }); + + it('a queued deletion still turns skipping off', async () => { + const { store, client, writes } = await setup([user('u1', 1)]); + + store.deleteQuery(12345); + client.applyMutationEvent({ type: 'update', typename: 'User', data: { id: 'u1', karma: 1 } }); + await drain(store); + + expect(writes()).toEqual([K('User', 'u1')]); + }); + + it('an entity a queued write stops referencing is written again when it is referenced again', async () => { + class Card extends Entity { + static cache = { gcTime: 60 }; + __typename = t.typename('Card'); + id = t.id; + deckId = t.string; + title = t.string; + } + class Deck extends Entity { + __typename = t.typename('Deck'); + id = t.id; + cards = t.liveArray(Card, { constraints: { deckId: (this as any).id } }); + } + class GetDeck extends RESTQuery { + path = '/deck'; + result = { deck: t.entity(Deck) }; + } + const card = { __typename: 'Card', id: 'c1', deckId: 'd1', title: 'Ace' }; + const delegate = new AsyncDelegate(3); + const store = writerStore(delegate); + const mockFetch = createMockFetch(); + mockFetch.get('/deck', { deck: { __typename: 'Deck', id: 'd1', cards: [card] } }); + const client = new QueryClient({ + store, + adapters: [new RESTQueryAdapter({ fetch: mockFetch as never, baseUrl: 'http://localhost' })], + log: { warn: () => {}, error: () => {} }, + } as never); + clients.push(client); + const deckQ = holdQuery(client, () => fetchQuery(GetDeck)); + await deckQ; + await drain(store); + await scanned(store); + + // Recreated with identical data before the deck's write, which drops c1, lands. + client.applyMutationEvent({ type: 'delete', typename: 'Card', data: 'c1' }); + client.applyMutationEvent({ type: 'create', typename: 'Card', data: card }); + await drain(store); + + expect((deckQ.value as any).deck.cards.map((c: any) => c.title)).toEqual(['Ace']); + expect(Array.from(delegate.kv[refIdsKeyFor(K('Deck', 'd1'))] as Uint32Array)).toEqual([K('Card', 'c1')]); + expect(asyncDoc(delegate, K('Card', 'c1'))).toMatchObject({ title: 'Ace' }); + }); +}); + +describe('an event for a typename whose other class is not registered this session', () => { + class Tag extends Entity { + __typename = t.typename('Tag'); + id = t.id; + label = t.string; + } + // A list-shaped event carries every CoinRow field, but not the detail's. + class CoinRow extends Entity { + __typename = t.typename('Coin'); + id = t.id; + symbol = t.string; + price = t.number; + } + class CoinDetail extends Entity { + __typename = t.typename('Coin'); + id = t.id; + symbol = t.string; + price = t.number; + about = t.string; + tag = t.entity(Tag); + } + class Market extends Entity { + __typename = t.typename('Market'); + id = t.id; + top = t.entity(CoinRow); + } + class GetCoinDetail extends RESTQuery { + path = '/coin'; + result = { coin: t.entity(CoinDetail) }; + } + class GetCoinRows extends RESTQuery { + path = '/coins'; + result = { coins: t.array(t.entity(CoinRow)) }; + } + class GetMarket extends RESTQuery { + path = '/market'; + result = { market: t.entity(Market) }; + } + + const clients: QueryClient[] = []; + afterEach(() => { + for (const client of clients.splice(0)) client.destroy(); + }); + + const detailCoin = { + __typename: 'Coin', + id: 'c1', + symbol: 'C', + price: 1, + about: 'cached', + tag: { __typename: 'Tag', id: 'g1', label: 'gold' }, + }; + const row = (id: string, price: number) => ({ __typename: 'Coin', id, symbol: 'C', price }); + const detailRecord = (price: number) => ({ + __typename: 'Coin', + id: 'c1', + symbol: 'C', + price, + about: 'cached', + tag: { __entityRef: K('Tag', 'g1') }, + }); + + function session(kv: MemoryPersistentStore, mockFetch: ReturnType): QueryClient { + const client = new QueryClient({ + store: new SyncQueryStore(kv), + adapters: [new RESTQueryAdapter({ fetch: mockFetch as never, baseUrl: 'http://localhost' })], + } as never); + clients.push(client); + return client; + } + + /** A first session caches the detail of c1, then ends. */ + async function cacheDetail(kv: MemoryPersistentStore): Promise> { + const mockFetch = createMockFetch(); + mockFetch.get('/coin', { coin: detailCoin }); + const first = session(kv, mockFetch); + await holdQuery(first, () => fetchQuery(GetCoinDetail)); + first.destroy(); + return mockFetch; + } + + /** The next session's detail query, served from the cache before its slow fetch lands. */ + async function detailAtNextStart( + kv: MemoryPersistentStore, + mockFetch: ReturnType, + ): Promise { + mockFetch.get('/coin', { coin: { ...detailCoin, about: 'network' } }, { delay: 200 }); + const next = session(kv, mockFetch); + const detailQ = holdQuery(next, () => fetchQuery(GetCoinDetail)); + await sleep(20); + return detailQ.isReady ? (detailQ.value as any).coin.about : 'not ready'; + } + + it('an update for an entity not in memory merges into its record instead of replacing it', async () => { + const kv = new MemoryPersistentStore(); + const mockFetch = await cacheDetail(kv); + + // Only the list's class is registered (a list of other coins); c1 is not in memory. + mockFetch.get('/coins', { coins: [row('c2', 2)] }); + const client = session(kv, mockFetch); + await holdQuery(client, () => fetchQuery(GetCoinRows)); + client.applyMutationEvent({ type: 'update', typename: 'Coin', data: row('c1', 3) }); + + expect(getDoc(kv, K('Coin', 'c1'))).toEqual(detailRecord(3)); + expect(refIds(kv, K('Coin', 'c1'))).toEqual([K('Tag', 'g1')]); + client.destroy(); + expect(await detailAtNextStart(kv, mockFetch)).toBe('cached'); + }); + + it('an entity an event builds under a root in memory merges into its record too', async () => { + const kv = new MemoryPersistentStore(); + const mockFetch = await cacheDetail(kv); + + mockFetch.get('/market', { market: { __typename: 'Market', id: 'm1', top: row('c2', 2) } }); + const client = session(kv, mockFetch); + await holdQuery(client, () => fetchQuery(GetMarket)); + client.applyMutationEvent({ + type: 'update', + typename: 'Market', + data: { __typename: 'Market', id: 'm1', top: row('c1', 3) }, + }); + client.applyMutationEvent({ type: 'update', typename: 'Coin', data: row('c1', 4) }); + + expect(getDoc(kv, K('Coin', 'c1'))).toEqual(detailRecord(4)); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + client.destroy(); + expect(await detailAtNextStart(kv, mockFetch)).toBe('cached'); + }); +}); diff --git a/packages/fetchium/src/__tests__/parsing/entity.test.ts b/packages/fetchium/src/__tests__/parsing/entity.test.ts index 11fb3e8..0a0cea0 100644 --- a/packages/fetchium/src/__tests__/parsing/entity.test.ts +++ b/packages/fetchium/src/__tests__/parsing/entity.test.ts @@ -39,11 +39,11 @@ describe('Entity parsing', () => { }, }; - await parseEntities(result, QueryResult, client, new Map()); + const parsed = (await parseEntities(result, QueryResult, client, new Map())) as { data: unknown }; - expect(result.data).toBeInstanceOf(Entity); - expect(result.data).toBeInstanceOf(User); - expect(Object.getPrototypeOf(result.data)).toBe(User.prototype); + expect(parsed.data).toBeInstanceOf(Entity); + expect(parsed.data).toBeInstanceOf(User); + expect(Object.getPrototypeOf(parsed.data)).toBe(User.prototype); }); describe('nested entities', () => { diff --git a/packages/fetchium/src/__tests__/same-value.test.ts b/packages/fetchium/src/__tests__/same-value.test.ts index d7a326f..d475788 100644 --- a/packages/fetchium/src/__tests__/same-value.test.ts +++ b/packages/fetchium/src/__tests__/same-value.test.ts @@ -38,8 +38,9 @@ describe('sameValue', () => { expect(sameValue(proxy, proxy)).toBe(true); }); - it('reports NaN as changed, since it is not equal to itself', () => { - expect(sameValue(Number.NaN, Number.NaN)).toBe(false); + it('treats NaN as unchanged and -0 as a change (Object.is semantics)', () => { + expect(sameValue(Number.NaN, Number.NaN)).toBe(true); + expect(sameValue(0, -0)).toBe(false); }); const different: [string, unknown, unknown][] = [ diff --git a/packages/fetchium/src/__tests__/skip-write-and-snapshot-fixes.test.ts b/packages/fetchium/src/__tests__/skip-write-and-snapshot-fixes.test.ts new file mode 100644 index 0000000..ce9c86f --- /dev/null +++ b/packages/fetchium/src/__tests__/skip-write-and-snapshot-fixes.test.ts @@ -0,0 +1,1446 @@ +import { describe, it, expect, vi, afterEach } from 'vitest'; +import { watcher, withContexts } from 'signalium'; +import { hashValue, snapshot } from 'signalium/utils'; +import { t, registerFormat } from '../typeDefs.js'; +import { Mask } from '../types.js'; +import { Entity } from '../proxy.js'; +import { RESTQuery } from '../rest/index.js'; +import { RESTQueryAdapter } from '../rest/RESTQueryAdapter.js'; +import { fetchQuery } from '../query.js'; +import { QueryClient, QueryClientContext } from '../QueryClient.js'; +import { SyncQueryStore, MemoryPersistentStore } from '../stores/sync.js'; +import { AsyncQueryStore, type AsyncPersistentStore } from '../stores/async.js'; +import { valueKeyFor, refIdsKeyFor, queueKeyFor, DEFAULT_MAX_COUNT } from '../stores/shared.js'; +import { createMockFetch, setupTestClient, testWithClient, sleep } from './utils.js'; + +// Invariants of the snapshot fast path, write skipping, events and store deletion reporting. + +function getDoc(kv: MemoryPersistentStore, key: number): Record | undefined { + const value = kv.getString(valueKeyFor(key)); + return value ? (JSON.parse(value) as Record) : undefined; +} + +/** Holds a query active and exposes `useReactive`-style snapshots of it. */ +function snapshotHarness(client: QueryClient, start: () => T) { + return withContexts([[QueryClientContext, client]], () => { + const query = start(); + let prev: unknown; + let computes = 0; + const snapshots = watcher(() => { + computes++; + return (prev = snapshot(query, prev)); + }); + snapshots.addListener(() => {}); + return { + query, + read: () => (snapshots.value as { value: any }).value, + computes: () => computes, + }; + }); +} + +function holdQuery(client: QueryClient, start: () => T): T { + return withContexts([[QueryClientContext, client]], () => { + const query = start(); + const w = watcher(() => (query as unknown as { value: unknown }).value); + w.addListener(() => {}); + return query; + }); +} + +function makeClient(store: SyncQueryStore | AsyncQueryStore, mockFetch: ReturnType) { + const warn = vi.fn(); + const client = new QueryClient({ + store, + adapters: [new RESTQueryAdapter({ fetch: mockFetch as never, baseUrl: 'http://localhost' })], + log: { warn, error: () => {} }, + } as never); + return { client, warn }; +} + +describe('hydration of a cached query whose entity is already in memory', () => { + const getClient = setupTestClient(); + class User extends Entity { + __typename = t.typename('User'); + id = t.id; + name = t.string; + profile = t.object({ city: t.string, since: t.optional(t.format('date')) }); + } + class GetA extends RESTQuery { + path = '/a'; + result = { user: t.entity(User) }; + config = { staleTime: 60_000 }; + } + class GetB extends RESTQuery { + path = '/b'; + result = { user: t.entity(User), extra: t.string }; + config = { staleTime: 60_000 }; + } + const payload = { __typename: 'User', id: 1, name: 'Alice', profile: { city: 'Rome', since: '2020-01-01' } }; + + it('keeps nested formatted values, notifies nothing, and leaves every consumer consistent', async () => { + const kv = new MemoryPersistentStore(); + const store = new SyncQueryStore(kv); + const mockFetch = createMockFetch(); + mockFetch.get('/a', { user: payload }); + mockFetch.get('/b', { user: payload, extra: 'e' }); + + const { client: first } = makeClient(store, mockFetch); + await holdQuery(first, () => fetchQuery(GetA)); + await holdQuery(first, () => fetchQuery(GetB)); + first.destroy(); + + // Session 2, offline: A hydrates, then B hydrates the same entity. + mockFetch.get('/a', { user: payload }, { delay: 10_000 }); + mockFetch.get('/b', { user: payload, extra: 'e' }, { delay: 10_000 }); + const { client, warn } = makeClient(store, mockFetch); + const error = vi.spyOn(console, 'error').mockImplementation(() => {}); + try { + const hA = snapshotHarness(client, () => fetchQuery(GetA)); + await sleep(20); + const a1 = hA.read(); + expect(a1.user.profile.since).toBeInstanceOf(Date); + const computesBefore = hA.computes(); + + const hB = snapshotHarness(client, () => fetchQuery(GetB)); + await sleep(20); + + // Not re-parsed, so A's consumer had no reason to recompute. + const instance = client.entityMap.getEntity(hashValue(['User', 1]))!; + expect((instance.data.profile as { since: unknown }).since).toBeDefined(); + expect((hA.query.value as any).user.profile.since).toBeInstanceOf(Date); + expect(hB.read().user.profile.since).toBeInstanceOf(Date); + expect(hA.read().user.profile.since).toBe(a1.user.profile.since); + expect(hA.computes()).toBe(computesBefore); + expect(warn).not.toHaveBeenCalled(); + expect(error).not.toHaveBeenCalled(); + } finally { + error.mockRestore(); + client.destroy(); + } + }); + + it('rejects the cached query when the in-memory entity lacks a field the query requires', async () => { + class Narrow extends Entity { + __typename = t.typename('Doc'); + id = t.id; + title = t.string; + } + class Wide extends Entity { + __typename = t.typename('Doc'); + id = t.id; + title = t.string; + body = t.string; + } + class GetNarrow extends RESTQuery { + path = '/narrow'; + result = { doc: t.entity(Narrow) }; + } + class GetWide extends RESTQuery { + path = '/wide'; + result = { doc: t.entity(Wide) }; + config = { staleTime: 60_000 }; + } + const kv = new MemoryPersistentStore(); + const store = new SyncQueryStore(kv); + const mockFetch = createMockFetch(); + mockFetch.get('/wide', { doc: { __typename: 'Doc', id: 1, title: 'T', body: 'B' } }); + mockFetch.get('/narrow', { doc: { __typename: 'Doc', id: 1, title: 'T' } }); + + const { client: first } = makeClient(store, mockFetch); + await holdQuery(first, () => fetchQuery(GetWide)); + first.destroy(); + + // The narrow query fetches fresh, so Doc:1 is in memory without `body`. + const { client, warn } = makeClient(store, mockFetch); + await holdQuery(client, () => fetchQuery(GetNarrow)); + mockFetch.get('/wide', { doc: { __typename: 'Doc', id: 1, title: 'T', body: 'B2' } }); + const wide = holdQuery(client, () => fetchQuery(GetWide)); + await wide; + expect(warn).toHaveBeenCalledWith(expect.stringContaining('query cache may be corrupted'), expect.anything()); + expect((wide.value as any).doc.body).toBe('B2'); + client.destroy(); + }); + + it('rejects the cached query when the in-memory entity lacks a nested entity the query requires', async () => { + class Org extends Entity { + __typename = t.typename('Org'); + id = t.id; + name = t.string; + } + class UserSummary extends Entity { + __typename = t.typename('User'); + id = t.id; + meta = t.object({ x: t.number }); + } + class UserFull extends Entity { + __typename = t.typename('User'); + id = t.id; + meta = t.object({ x: t.number, owner: t.entity(Org) }); + } + class GetSummary extends RESTQuery { + path = '/summary'; + result = { user: t.entity(UserSummary) }; + config = { staleTime: 60_000 }; + } + class GetFull extends RESTQuery { + path = '/full'; + result = { user: t.entity(UserFull) }; + config = { staleTime: 60_000 }; + } + const full = { __typename: 'User', id: 1, meta: { x: 1, owner: { __typename: 'Org', id: 'o1', name: 'Acme' } } }; + const kv = new MemoryPersistentStore(); + const store = new SyncQueryStore(kv); + const mockFetch = createMockFetch(); + mockFetch.get('/summary', { user: { __typename: 'User', id: 1, meta: { x: 1 } } }); + mockFetch.get('/full', { user: full }); + + const { client: first } = makeClient(store, mockFetch); + await holdQuery(first, () => fetchQuery(GetSummary)); + await holdQuery(first, () => fetchQuery(GetFull)); + first.destroy(); + + // Session 2: the summary hydrates User:1 without `meta.owner`. + mockFetch.get('/full', { user: { ...full, meta: { ...full.meta, x: 2 } } }, { delay: 50 }); + const { client, warn } = makeClient(store, mockFetch); + const summary = holdQuery(client, () => fetchQuery(GetSummary)); + await sleep(10); + expect((summary.value as any).user.meta.x).toBe(1); + + const wide = holdQuery(client, () => fetchQuery(GetFull)); + await sleep(10); + expect(wide.value).toBeUndefined(); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('query cache may be corrupted'), expect.anything()); + await wide; + expect((wide.value as any).user.meta.owner.name).toBe('Acme'); + expect((wide.value as any).user.meta.x).toBe(2); + client.destroy(); + }); + + it('a cached list holding an in-memory entity the query cannot use is refetched whole, not served with a hole', async () => { + class Narrow extends Entity { + __typename = t.typename('Doc'); + id = t.id; + title = t.string; + } + class Wide extends Entity { + __typename = t.typename('Doc'); + id = t.id; + title = t.string; + body = t.string; + } + class GetNarrow extends RESTQuery { + params = { id: t.id }; + path = `/narrow/${this.params.id}`; + result = { doc: t.entity(Narrow) }; + } + class GetDocs extends RESTQuery { + path = '/docs'; + result = { docs: t.array(t.entity(Wide)) }; + config = { staleTime: 60_000 }; + } + const kv = new MemoryPersistentStore(); + const store = new SyncQueryStore(kv); + const mockFetch = createMockFetch(); + const docs = [ + { __typename: 'Doc', id: 1, title: 'T1', body: 'B1' }, + { __typename: 'Doc', id: 2, title: 'T2', body: 'B2' }, + ]; + mockFetch.get('/docs', { docs }); + mockFetch.get('/narrow/1', { doc: { __typename: 'Doc', id: 1, title: 'T1' } }); + + const { client: first } = makeClient(store, mockFetch); + await holdQuery(first, () => fetchQuery(GetDocs)); + first.destroy(); + + // Doc:1 comes into memory through the narrow query, without `body`. + const { client, warn } = makeClient(store, mockFetch); + await holdQuery(client, () => fetchQuery(GetNarrow, { id: 1 })); + mockFetch.get('/docs', { docs: docs.map(d => ({ ...d, body: `${d.body}!` })) }); + const list = holdQuery(client, () => fetchQuery(GetDocs)); + await list; + expect(warn).toHaveBeenCalledWith(expect.stringContaining('query cache may be corrupted'), expect.anything()); + expect((list.value as any).docs.map((d: any) => d.body)).toEqual(['B1!', 'B2!']); + client.destroy(); + }); + + it('a member the query narrows out anyway does not reject the cached query', async () => { + class UserPreview extends Entity { + __typename = t.typename('User'); + id = t.id; + name = t.string; + } + class UserFull extends Entity { + __typename = t.typename('User'); + id = t.id; + name = t.string; + email = t.string; + } + class Team extends Entity { + __typename = t.typename('Team'); + id = t.id; + owner = t.entity(UserPreview); + members = t.array(t.entity(UserFull)); + } + class GetTeam extends RESTQuery { + path = '/team'; + result = { team: t.entity(Team) }; + config = { staleTime: 60_000 }; + } + class GetOwner extends RESTQuery { + path = '/owner'; + result = { user: t.entity(UserPreview) }; + } + const kv = new MemoryPersistentStore(); + const store = new SyncQueryStore(kv); + const mockFetch = createMockFetch(); + mockFetch.get('/team', { + team: { + __typename: 'Team', + id: 't-1', + owner: { __typename: 'User', id: 'u-1', name: 'Alice' }, + members: [ + { __typename: 'User', id: 'u-1', name: 'Alice' }, + { __typename: 'User', id: 'u-2', name: 'Bob', email: 'b@x' }, + ], + }, + }); + mockFetch.get('/owner', { user: { __typename: 'User', id: 'u-1', name: 'Alice' } }); + const { client: first } = makeClient(store, mockFetch); + await holdQuery(first, () => fetchQuery(GetTeam)); + first.destroy(); + + // u-1 lacks `email`, but the members array narrows it out on read. + const { client, warn } = makeClient(store, mockFetch); + await holdQuery(client, () => fetchQuery(GetOwner)); + mockFetch.get( + '/team', + { team: { __typename: 'Team', id: 't-1', owner: { __typename: 'User', id: 'u-1', name: 'FRESH' }, members: [] } }, + { delay: 5000 }, + ); + const team = holdQuery(client, () => fetchQuery(GetTeam)); + await sleep(20); + expect(warn).not.toHaveBeenCalled(); + expect((team.value as any).team.members.map((m: any) => m.id)).toEqual(['u-2']); + expect((team.value as any).team.owner.name).toBe('Alice'); + client.destroy(); + }); + + it('an entity built from events is completed from its record when a cached query hydrates it', async () => { + class Post extends Entity { + __typename = t.typename('Post'); + id = t.id; + feedId = t.string; + title = t.string; + body = t.optional(t.string); + tags = t.optional(t.array(t.string)); + } + class GetPost extends RESTQuery { + params = { id: t.id }; + path = `/post/${this.params.id}`; + result = { post: t.entity(Post) }; + config = { staleTime: 60_000 }; + } + class Feed extends Entity { + __typename = t.typename('Feed'); + id = t.id; + posts = t.liveArray(Post, { constraints: { feedId: (this as any).id } }); + } + class GetFeed extends RESTQuery { + params = { id: t.id }; + path = `/feed/${this.params.id}`; + result = { feed: t.entity(Feed) }; + } + const kv = new MemoryPersistentStore(); + const store = new SyncQueryStore(kv); + const mockFetch = createMockFetch(); + mockFetch.get('/post/p1', { + post: { __typename: 'Post', id: 'p1', feedId: 'f1', title: 'T', body: 'hello', tags: ['a', 'b'] }, + }); + mockFetch.get('/feed/f1', { feed: { __typename: 'Feed', id: 'f1', posts: [] } }); + const { client: first } = makeClient(store, mockFetch); + await holdQuery(first, () => fetchQuery(GetPost, { id: 'p1' })); + first.destroy(); + + // An event brings the post into memory with its title only. + const { client, warn } = makeClient(store, mockFetch); + const feed = holdQuery(client, () => fetchQuery(GetFeed, { id: 'f1' })); + await feed; + client.applyMutationEvent({ + type: 'create', + typename: 'Post', + data: { __typename: 'Post', id: 'p1', feedId: 'f1', title: 'T2' }, + }); + await sleep(5); + expect((feed.value as any).feed.posts.map((p: any) => p.title)).toEqual(['T2']); + const instance = client.entityMap.getEntity(hashValue(['Post', 'p1']))!; + expect(instance._partial).toBe(true); + expect(instance.data.body).toBeUndefined(); + + mockFetch.get( + '/post/p1', + { post: { __typename: 'Post', id: 'p1', title: 'FRESH', body: 'fresh' } }, + { delay: 5000 }, + ); + const post = holdQuery(client, () => fetchQuery(GetPost, { id: 'p1' })); + await sleep(20); + expect(warn).not.toHaveBeenCalled(); + expect((post.value as any).post.title).toBe('T2'); + expect((post.value as any).post.body).toBe('hello'); + expect((post.value as any).post.tags).toEqual(['a', 'b']); + expect((feed.value as any).feed.posts[0].body).toBe('hello'); + expect(instance._partial).toBe(false); + client.destroy(); + }); + + it('a frozen payload with undeclared nested values is applied without being written to', async () => { + const { client, mockFetch } = getClient(); + class Doc extends Entity { + __typename = t.typename('Doc'); + id = t.id; + title = t.string; + meta = t.object({ views: t.number }); + } + class GetDoc extends RESTQuery { + path = '/doc'; + result = { doc: t.entity(Doc) }; + } + mockFetch.get('/doc', { doc: { __typename: 'Doc', id: 'd', title: 'T', meta: { views: 1, deep: { x: [1] } } } }); + const h = snapshotHarness(client, () => fetchQuery(GetDoc)); + await h.query; + const snap = h.read().doc; + const warn = vi.spyOn(client.getContext().log as { warn: (...args: unknown[]) => void }, 'warn'); + + const frozen = Object.freeze({ + id: 'd', + title: 'T2', + meta: Object.freeze({ views: 2, deep: Object.freeze({ x: Object.freeze([2]) }) }), + }); + client.applyMutationEvent({ type: 'update', typename: 'Doc', data: frozen }); + expect(warn).not.toHaveBeenCalled(); + expect(h.read().doc.title).toBe('T2'); + expect(h.read().doc.meta.views).toBe(2); + expect(snap.meta.views).toBe(1); + }); + + it('a frozen fetch result is applied without being written to, and later merges reach its nested values', async () => { + class DocNote extends Entity { + __typename = t.typename('DocNote'); + id = t.id; + docId = t.string; + } + class Doc extends Entity { + __typename = t.typename('Doc'); + id = t.id; + title = t.string; + meta = t.object({ views: t.number, stats: t.object({ likes: t.number }) }); + counts = t.record(t.number); + tags = t.array(t.string); + notes = t.liveArray(DocNote, { constraints: { docId: (this as any).id } }); + } + class GetDoc extends RESTQuery { + path = '/doc'; + result = { doc: t.entity(Doc), note: t.object({ text: t.string }) }; + } + const deepFreeze = (value: T): T => { + if (typeof value === 'object' && value !== null) { + for (const v of Object.values(value)) deepFreeze(v); + Object.freeze(value); + } + return value; + }; + const body = deepFreeze({ + doc: { + __typename: 'Doc', + id: 'd', + title: 'T', + meta: { views: 1, stats: { likes: 3 } }, + counts: { a: 1 }, + tags: ['x'], + notes: [], + }, + note: { text: 'n' }, + }); + const before = JSON.stringify(body); + // The same frozen object on every call, as an adapter may hand out. + const fetch = async () => ({ ok: true, status: 200, headers: new Headers(), json: async () => body }); + const { client, warn } = makeClient( + new SyncQueryStore(new MemoryPersistentStore()), + fetch as unknown as ReturnType, + ); + const h = snapshotHarness(client, () => fetchQuery(GetDoc)); + await h.query; + expect(h.read().doc.meta.stats.likes).toBe(3); + expect(h.read().note.text).toBe('n'); + + client.applyMutationEvent({ + type: 'update', + typename: 'Doc', + data: { id: 'd', meta: { views: 2, stats: { likes: 4 } }, counts: { a: 2 } }, + }); + expect(warn).not.toHaveBeenCalled(); + expect(h.read().doc.meta.views).toBe(2); + expect(h.read().doc.meta.stats.likes).toBe(4); + expect(h.read().doc.counts.a).toBe(2); + // The live array grows its own array, not the frozen response's. + client.applyMutationEvent({ + type: 'create', + typename: 'DocNote', + data: { __typename: 'DocNote', id: 'n1', docId: 'd' }, + }); + expect(h.read().doc.notes.map((n: { id: string }) => n.id)).toEqual(['n1']); + expect(h.read().doc.tags).toEqual(['x']); + + // A refetch of the same (frozen) response puts the old values back. + await (h.query as any).value.__refetch(); + await sleep(5); + expect(h.read().doc.meta.stats.likes).toBe(3); + expect(h.read().doc.counts.a).toBe(1); + expect(warn).not.toHaveBeenCalled(); + expect(JSON.stringify(body)).toBe(before); + client.destroy(); + }); +}); + +describe('mutation events and the store', () => { + const getClient = setupTestClient(); + + class User extends Entity { + __typename = t.typename('User'); + id = t.id; + name = t.string; + karma = t.number; + badge = t.optional(t.entity(Badge)); + } + class Badge extends Entity { + __typename = t.typename('Badge'); + id = t.id; + label = t.string; + } + class Comment extends Entity { + __typename = t.typename('Comment'); + id = t.id; + postId = t.string; + body = t.string; + author = t.entity(User); + } + class Post extends Entity { + __typename = t.typename('Post'); + id = t.id; + title = t.string; + comments = t.liveArray(Comment, { constraints: { postId: (this as any).id } }); + } + class GetPost extends RESTQuery { + params = { id: t.id }; + path = `/post/${this.params.id}`; + result = { post: t.entity(Post) }; + } + class GetUser extends RESTQuery { + params = { id: t.id }; + path = `/user/${this.params.id}`; + result = { user: t.entity(User) }; + } + const u1Key = hashValue(['User', 'u1']); + const badgeKey = hashValue(['Badge', 'b1']); + + async function seed(client: QueryClient, mockFetch: ReturnType, withPost: boolean) { + mockFetch.get('/user/u1', { user: { __typename: 'User', id: 'u1', name: 'Alice', karma: 10 } }); + mockFetch.get('/post/p1', { post: { __typename: 'Post', id: 'p1', title: 'T', comments: [] } }); + const userQ = holdQuery(client, () => fetchQuery(GetUser, { id: 'u1' })); + await userQ; + const postQ = withPost ? holdQuery(client, () => fetchQuery(GetPost, { id: 'p1' })) : undefined; + if (postQ) await postQ; + return { userQ, postQ }; + } + + const createEvent = (withBadge: boolean) => ({ + type: 'create' as const, + typename: 'Comment', + data: { + __typename: 'Comment', + id: 'c9', + postId: 'p1', + body: 'hi', + author: { + __typename: 'User', + id: 'u1', + name: 'Alice', + karma: 11, + ...(withBadge ? { badge: { __typename: 'Badge', id: 'b1', label: 'gold' } } : {}), + }, + }, + }); + + it('a create routed into a live array writes the existing author it changed, and the new badge under it', async () => { + const { client, mockFetch, kv } = getClient(); + await seed(client, mockFetch, true); + expect(getDoc(kv, u1Key)!.karma).toBe(10); + + client.applyMutationEvent(createEvent(true)); + await sleep(5); + + expect(getDoc(kv, u1Key)!.karma).toBe(11); + expect(getDoc(kv, badgeKey)).toMatchObject({ label: 'gold' }); + expect(Array.from(kv.getBuffer(refIdsKeyFor(u1Key)) ?? [])).toEqual([badgeKey]); + expect(getDoc(kv, hashValue(['Comment', 'c9']))).toMatchObject({ body: 'hi' }); + }); + + it('a create nothing routes still writes the existing author it changed, and writes no orphan', async () => { + const { client, mockFetch, kv } = getClient(); + await seed(client, mockFetch, true); + + // The post's live array is constrained to postId p1; this comment is for another post. + const event = createEvent(false); + client.applyMutationEvent({ ...event, data: { ...event.data, postId: 'p2' } }); + await sleep(5); + + expect(getDoc(kv, u1Key)!.karma).toBe(11); + expect(getDoc(kv, hashValue(['Comment', 'c9']))).toBeUndefined(); + expect(client.entityMap.getEntity(hashValue(['Comment', 'c9']))).toBeUndefined(); + }); + + it('a full-payload update for an entity on disk but not in memory refreshes its record', async () => { + const { client, mockFetch, kv, store } = getClient(); + await seed(client, mockFetch, false); + // Its record stays referenced by the cached query. + client.entityMap.getEntity(u1Key)!.evict(); + expect(client.entityMap.getEntity(u1Key)).toBeUndefined(); + expect(getDoc(kv, u1Key)!.name).toBe('Alice'); + + client.applyMutationEvent({ + type: 'update', + typename: 'User', + data: { __typename: 'User', id: 'u1', name: 'Alicia', karma: 12 }, + }); + await sleep(5); + + expect(client.entityMap.getEntity(u1Key)).toBeUndefined(); + expect(getDoc(kv, u1Key)).toMatchObject({ name: 'Alicia', karma: 12 }); + expect(store.hasEntity(u1Key)).toBe(true); + + client.applyMutationEvent({ + type: 'update', + typename: 'User', + data: { __typename: 'User', id: 'u404', name: 'Nobody', karma: 0 }, + }); + await sleep(5); + expect(getDoc(kv, hashValue(['User', 'u404']))).toBeUndefined(); + }); +}); + +describe('write skipping stays consistent with the store', () => { + class User extends Entity { + __typename = t.typename('User'); + id = t.id; + name = t.string; + } + class GetUser extends RESTQuery { + path = '/user'; + result = { user: t.entity(User) }; + } + const userKey = hashValue(['User', 1]); + + it('a store write that throws is retried by the next unchanged apply', async () => { + const kv = new MemoryPersistentStore(); + const store = new SyncQueryStore(kv); + const mockFetch = createMockFetch(); + mockFetch.get('/user', { user: { __typename: 'User', id: 1, name: 'Alice' } }); + const { client } = makeClient(store, mockFetch); + + const relay = holdQuery(client, () => fetchQuery(GetUser)); + await relay; + expect(getDoc(kv, userKey)).toMatchObject({ name: 'Alice' }); + + // The next refetch changes the name, and the store rejects that write. + const setString = kv.setString.bind(kv); + let failing = true; + vi.spyOn(kv, 'setString').mockImplementation((key, value) => { + if (failing && key === valueKeyFor(userKey)) throw new Error('quota exceeded'); + setString(key, value); + }); + mockFetch.get('/user', { user: { __typename: 'User', id: 1, name: 'Alicia' } }); + await expect((relay.value as any).__refetch()).rejects.toThrow('quota exceeded'); + expect((relay.value as any)?.user?.name ?? client.entityMap.getEntity(userKey)!.data.name).toBe('Alicia'); + expect(getDoc(kv, userKey)).toMatchObject({ name: 'Alice' }); + + // The refetch matches memory, yet the record must be rewritten. + failing = false; + await withContexts([[QueryClientContext, client]], () => fetchQuery(GetUser).value ?? undefined); + await (client as any).queryInstances.values().next().value.refetch(); + await sleep(20); + expect(getDoc(kv, userKey)).toMatchObject({ name: 'Alicia' }); + client.destroy(); + }); + + it('QueryClient.destroy() removes its store listeners', () => { + const store = new SyncQueryStore(new MemoryPersistentStore()); + const listeners = () => (store as unknown as { deleteListeners: unknown[] }).deleteListeners.length; + const clients = Array.from({ length: 5 }, () => new QueryClient({ store })); + expect(listeners()).toBe(5); + for (const client of clients) client.destroy(); + expect(listeners()).toBe(0); + }); +}); + +// AsyncQueryStore writer + +class AsyncDelegate implements AsyncPersistentStore { + readonly kv: Record = Object.create(null); + failNextSetString: string | undefined; + failHas = 0; + private async tick(): Promise { + await new Promise(resolve => setTimeout(resolve, 0)); + } + async has(key: string) { + await this.tick(); + if (this.failHas > 0) { + this.failHas--; + throw new Error('injected failure'); + } + return key in this.kv; + } + async getString(key: string) { + await this.tick(); + return this.kv[key] as string | undefined; + } + async setString(key: string, value: string) { + await this.tick(); + if (this.failNextSetString === key) { + this.failNextSetString = undefined; + throw new Error('injected failure'); + } + this.kv[key] = value; + } + async getNumber(key: string) { + await this.tick(); + return this.kv[key] as number | undefined; + } + async setNumber(key: string, value: number) { + await this.tick(); + this.kv[key] = value; + } + async getBuffer(key: string) { + await this.tick(); + return this.kv[key] as Uint32Array | undefined; + } + async setBuffer(key: string, value: Uint32Array) { + await this.tick(); + this.kv[key] = value; + } + async delete(key: string) { + await this.tick(); + delete this.kv[key]; + } + async getAllKeys() { + await this.tick(); + return Object.keys(this.kv); + } +} + +function writerStore(delegate: AsyncPersistentStore): AsyncQueryStore { + return new AsyncQueryStore({ + isWriter: true, + delegate, + connect: handleMessage => ({ sendMessage: msg => handleMessage(msg) }), + }); +} + +async function drain(store: AsyncQueryStore): Promise { + for (let i = 0; i < 5000; i++) { + if (store.isSettled()) return; + await sleep(2); + } + throw new Error('writer queue did not drain'); +} + +describe('AsyncQueryStore writer used in-process', () => { + const clients: QueryClient[] = []; + afterEach(() => { + for (const client of clients.splice(0)) client.destroy(); + }); + + class User extends Entity { + __typename = t.typename('User'); + id = t.id; + name = t.string; + } + class GetProfile extends RESTQuery { + static cache = { maxCount: 2 }; + params = { id: t.id }; + path = `/user/profile/${this.params.id}`; + result = { user: t.entity(User) }; + } + const userKey = (id: string | number) => hashValue(['User', id]); + + it('an unchanged refetch racing a queued eviction still leaves the query hydratable', async () => { + const delegate = new AsyncDelegate(); + const store = writerStore(delegate); + const mockFetch = createMockFetch(); + for (const id of ['1', '2', '3']) { + mockFetch.get(`/user/profile/${id}`, { user: { __typename: 'User', id, name: `User ${id}` } }); + } + const { client, warn } = makeClient(store, mockFetch); + clients.push(client); + + const q1 = holdQuery(client, () => fetchQuery(GetProfile, { id: '1' })); + await q1; + await holdQuery(client, () => fetchQuery(GetProfile, { id: '2' })); + await drain(store); + expect(delegate.kv[valueKeyFor(userKey('1'))]).toBeDefined(); + + // Q3's queued SaveQuery will evict Q1 and cascade to User 1. Q1 refetches first. + await holdQuery(client, () => fetchQuery(GetProfile, { id: '3' })); + expect(store.isSettled()).toBe(false); + await (q1.value as any).__refetch(); + await drain(store); + + expect(delegate.kv[valueKeyFor(userKey('1'))]).toBeDefined(); + expect(warn).not.toHaveBeenCalled(); + + client.destroy(); + clients.pop(); + mockFetch.reset(); + mockFetch.get('/user/profile/1', { user: { __typename: 'User', id: '1', name: 'Refetched' } }, { delay: 2000 }); + const cold = makeClient(store, mockFetch); + clients.push(cold.client); + const relay = holdQuery(cold.client, () => fetchQuery(GetProfile, { id: '1' })); + await sleep(150); + expect(cold.warn).not.toHaveBeenCalled(); + expect((relay.value as any)?.user?.name).toBe('User 1'); + }); + + it('a SaveEntity the delegate rejected is written again by the next unchanged apply', async () => { + const delegate = new AsyncDelegate(); + const store = writerStore(delegate); + const mockFetch = createMockFetch(); + mockFetch.get('/user/profile/1', { user: { __typename: 'User', id: '1', name: 'Alice' } }); + const { client } = makeClient(store, mockFetch); + clients.push(client); + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {}); + try { + delegate.failNextSetString = valueKeyFor(userKey('1')); + const q1 = holdQuery(client, () => fetchQuery(GetProfile, { id: '1' })); + await q1; + await drain(store); + expect(delegate.kv[valueKeyFor(userKey('1'))]).toBeUndefined(); + + await (q1.value as any).__refetch(); + await drain(store); + expect(delegate.kv[valueKeyFor(userKey('1'))]).toBeDefined(); + } finally { + consoleError.mockRestore(); + } + }); + + it('a rejected SaveEntity over an existing record still treats the record as held', async () => { + const delegate = new AsyncDelegate(); + const store = writerStore(delegate); + const mockFetch = createMockFetch(); + mockFetch.get('/user/profile/1', { user: { __typename: 'User', id: '1', name: 'Alice' } }); + const { client } = makeClient(store, mockFetch); + clients.push(client); + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {}); + const name = () => JSON.parse(delegate.kv[valueKeyFor(userKey('1'))] as string).name; + const update = (newName: string) => + client.applyMutationEvent({ + type: 'update', + typename: 'User', + data: { __typename: 'User', id: '1', name: newName }, + }); + try { + await holdQuery(client, () => fetchQuery(GetProfile, { id: '1' })); + await drain(store); + + delegate.failNextSetString = valueKeyFor(userKey('1')); + update('Bob'); + await drain(store); + expect(name()).toBe('Alice'); + expect(store.hasEntity!(userKey('1'))).toBe(true); + + // With the entity out of memory, the next update must still reach the record. + client.entityMap.getEntity(userKey('1'))!.evict(); + update('Carol'); + await drain(store); + expect(name()).toBe('Carol'); + } finally { + consoleError.mockRestore(); + } + }); + + it('a rejected SaveEntity whose recheck also fails keeps the record held', async () => { + class Person extends Entity { + __typename = t.typename('Person'); + id = t.id; + name = t.string; + role = t.optional(t.string); + } + class GetPerson extends RESTQuery { + path = '/person'; + result = { person: t.entity(Person) }; + } + const key = hashValue(['Person', '1']); + const delegate = new AsyncDelegate(); + const store = writerStore(delegate); + const mockFetch = createMockFetch(); + mockFetch.get('/person', { person: { __typename: 'Person', id: '1', name: 'Alice', role: 'admin' } }); + const { client } = makeClient(store, mockFetch); + clients.push(client); + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {}); + const update = (name: string) => + client.applyMutationEvent({ type: 'update', typename: 'Person', data: { __typename: 'Person', id: '1', name } }); + try { + await holdQuery(client, () => fetchQuery(GetPerson)); + await drain(store); + client.entityMap.getEntity(key)!.evict(); + + delegate.failNextSetString = valueKeyFor(key); + delegate.failHas = 3; + update('Bob'); + await drain(store); + expect(store.hasEntity!(key)).toBe(true); + + update('Carol'); + await drain(store); + expect(JSON.parse(delegate.kv[valueKeyFor(key)] as string)).toMatchObject({ name: 'Carol', role: 'admin' }); + } finally { + consoleError.mockRestore(); + } + }); + + it('an event refresh of a record the writer wrongly holds writes nothing, and the entity is rewritten whole', async () => { + class Task extends Entity { + __typename = t.typename('Task'); + id = t.id; + boardId = t.string; + title = t.string; + done = t.optional(t.boolean); + } + class Board extends Entity { + __typename = t.typename('Board'); + id = t.id; + edits = t.liveValue(t.number, Task, { + constraints: { boardId: (this as any).id }, + onCreate: (n: number) => n, + onUpdate: (n: number) => n + 1, + onDelete: (n: number) => n, + }); + } + class GetBoard extends RESTQuery { + path = '/board'; + result = { board: t.entity(Board) }; + } + class GetTask extends RESTQuery { + path = '/task'; + result = { task: t.entity(Task) }; + } + const key = hashValue(['Task', 't1']); + const delegate = new AsyncDelegate(); + const store = writerStore(delegate); + const deleted: number[] = []; + store.onDelete!(k => deleted.push(k)); + const mockFetch = createMockFetch(); + mockFetch.get('/board', { board: { __typename: 'Board', id: 'b1', edits: 0 } }); + mockFetch.get('/task', { task: { __typename: 'Task', id: 't1', boardId: 'b1', title: 'A', done: false } }); + const { client } = makeClient(store, mockFetch); + clients.push(client); + const update = (title: string) => + client.applyMutationEvent({ + type: 'update', + typename: 'Task', + data: { __typename: 'Task', id: 't1', boardId: 'b1', title }, + }); + + await holdQuery(client, () => fetchQuery(GetBoard)); + await holdQuery(client, () => fetchQuery(GetTask)); + await drain(store); + // The record is lost behind the writer's back, so it still counts it held. + delete delegate.kv[valueKeyFor(key)]; + client.entityMap.getEntity(key)!.evict(); + + update('B'); + await drain(store); + expect(delegate.kv[valueKeyFor(key)]).toBeUndefined(); + expect(deleted).toContain(key); + expect(store.hasEntity!(key)).toBe(false); + + update('C'); + await drain(store); + expect(JSON.parse(delegate.kv[valueKeyFor(key)] as string)).toEqual({ + __typename: 'Task', + id: 't1', + boardId: 'b1', + title: 'C', + }); + }); + + it('resizes a persisted LRU queue in both directions and defaults a missing maxCount', async () => { + const delegate = new AsyncDelegate(); + const store = writerStore(delegate); + const defId = 'upgrade-def'; + const def = (maxCount: number) => ({ statics: { id: defId, cache: { maxCount } } }) as never; + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {}); + try { + // A full 4-slot queue from an older install. + const old = new Uint32Array([4, 3, 2, 1]); + delegate.kv[queueKeyFor(defId)] = old; + for (const key of old) { + delegate.kv[valueKeyFor(key)] = JSON.stringify({ __entityRef: key }); + delegate.kv[`sq:doc:updatedAt:${key}`] = 1; + } + delegate.kv[valueKeyFor(9)] = JSON.stringify({ __entityRef: 9 }); + + // Grow: the buffer is reallocated, the order kept, nothing evicted. + store.activateQuery(def(6), 9); + await drain(store); + let queue = delegate.kv[queueKeyFor(defId)] as Uint32Array; + expect(Array.from(queue)).toEqual([9, 4, 3, 2, 1, 0]); + expect(consoleError).not.toHaveBeenCalled(); + + // Shrink after a restart: keys that no longer fit are evicted, not stranded. + const restarted = writerStore(delegate); + restarted.activateQuery(def(3), 9); + await drain(restarted); + queue = delegate.kv[queueKeyFor(defId)] as Uint32Array; + expect(Array.from(queue)).toEqual([9, 4, 3]); + expect(delegate.kv[valueKeyFor(2)]).toBeUndefined(); + expect(delegate.kv[valueKeyFor(1)]).toBeUndefined(); + expect(delegate.kv[valueKeyFor(4)]).toBeDefined(); + + // A message from a reader that predates `maxCount` uses the default size. + const fresh = new AsyncDelegate(); + const freshStore = writerStore(fresh); + fresh.kv[valueKeyFor(7)] = JSON.stringify({ __entityRef: 7 }); + (freshStore as unknown as { handleMessage(msg: unknown): void }).handleMessage({ + type: 2, + queryDefId: 'legacy-def', + queryKey: 7, + cacheTime: 1440, + }); + await drain(freshStore); + const legacyQueue = fresh.kv[queueKeyFor('legacy-def')] as Uint32Array; + expect(legacyQueue.length).toBe(DEFAULT_MAX_COUNT); + expect(legacyQueue[0]).toBe(7); + expect(consoleError).not.toHaveBeenCalled(); + } finally { + consoleError.mockRestore(); + } + }); +}); + +describe('snapshot fast path follow-ups', () => { + const getClient = setupTestClient(); + + it('a NaN in a static field neither reports drift nor churns snapshot identity', async () => { + const { client, mockFetch } = getClient(); + // JSON cannot carry NaN; a lenient string-to-number format can. + registerFormat( + 'loose-number', + Mask.STRING, + (value: string) => Number(value), + (value: number) => String(value), + ); + class Stat extends Entity { + __typename = t.typename('Stat'); + id = t.id; + label = t.string; + ratio = t.format('loose-number' as never); + series = t.array(t.format('loose-number' as never)); + } + class GetStats extends RESTQuery { + path = '/stats'; + result = { stats: t.array(t.entity(Stat)) }; + } + mockFetch.get('/stats', { + stats: [ + { __typename: 'Stat', id: 's', label: 'a', ratio: 'N/A', series: ['1', 'N/A'] }, + { __typename: 'Stat', id: 'other', label: 'b', ratio: '1', series: [] }, + ], + }); + const error = vi.spyOn(console, 'error').mockImplementation(() => {}); + try { + const h = snapshotHarness(client, () => fetchQuery(GetStats)); + await h.query; + const first = h.read().stats[0]; + client.applyMutationEvent({ type: 'update', typename: 'Stat', data: { id: 'other', label: 'c' } }); + const second = h.read().stats[0]; + expect(Number.isNaN(second.ratio)).toBe(true); + expect(second).toBe(first); + expect(error).not.toHaveBeenCalled(); + } finally { + error.mockRestore(); + } + }); + + it('two classes sharing a typename: a child entity under a field the other class declares entity-free still reaches the consumer', async () => { + const { client, mockFetch } = getClient(); + class Org extends Entity { + __typename = t.typename('Org'); + id = t.id; + name = t.string; + } + class UserSummary extends Entity { + __typename = t.typename('User'); + id = t.id; + name = t.string; + meta = t.object({ x: t.number }); + } + class UserFull extends Entity { + __typename = t.typename('User'); + id = t.id; + name = t.string; + meta = t.object({ x: t.number, owner: t.entity(Org) }); + } + class GetSummary extends RESTQuery { + path = '/summary'; + result = { user: t.entity(UserSummary) }; + } + class GetFull extends RESTQuery { + path = '/full'; + result = { user: t.entity(UserFull) }; + } + mockFetch.get('/summary', { user: { __typename: 'User', id: 1, name: 'Alice', meta: { x: 1 } } }); + mockFetch.get('/full', { + user: { + __typename: 'User', + id: 1, + name: 'Alice', + meta: { x: 1, owner: { __typename: 'Org', id: 7, name: 'ACME' } }, + }, + }); + const error = vi.spyOn(console, 'error').mockImplementation(() => {}); + try { + const summary = snapshotHarness(client, () => fetchQuery(GetSummary)); + await summary.query; + const full = snapshotHarness(client, () => fetchQuery(GetFull)); + await full.query; + expect(summary.read().user.meta.owner.name).toBe('ACME'); + + client.applyMutationEvent({ type: 'update', typename: 'Org', data: { id: 7, name: 'NEW' } }); + expect(summary.read().user.meta.owner.name).toBe('NEW'); + client.applyMutationEvent({ type: 'update', typename: 'Org', data: { id: 7, name: 'NEWER' } }); + expect(summary.read().user.meta.owner.name).toBe('NEWER'); + // Served by the fast path's own bookkeeping, not by the dev guard's re-read. + expect(error).not.toHaveBeenCalled(); + } finally { + error.mockRestore(); + } + }); + + it('a reactive consumer of a narrowed entity array learns when a member becomes eligible', async () => { + const { client, mockFetch } = getClient(); + class UserPreview extends Entity { + __typename = t.typename('User'); + id = t.id; + name = t.string; + } + class UserFull extends Entity { + __typename = t.typename('User'); + id = t.id; + name = t.string; + email = t.string; + } + class Team extends Entity { + __typename = t.typename('Team'); + id = t.id; + owner = t.entity(UserPreview); + members = t.array(t.entity(UserFull)); + } + class GetTeam extends RESTQuery { + path = '/team'; + result = { team: t.entity(Team) }; + } + class GetUser extends RESTQuery { + params = { id: t.id }; + path = `/user/${this.params.id}`; + result = { user: t.entity(UserFull) }; + } + mockFetch.get('/team', { + team: { + __typename: 'Team', + id: 't-1', + owner: { __typename: 'User', id: 'u-1', name: 'Alice' }, + members: [ + { __typename: 'User', id: 'u-1', name: 'Alice', email: 'a@x' }, + { __typename: 'User', id: 'u-2', name: 'Bob', email: 'b@x' }, + ], + }, + }); + mockFetch.get('/user/u-1', { user: { __typename: 'User', id: 'u-1', name: 'Alice', email: 'a@x' } }); + + await testWithClient(client, async () => { + const team = fetchQuery(GetTeam); + await team; + const ids = watcher(() => (team.value as any).team.members.map((m: any) => m.id).join(',')); + ids.addListener(() => {}); + // u-1 was parsed with the preview def first, so it is narrowed out. + expect(ids.value).toBe('u-2'); + + await fetchQuery(GetUser, { id: 'u-1' }); + await sleep(5); + expect(ids.value).toBe('u-1,u-2'); + }); + }); + + it('query extras attached to a shared entity reach the query itself without re-running its other consumers', async () => { + const { client, mockFetch } = getClient(); + class Profile extends Entity { + __typename = t.typename('Profile'); + id = t.id; + name = t.string; + } + class GetWrapped extends RESTQuery { + path = '/wrapped'; + result = { profile: t.entity(Profile) }; + } + class GetRoot extends RESTQuery { + path = '/root'; + result = t.entity(Profile); + } + const profile = { __typename: 'Profile', id: 1, name: 'Alice' }; + mockFetch.get('/wrapped', { profile }); + mockFetch.get('/root', profile); + mockFetch.get('/wrapped', { profile: { ...profile, name: 'Alicia' } }); + + const wrapped = snapshotHarness(client, () => fetchQuery(GetWrapped)); + await wrapped.query; + const before = wrapped.read(); + const computesBefore = wrapped.computes(); + expect('__refetch' in before.profile).toBe(false); + + // The wrapped query's consumer is not re-run for the root query's extras. + const root = snapshotHarness(client, () => fetchQuery(GetRoot)); + await root.query; + await sleep(5); + expect(typeof root.read().__refetch).toBe('function'); + expect(wrapped.read()).toBe(before); + expect(wrapped.computes()).toBe(computesBefore); + + // Its next recompute (here, a change to the entity) picks them up. + await (wrapped.query as any).value.__refetch(); + await sleep(5); + expect(wrapped.read().profile.name).toBe('Alicia'); + expect(typeof wrapped.read().profile.__refetch).toBe('function'); + }); + + it('a length-only consumer of a narrowed entity array does not recompute when a member it holds changes', async () => { + const { client, mockFetch } = getClient(); + class UserPreview extends Entity { + __typename = t.typename('User'); + id = t.id; + name = t.string; + } + class UserFull extends Entity { + __typename = t.typename('User'); + id = t.id; + name = t.string; + email = t.string; + } + class Team extends Entity { + __typename = t.typename('Team'); + id = t.id; + owner = t.entity(UserPreview); + members = t.array(t.entity(UserFull)); + } + class GetTeam extends RESTQuery { + path = '/team'; + result = { team: t.entity(Team) }; + } + mockFetch.get('/team', { + team: { + __typename: 'Team', + id: 't-1', + owner: { __typename: 'User', id: 'u-1', name: 'Alice' }, + members: [ + { __typename: 'User', id: 'u-1', name: 'Alice', email: 'a@x' }, + { __typename: 'User', id: 'u-2', name: 'Bob', email: 'b@x' }, + ], + }, + }); + + const team = holdQuery(client, () => fetchQuery(GetTeam)); + await team; + let computes = 0; + const count = withContexts([[QueryClientContext, client]], () => { + const w = watcher(() => { + computes++; + return (team.value as any).team.members.length; + }); + w.addListener(() => {}); + return w; + }); + await sleep(5); + expect(count.value).toBe(1); + const base = computes; + + // A member the array holds changes a field the consumer never read. + client.applyMutationEvent({ type: 'update', typename: 'User', data: { id: 'u-2', name: 'Robert' } }); + await sleep(5); + expect((team.value as any).team.members[0].name).toBe('Robert'); + expect(count.value).toBe(1); + expect(computes).toBe(base); + + // A member the array left out becomes eligible: that is a change of the array. + client.applyMutationEvent({ type: 'update', typename: 'User', data: { id: 'u-1', email: 'alice@x' } }); + await sleep(5); + expect(count.value).toBe(2); + expect(computes).toBe(base + 1); + }); + + it('applies -0 over 0 and keeps NaN, at every level', async () => { + const { client, mockFetch } = getClient(); + class Quote extends Entity { + __typename = t.typename('Quote'); + id = t.id; + price = t.number; + history = t.array(t.number); + } + class GetQuote extends RESTQuery { + path = '/quote'; + result = { quote: t.entity(Quote) }; + } + mockFetch.get('/quote', { quote: { __typename: 'Quote', id: 'q', price: 0, history: [0, 1] } }); + const h = snapshotHarness(client, () => fetchQuery(GetQuote)); + await h.query; + const before = h.computes(); + + client.applyMutationEvent({ type: 'update', typename: 'Quote', data: { id: 'q', price: -0, history: [-0, 1] } }); + expect(Object.is((h.query.value as any).quote.price, -0)).toBe(true); + expect(Object.is(h.read().quote.price, -0)).toBe(true); + expect(Object.is(h.read().quote.history[0], -0)).toBe(true); + expect(h.computes()).toBe(before + 1); + + client.applyMutationEvent({ type: 'update', typename: 'Quote', data: { id: 'q', price: -0, history: [-0, 1] } }); + expect(h.computes()).toBe(before + 1); + }); + + it('a snapshot handed back to the parser is read, not written to', async () => { + const { client, mockFetch } = getClient(); + class Doc extends Entity { + __typename = t.typename('Doc'); + id = t.id; + title = t.string; + meta = t.object({ views: t.number, tags: t.array(t.string) }); + extra = t.record(t.number); + } + class GetDoc extends RESTQuery { + path = '/doc'; + result = { doc: t.entity(Doc) }; + } + mockFetch.get('/doc', { + doc: { __typename: 'Doc', id: 'd', title: 'T', meta: { views: 1, tags: ['a'] }, extra: { k: 1 } }, + }); + const h = snapshotHarness(client, () => fetchQuery(GetDoc)); + await h.query; + const snap = h.read().doc; + expect(Object.isFrozen(snap.meta)).toBe(true); + const warn = vi.spyOn(client.getContext().log as { warn: (...args: unknown[]) => void }, 'warn'); + + // A snapshot forwarded as an event payload must not be written into. + client.applyMutationEvent({ + type: 'update', + typename: 'Doc', + data: { id: 'd', title: 'T2', meta: { ...snap.meta, views: 2 }, extra: snap.extra }, + }); + const meta = { views: 3, tags: snap.meta.tags }; + client.applyMutationEvent({ type: 'update', typename: 'Doc', data: { id: 'd', meta, extra: snap.extra } }); + + expect(warn).not.toHaveBeenCalled(); + expect(h.read().doc.title).toBe('T2'); + expect(h.read().doc.meta.views).toBe(3); + expect(h.read().doc.extra.k).toBe(1); + expect(snap.meta.views).toBe(1); + expect(Object.isFrozen(h.read().doc.meta)).toBe(true); + }); + + it('two events sharing one nested object do not alias entity data', async () => { + const { client, mockFetch } = getClient(); + class Token extends Entity { + __typename = t.typename('Token'); + id = t.id; + walletId = t.string; + metadata = t.object({ image: t.string }); + } + class Wallet extends Entity { + __typename = t.typename('Wallet'); + id = t.id; + tokens = t.liveArray(Token, { constraints: { walletId: (this as any).id } }); + } + class GetWallet extends RESTQuery { + path = '/wallet'; + result = { wallet: t.entity(Wallet) }; + } + mockFetch.get('/wallet', { wallet: { __typename: 'Wallet', id: 'w', tokens: [] } }); + const h = snapshotHarness(client, () => fetchQuery(GetWallet)); + await h.query; + await sleep(5); + const error = vi.spyOn(console, 'error').mockImplementation(() => {}); + try { + const shared = { image: 'shared.png' }; + client.applyMutationEvent({ + type: 'create', + typename: 'Token', + data: { __typename: 'Token', id: 'tok-0', walletId: 'w', metadata: shared }, + }); + client.applyMutationEvent({ + type: 'create', + typename: 'Token', + data: { __typename: 'Token', id: 'tok-1', walletId: 'w', metadata: shared }, + }); + await sleep(5); + const tokens = () => h.read().wallet.tokens; + expect(tokens()).toHaveLength(2); + const tok1 = tokens()[1]; + client.applyMutationEvent({ + type: 'update', + typename: 'Token', + data: { id: 'tok-0', metadata: { image: 'new.png' } }, + }); + await sleep(5); + + expect(tokens()[0].metadata.image).toBe('new.png'); + expect(tokens()[1].metadata.image).toBe('shared.png'); + expect(tokens()[1]).toBe(tok1); + expect((h.query.value as any).wallet.tokens[1].metadata.image).toBe('shared.png'); + expect(shared.image).toBe('shared.png'); + expect(error).not.toHaveBeenCalled(); + } finally { + error.mockRestore(); + } + }); + + it('a Set-valued format with many object members is verified in linear time', async () => { + const { client, mockFetch } = getClient(); + registerFormat( + 'id-set', + Mask.STRING, + (value: string) => new Set(value.split(',').map(id => ({ id: Number(id) }))), + (value: Set<{ id: number }>) => [...value].map(item => item.id).join(','), + ); + class Roster extends Entity { + __typename = t.typename('Roster'); + id = t.id; + label = t.string; + members = t.format('id-set' as never); + } + class GetRoster extends RESTQuery { + path = '/roster'; + result = { roster: t.entity(Roster), other: t.entity(Roster) }; + } + const ids = Array.from({ length: 3000 }, (_, i) => i).join(','); + mockFetch.get('/roster', { + roster: { __typename: 'Roster', id: 'r', label: 'a', members: ids }, + other: { __typename: 'Roster', id: 'o', label: 'b', members: '0' }, + }); + const h = snapshotHarness(client, () => fetchQuery(GetRoster)); + await h.query; + expect(h.read().roster.members.size).toBe(3000); + + // Each event makes the dev guard compare a rebuilt Set with the previous one. + const started = performance.now(); + for (let i = 0; i < 20; i++) { + client.applyMutationEvent({ type: 'update', typename: 'Roster', data: { id: 'o', label: `b${i}` } }); + expect(h.read().roster.members.size).toBe(3000); + } + expect(performance.now() - started).toBeLessThan(2000); + }); +}); diff --git a/packages/fetchium/src/__tests__/store-on-delete.test.ts b/packages/fetchium/src/__tests__/store-on-delete.test.ts index b8e44ec..a6c2d6c 100644 --- a/packages/fetchium/src/__tests__/store-on-delete.test.ts +++ b/packages/fetchium/src/__tests__/store-on-delete.test.ts @@ -112,6 +112,123 @@ describe('QueryStore.onDelete', () => { await sleep(20); expect(deleted).toEqual([q1]); }); + + it('SyncQueryStore still notifies the next listener when one unsubscribes during a deletion', () => { + const store = new SyncQueryStore(new MemoryPersistentStore()); + const deleted: number[] = []; + const unsubscribe = store.onDelete(() => unsubscribe()); + store.onDelete(key => deleted.push(key)); + + const q1 = hashValue(['GET:/u', { id: '1' }]); + store.saveQuery(queryDef('GET:/u', 10), q1, { __entityRef: 0 }, Date.now(), new Set()); + store.deleteQuery(q1); + expect(deleted).toEqual([q1]); + }); + + it('AsyncQueryStore writer still notifies the next listener when one unsubscribes during a deletion', async () => { + const writer = new AsyncQueryStore({ + isWriter: true, + delegate: new MockAsyncPersistentStore(), + connect: (_handle: (msg: StoreMessage) => void) => ({ sendMessage: () => {} }), + }); + const deleted: number[] = []; + const unsubscribe = writer.onDelete!(() => unsubscribe()); + writer.onDelete!(key => deleted.push(key)); + + const q1 = hashValue(['GET:/v', { id: '1' }]); + writer.saveQuery(queryDef('GET:/v', 10), q1, { __entityRef: 0 }, Date.now(), new Set()); + writer.deleteQuery(q1); + await sleep(20); + expect(deleted).toEqual([q1]); + }); + + it('SyncQueryStore finishes a cascade and notifies every listener when one throws', () => { + const kv = new MemoryPersistentStore(); + const store = new SyncQueryStore(kv); + const deleted: number[] = []; + store.onDelete(() => { + throw new Error('listener failed'); + }); + store.onDelete(key => deleted.push(key)); + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {}); + try { + const q1 = hashValue(['GET:/t', { id: '1' }]); + const e1 = hashValue(['User', 1]); + store.saveEntity(e1, { id: 1 }); + store.saveQuery(queryDef('GET:/t', 10), q1, { __entityRef: e1 }, Date.now(), new Set([e1])); + store.deleteQuery(q1); + expect(deleted).toEqual([q1, e1]); + expect(kv.has(valueKeyFor(e1))).toBe(false); + expect(consoleError).toHaveBeenCalled(); + } finally { + consoleError.mockRestore(); + } + }); + + it('AsyncQueryStore writer finishes a cascade when a listener throws', async () => { + const delegate = new MockAsyncPersistentStore(); + const writer = new AsyncQueryStore({ + isWriter: true, + delegate, + connect: (_handle: (msg: StoreMessage) => void) => ({ sendMessage: () => {} }), + }); + const deleted: number[] = []; + writer.onDelete!(() => { + throw new Error('listener failed'); + }); + writer.onDelete!(key => deleted.push(key)); + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {}); + try { + const q1 = hashValue(['GET:/w', { id: '1' }]); + const e1 = hashValue(['User', 1]); + writer.saveEntity(e1, { id: 1 }); + writer.saveQuery(queryDef('GET:/w', 10), q1, { __entityRef: e1 }, Date.now(), new Set([e1])); + writer.deleteQuery(q1); + await sleep(20); + expect(deleted).toEqual([q1, e1]); + expect(await delegate.has(valueKeyFor(e1))).toBe(false); + } finally { + consoleError.mockRestore(); + } + }); + + it('AsyncQueryStore writer keeps processing when a listener throws over a failed write', async () => { + class FlakyStore extends MockAsyncPersistentStore { + failNext: string | undefined; + override async setString(key: string, value: string) { + if (key === this.failNext) { + this.failNext = undefined; + throw new Error('injected failure'); + } + return super.setString(key, value); + } + } + const delegate = new FlakyStore(); + const writer = new AsyncQueryStore({ + isWriter: true, + delegate, + connect: (_handle: (msg: StoreMessage) => void) => ({ sendMessage: () => {} }), + }); + const deleted: number[] = []; + writer.onDelete!(() => { + throw new Error('listener failed'); + }); + writer.onDelete!(key => deleted.push(key)); + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {}); + try { + const e1 = hashValue(['User', 1]); + const e2 = hashValue(['User', 2]); + delegate.failNext = valueKeyFor(e1); + writer.saveEntity(e1, { id: 1 }); + writer.saveEntity(e2, { id: 2 }); + await sleep(20); + expect(deleted).toEqual([e1]); + expect(await delegate.has(valueKeyFor(e2))).toBe(true); + expect(writer.isSettled()).toBe(true); + } finally { + consoleError.mockRestore(); + } + }); }); describe('stores without onDelete', () => { diff --git a/packages/fetchium/src/applyEntities.ts b/packages/fetchium/src/applyEntities.ts index 6147d1a..5389a15 100644 --- a/packages/fetchium/src/applyEntities.ts +++ b/packages/fetchium/src/applyEntities.ts @@ -27,6 +27,13 @@ export interface ApplyResult { entityRefs: Map; } +/** + * `false` writes nothing (cache hydration). `'existing'` writes only entities + * already in memory. Created ones are written once a written record references + * them, or by the caller once it knows the root is retained. + */ +export type PersistMode = boolean | 'existing'; + /** * Single depth-first walk from the root that applies entities to the store, * replaces parsed data objects with entity proxies, and counts child refs. @@ -34,15 +41,24 @@ export interface ApplyResult { export function applyEntityRefs( ctx: ParseContext, rootData: unknown, - persist: boolean, + persist: PersistMode, appendMode: boolean = false, + created?: Set, ): ApplyResult { const queryClient = ctx.queryClient!; queryClient.currentParseId++; const seen = ctx.seen!; const entityRefs = new Map(); - const data = reifyAndApply(rootData, seen, queryClient, persist, entityRefs, appendMode); + let data: unknown; + try { + data = reifyAndApply(rootData, seen, queryClient, persist, entityRefs, appendMode, created); + } catch (e) { + queryClient.discardDeferredWrites(); + throw e; + } + + queryClient.flushDeferredWrites(); return { data, entityRefs }; } @@ -55,22 +71,25 @@ function reifyAndApply( value: unknown, seen: Map, ParsedEntity>, queryClient: QueryClient, - persist: boolean, + persist: PersistMode, entityRefs: Map, appendMode: boolean, + created: Set | undefined, ): unknown { if (typeof value !== 'object' || value === null) return value; const entity = seen.get(value as Record); if (entity !== undefined) { - return applyEntity(entity, seen, queryClient, persist, entityRefs, appendMode); + return applyEntity(entity, seen, queryClient, persist, entityRefs, appendMode, created); } + // Assign only on change: values the parser didn't copy may be frozen. if (Array.isArray(value)) { for (let i = 0; i < value.length; i++) { const item = value[i]; if (typeof item === 'object' && item !== null && !(item instanceof FormattedValue) && !PROXY_ID.has(item)) { - value[i] = reifyAndApply(item, seen, queryClient, persist, entityRefs, appendMode); + const reified = reifyAndApply(item, seen, queryClient, persist, entityRefs, appendMode, created); + if (reified !== item) value[i] = reified; } } return value; @@ -81,7 +100,8 @@ function reifyAndApply( for (const key of Object.keys(obj)) { const v = obj[key]; if (typeof v === 'object' && v !== null && !(v instanceof FormattedValue) && !PROXY_ID.has(v)) { - obj[key] = reifyAndApply(v, seen, queryClient, persist, entityRefs, appendMode); + const reified = reifyAndApply(v, seen, queryClient, persist, entityRefs, appendMode, created); + if (reified !== v) obj[key] = reified; } } } @@ -101,14 +121,22 @@ function applyEntity( entity: ParsedEntity, seen: Map, ParsedEntity>, queryClient: QueryClient, - persist: boolean, + persist: PersistMode, parentEntityRefs: Map, appendMode: boolean, + created: Set | undefined, ): Record { - const { key, data, shape: entityShape, rawKeys } = entity; + const { key, data, shape: entityShape, rawKeys, eventKeys, fillsPartial } = entity; const shapeFields = entityShape.shape; - const entityInstance = queryClient.prepareEntity(key, data, entityShape); + // Already applied from another slot: re-applying would release every child ref. + const applied = queryClient.entityMap.getEntity(key); + if (applied !== undefined && applied.parseId === queryClient.currentParseId) { + parentEntityRefs.set(applied, (parentEntityRefs.get(applied) ?? 0) + 1); + return applied.getProxy(entityShape); + } + + const entityInstance = queryClient.prepareEntity(key, data, entityShape, applied); const existingData = entityInstance.data; const isUpdate = existingData !== data; @@ -120,24 +148,47 @@ function applyEntity( : new Map(); let changed = true; - if (isUpdate) { - changed = mergeFields( - shapeFields, - data, - existingData, - rawKeys, - entityInstance, - existingData, - seen, - queryClient, - persist, - childRefs, - appendMode, - ); - if (changed) entityInstance.notify(); - } else { - initFields(shapeFields, data, entityInstance, data, seen, queryClient, persist, childRefs, appendMode); + // A write reaching this instance from inside its own subtree is deferred. + entityInstance._applying = true; + try { + if (isUpdate) { + changed = mergeFields( + shapeFields, + data, + existingData, + rawKeys, + entityInstance, + existingData, + seen, + queryClient, + persist, + childRefs, + appendMode, + created, + ); + if ((persist === true && rawKeys === undefined) || fillsPartial) { + entityInstance._partial = false; + entityInstance._partialKeys = undefined; + } else if (entityInstance._partial && rawKeys !== undefined) { + // Stays partial: another class sharing the typename may hold undeclared fields. + for (const k of rawKeys) entityInstance._partialKeys!.add(k); + } + } else { + initFields(shapeFields, data, entityInstance, data, seen, queryClient, persist, childRefs, appendMode, created); + if (eventKeys !== undefined) { + entityInstance._partial = true; + entityInstance._partialKeys = new Set(eventKeys); + if (entityShape.typenameField !== undefined) entityInstance._partialKeys.add(entityShape.typenameField); + if (typeof entityShape.idField === 'string') entityInstance._partialKeys.add(entityShape.idField); + } else if (persist === false) { + entityInstance._recorded = true; + } + if (persist === 'existing') created?.add(entityInstance); + } + } finally { + entityInstance._applying = false; } + if (isUpdate && changed) entityInstance.notify(); if (appendMode && entityInstance.liveCollections.length > 0) { for (const binding of entityInstance.liveCollections) { @@ -158,8 +209,15 @@ function applyEntity( const newRefs = childRefs.size > 0 ? childRefs : undefined; const refsChanged = !sameRefs(entityInstance.entityRefs, newRefs); // An entity hydrated from the store was never written, so there is no write to skip. - const needsPersist = changed || refsChanged || !entityInstance._persisted || !queryClient.storeReportsDeletes; - entityInstance.setChildRefs(newRefs, persist && needsPersist); + // A queued store deletion could drop the record this apply trusts. + const needsPersist = + changed || + refsChanged || + (!entityInstance._persisted && entityInstance._pendingWrites === 0) || + !queryClient.storeReportsDeletes || + !queryClient.storeIsSettled(); + const writes = persist === true || (persist === 'existing' && isUpdate); + entityInstance.setChildRefs(newRefs, writes && needsPersist); const proxy = entityInstance.getProxy(entityShape); @@ -181,16 +239,19 @@ function mergeFields( entityData: Record, seen: Map, ParsedEntity>, queryClient: QueryClient, - persist: boolean, + persist: PersistMode, childRefs: Map, appendMode: boolean, + created: Set | undefined, ): boolean { let changed = false; for (const [fieldKey, propShape] of entries(shape)) { if (rawKeys !== undefined && !rawKeys.has(fieldKey)) continue; - if (shouldReify(data[fieldKey])) { - data[fieldKey] = reifyAndApply(data[fieldKey], seen, queryClient, persist, childRefs, appendMode); + const raw = data[fieldKey]; + if (shouldReify(raw)) { + const reified = reifyAndApply(raw, seen, queryClient, persist, childRefs, appendMode, created); + if (reified !== raw) data[fieldKey] = reified; } if (propShape instanceof ValidatorDef && propShape._liveConfig !== undefined) { @@ -206,13 +267,14 @@ function mergeFields( entityInstance, entityData, queryClient, + fieldKey, ); changed = true; } } else { const newVal = data[fieldKey]; const oldVal = existingData[fieldKey]; - if (newVal === oldVal) continue; + if (Object.is(newVal, oldVal)) continue; // Replace a union wholesale instead of merging by field, otherwise a // changed variant keeps the old variant's fields. Unlike entity unions, // there is no partial-update path to preserve here: a partial variant @@ -228,12 +290,14 @@ function mergeFields( !(propShape.shape instanceof Set) ? (propShape.shape as Record) : undefined; + // May be frozen by the adapter. + const target = Object.isFrozen(oldVal) ? { ...oldVal } : oldVal; if (nestedShape !== undefined) { if ( mergeFields( nestedShape, newVal, - oldVal, + target, undefined, entityInstance, entityData, @@ -242,20 +306,26 @@ function mergeFields( persist, childRefs, appendMode, + created, ) ) { changed = true; + existingData[fieldKey] = target; + } else { + existingData[fieldKey] = oldVal; } - existingData[fieldKey] = oldVal; } else if (sameKeys(oldVal, newVal)) { - // Copy only differing fields so equal values keep their identity. + // Shapeless object (e.g. a record) with the same keys: copy only the + // fields that differ, so matching values keep their identity. + let merged = false; for (const k of Object.keys(newVal)) { if (!sameValue(oldVal[k], newVal[k])) { - oldVal[k] = newVal[k]; - changed = true; + target[k] = newVal[k]; + merged = true; } } - existingData[fieldKey] = oldVal; + if (merged) changed = true; + existingData[fieldKey] = merged ? target : oldVal; } else { // Copying field by field can't apply a key removal. existingData[fieldKey] = newVal; @@ -281,7 +351,7 @@ export function sameKeys(a: Record, b: Record) /** Structural equality. Proxies compare by identity, formatted values by raw input. */ export function sameValue(a: unknown, b: unknown): boolean { - if (a === b) return true; + if (Object.is(a, b)) return true; if (typeof a !== 'object' || typeof b !== 'object' || a === null || b === null) return false; if (a instanceof FormattedValue || b instanceof FormattedValue) { return a instanceof FormattedValue && b instanceof FormattedValue && a._raw === b._raw; @@ -333,15 +403,18 @@ function initFields( entityData: Record, seen: Map, ParsedEntity>, queryClient: QueryClient, - persist: boolean, + persist: PersistMode, childRefs: Map, appendMode: boolean, + created: Set | undefined, ): void { for (const [fieldKey, propShape] of entries(shape)) { if (!(fieldKey in data)) continue; - if (shouldReify(data[fieldKey])) { - data[fieldKey] = reifyAndApply(data[fieldKey], seen, queryClient, persist, childRefs, appendMode); + const raw = data[fieldKey]; + if (shouldReify(raw)) { + const reified = reifyAndApply(raw, seen, queryClient, persist, childRefs, appendMode, created); + if (reified !== raw) data[fieldKey] = reified; } if (propShape instanceof ValidatorDef && propShape._liveConfig !== undefined) { @@ -351,6 +424,7 @@ function initFields( entityInstance, entityData, queryClient, + fieldKey, ); } else { const val = data[fieldKey]; @@ -365,7 +439,18 @@ function initFields( ? (propShape.shape as Record) : undefined; if (nestedShape !== undefined) { - initFields(nestedShape, val, entityInstance, entityData, seen, queryClient, persist, childRefs, appendMode); + initFields( + nestedShape, + val, + entityInstance, + entityData, + seen, + queryClient, + persist, + childRefs, + appendMode, + created, + ); } } } diff --git a/packages/fetchium/src/errors.ts b/packages/fetchium/src/errors.ts index 270975c..044db65 100644 --- a/packages/fetchium/src/errors.ts +++ b/packages/fetchium/src/errors.ts @@ -169,3 +169,14 @@ export function typeError(path: string, expectedType: InternalObjectFieldTypeDef } return new TypeError(`Validation error at ${path}`); } + +/** + * An in-memory entity doesn't satisfy a cached query's shape. Drops the whole + * cached query rather than filtering the item out. + */ +export class CachedEntityMismatchError extends Error { + constructor(message: string) { + super(message); + this.name = 'CachedEntityMismatchError'; + } +} diff --git a/packages/fetchium/src/parseEntities.ts b/packages/fetchium/src/parseEntities.ts index 1563da6..5dbec38 100644 --- a/packages/fetchium/src/parseEntities.ts +++ b/packages/fetchium/src/parseEntities.ts @@ -7,6 +7,7 @@ import { hashValue } from 'signalium/utils'; import type { QueryClient, PreloadedEntityMap } from './QueryClient.js'; +import type { EntityInstance } from './EntityInstance.js'; import { CaseInsensitiveSet, FormattedValue, @@ -15,7 +16,7 @@ import { VariantGroup, VariantSet, } from './typeDefs.js'; -import { typeError, typeToString, UnknownUnionVariantError } from './errors.js'; +import { typeError, typeToString, UnknownUnionVariantError, CachedEntityMismatchError } from './errors.js'; import { ARRAY_KEY, ArrayDef, @@ -49,8 +50,15 @@ export interface ParsedEntity { data: Record; /** Set for partial event updates — restricts mergeFields to only these keys. */ rawKeys: Set | undefined; + /** The keys a streamed event carried. */ + eventKeys: Set | undefined; + /** A cached record filling in fields an entity built from events lacks. */ + fillsPartial: boolean; } +/** `'pending'` while a check is in progress, so a cycle counts as satisfied. */ +export type TrustMemo = Map, boolean | 'pending'>>; + // ====================================================== // Parse context — bundles threading parameters // ====================================================== @@ -64,6 +72,12 @@ export class ParseContext { isPartialEvent: boolean = false; seen: Map, ParsedEntity> | undefined = undefined; seenByKey: Map | undefined = undefined; + trusted: TrustMemo | undefined = undefined; + /** + * On for input the client doesn't own and may see again (events, effects). + * Off for a fetch result or cached record, copied only where a value differs. + */ + copyInput: boolean = true; reset( queryClient: QueryClient | undefined, @@ -75,6 +89,8 @@ export class ParseContext { this.preloadedEntities = preloadedEntities; this.warn = warn; this.isPartialEvent = isPartialEvent; + this.trusted = undefined; + this.copyInput = true; if (queryClient !== undefined) { if (this.seen === undefined) { this.seen = new Map(); @@ -375,34 +391,45 @@ function parseUnionData( // ====================================================== function parseArrayData(array: unknown[], itemShape: ComplexTypeDef, ctx: ParseContext, path: string): unknown[] { - const result: unknown[] = []; + // An empty array is always new, since a live array grows the one it holds. + let result: unknown[] | undefined = ctx.copyInput || array.length === 0 ? [] : undefined; for (let i = 0; i < array.length; i++) { + const item = array[i]; try { - result.push(parseData(array[i], itemShape as unknown as TypeDef, ctx, `${path}[${i}]`)); + const parsed = parseData(item, itemShape as unknown as TypeDef, ctx, `${path}[${i}]`); + if (result !== undefined) result.push(parsed); + else if (parsed !== item) (result = array.slice(0, i)).push(parsed); } catch (e) { + if (e instanceof CachedEntityMismatchError) throw e; + result ??= array.slice(0, i); ctx.warn('Failed to parse array item, filtering out', { index: i, - value: array[i], + value: item, error: e instanceof Error ? e.message : String(e), }); } } - return result; + return result ?? array; } +// Neither walker writes into its input. + function parseRecordData( record: Record, valueShape: ComplexTypeDef, ctx: ParseContext, path: string, ): Record { + let result: Record | undefined = ctx.copyInput ? { ...record } : undefined; for (const [key, value] of entries(record)) { - record[key] = parseData(value, valueShape as unknown as TypeDef, ctx, `${path}["${key}"]`); + const parsed = parseData(value, valueShape as unknown as TypeDef, ctx, `${path}["${key}"]`); + if (result !== undefined) result[key] = parsed; + else if (parsed !== value) (result = { ...record })[key] = parsed; } - return record; + return result ?? record; } function parseObjectData( @@ -416,12 +443,16 @@ function parseObjectData( } const shape = objectShape.shape; + let result: Record | undefined = ctx.copyInput ? { ...obj } : undefined; for (const [key, propShape] of entries(shape)) { - obj[key] = parseData(obj[key], propShape as unknown as TypeDef, ctx, `${path}.${key}`); + const value = obj[key]; + const parsed = parseData(value, propShape as unknown as TypeDef, ctx, `${path}.${key}`); + if (result !== undefined) result[key] = parsed; + else if (parsed !== value || (value === undefined && !(key in obj))) (result = { ...obj })[key] = parsed; } - return obj; + return result ?? obj; } // ====================================================== @@ -459,33 +490,90 @@ function parseEntityData( return existingEntry.data; } - if (preloadedEntities !== undefined) { - const existing = queryClient.entityMap.getEntity(key); - const preloaded = existing?.data ?? preloadedEntities.get(key); - - if (preloaded === undefined) { - throw new Error(`Cached entity ${key} not found in preloaded map`); - } - - obj = preloaded; - } - const parsedData: Record = {}; // For symbol id fields (QUERY_ID), copy the id onto parsedData so // getOrCreateEntity can read it. entries(shape) skips symbol keys. if (typeof entityShape.idField === 'symbol') { parsedData[entityShape.idField] = id; } + + if (preloadedEntities !== undefined) { + const existing = queryClient.entityMap.getEntity(key); + + let fillKeys: Set | undefined; + const preloaded = preloadedEntities.get(key); + + if (existing !== undefined && existing._partial && existing._partialKeys !== undefined && preloaded !== undefined) { + // Built from events: parse only the fields the record adds. + fillKeys = existing._partialKeys; + obj = preloaded; + } else if (existing !== undefined) { + // A live entity is newer than the cache, and re-parsing parsed values + // corrupts them. Merge nothing, but check it satisfies this shape. + ctx.trusted ??= new Map(); + if (!dataSatisfiesDef(existing.data, entityShape as unknown as ValidatorDef, queryClient, ctx.trusted)) { + throw new CachedEntityMismatchError( + `Cached entity ${entityShape.typenameValue}:${String(existing.id)} in memory does not satisfy the query's shape`, + ); + } + const entry: ParsedEntity = { + key, + shape: entityShape, + data: parsedData, + rawKeys: new Set(), + eventKeys: undefined, + fillsPartial: false, + }; + ctx.seen!.set(parsedData, entry); + ctx.seenByKey!.set(key, entry); + return parsedData; + } else { + if (preloaded === undefined) { + throw new Error(`Cached entity ${key} not found in preloaded map`); + } + obj = preloaded; + } + + if (fillKeys !== undefined) { + const shapeKeys = Object.keys(entityShape.shape); + const rawKeys = new Set(); + for (const k of shapeKeys) if (!fillKeys.has(k)) rawKeys.add(k); + const entry: ParsedEntity = { + key, + shape: entityShape, + data: parsedData, + rawKeys, + eventKeys: undefined, + fillsPartial: true, + }; + ctx.seen!.set(parsedData, entry); + ctx.seenByKey!.set(key, entry); + const entityDesc = `[[${entityShape.typenameValue}:${id}]]`; + for (const [fieldKey, propShape] of entries(entityShape.shape)) { + if (!rawKeys.has(fieldKey)) continue; + parsedData[fieldKey] = parseData( + obj[fieldKey], + propShape as unknown as TypeDef, + ctx, + `${entityDesc}.${fieldKey}`, + ); + } + return parsedData; + } + } // For mutation events updating existing entities, track which keys are // present so mergeFields only touches those fields (true partial update). const existingInStore = queryClient.entityMap.getEntity(key); const isPartial = ctx.isPartialEvent && existingInStore !== undefined; + const eventKeys = ctx.isPartialEvent ? new Set(Object.keys(obj)) : undefined; const entry: ParsedEntity = { key, shape: entityShape, data: parsedData, - rawKeys: isPartial ? new Set(Object.keys(obj)) : undefined, + rawKeys: isPartial ? eventKeys : undefined, + eventKeys, + fillsPartial: false, }; ctx.seen!.set(parsedData, entry); ctx.seenByKey!.set(key, entry); @@ -508,6 +596,167 @@ function parseEntityData( // entitySatisfiesShape // ====================================================== +/** + * Whether parsed entity data satisfies `def` deeply, unlike the top-level + * `entitySatisfiesShape`. Values that can't be judged are accepted. + */ +export function dataSatisfiesDef( + data: Record, + def: ValidatorDef, + queryClient: QueryClient, + visiting: TrustMemo, +): boolean { + const shape = def.shape as Record | undefined; + if (shape === undefined || shape === null) return true; + for (const key of Object.keys(shape)) { + if (key === def.typenameField) continue; + if (!valueSatisfiesDef(data[key], shape[key], queryClient, visiting)) return false; + } + return true; +} + +function allowsMissing(mask: number): boolean { + return (mask & Mask.UNDEFINED) !== 0; +} + +function maskOf(value: unknown): number { + switch (typeof value) { + case 'number': + return Mask.NUMBER; + case 'string': + return Mask.STRING; + case 'boolean': + return Mask.BOOLEAN; + case 'undefined': + return Mask.UNDEFINED; + case 'object': + return value === null ? Mask.NULL : Array.isArray(value) ? Mask.ARRAY : Mask.OBJECT; + default: + return 0; + } +} + +function valueSatisfiesDef(value: unknown, fieldDef: unknown, queryClient: QueryClient, visiting: TrustMemo): boolean { + // Literals: the parser fills a missing literal in, and rejects any other value. + if (fieldDef instanceof VariantSet) return value === fieldDef.value; + if (typeof fieldDef === 'string') return value === fieldDef; + if (fieldDef instanceof Set) return fieldDef.has(value as never); + + if (typeof fieldDef === 'number') { + if (value === undefined) return allowsMissing(fieldDef); + if (value === null) return (fieldDef & Mask.NULL) !== 0; + if ((fieldDef & Mask.HAS_FORMAT) !== 0) return value instanceof FormattedValue; + return (fieldDef & maskOf(value)) !== 0; + } + + if (!(fieldDef instanceof ValidatorDef)) return true; + const mask = fieldDef.mask; + if (value === undefined) return allowsMissing(mask); + if (value === null) return (mask & Mask.NULL) !== 0; + + if (fieldDef._liveConfig !== undefined) return true; + if ((mask & Mask.PARSE_RESULT) !== 0) return typeof value === 'object' && 'success' in (value as object); + if (typeof value !== 'object') { + if ((mask & Mask.HAS_FORMAT) !== 0 && (mask & maskOf(value)) !== 0) return false; + return (mask & maskOf(value)) !== 0 || fieldDef.values?.has(value as never) === true; + } + if (value instanceof FormattedValue) return (mask & Mask.HAS_FORMAT) !== 0; + + if ((mask & Mask.ENTITY) !== 0 && (mask & Mask.UNION) === 0) { + return entitySatisfies(value as object, fieldDef, queryClient, visiting); + } + + if ((mask & Mask.UNION) !== 0) { + const members = fieldDef.shape as Record | undefined; + if (members === undefined || members === null) return true; + if (Array.isArray(value)) { + const itemDef = members[ARRAY_KEY]; + if (itemDef === undefined || typeof itemDef === 'number') return true; + return arraySatisfies(value, itemDef, queryClient, visiting); + } + const entityKey = PROXY_ID.get(value as object); + const typename = + entityKey !== undefined + ? queryClient.entityMap.getEntity(entityKey)?.typename + : ((value as Record)[fieldDef.typenameField ?? '__typename'] as string | undefined); + const member = typename !== undefined ? members[typename] : undefined; + if (member instanceof VariantGroup) { + const source = + entityKey !== undefined ? queryClient.entityMap.getEntity(entityKey)?.data : (value as Record); + const variant = source?.[member.variantField] as string | undefined; + const variantDef = variant !== undefined ? member.defs[variant] : undefined; + return variantDef === undefined ? true : valueSatisfiesDef(value, variantDef, queryClient, visiting); + } + if (member === undefined || typeof member === 'number') { + const recordDef = members[RECORD_KEY]; + if (typename !== undefined || recordDef === undefined || entityKey !== undefined) return true; + for (const item of Object.values(value as Record)) { + if (!valueSatisfiesDef(item, recordDef, queryClient, visiting)) return false; + } + return true; + } + return valueSatisfiesDef(value, member, queryClient, visiting); + } + + if ((mask & Mask.ARRAY) !== 0) { + if (!Array.isArray(value)) return false; + return arraySatisfies(value, fieldDef.shape, queryClient, visiting); + } + + if ((mask & Mask.RECORD) !== 0) { + if (typeof value !== 'object' || Array.isArray(value)) return false; + for (const item of Object.values(value as Record)) { + if (!valueSatisfiesDef(item, fieldDef.shape, queryClient, visiting)) return false; + } + return true; + } + + if ((mask & Mask.OBJECT) !== 0) { + if (typeof value !== 'object' || Array.isArray(value)) return false; + if (PROXY_ID.has(value as object)) return true; + return dataSatisfiesDef(value as Record, fieldDef, queryClient, visiting); + } + + return (mask & maskOf(value)) !== 0; +} + +/** An instance no longer in memory is accepted, since a re-parse would hand out its data too. */ +function entitySatisfies( + value: object, + def: ValidatorDef, + queryClient: QueryClient, + visiting: TrustMemo, +): boolean { + const entityKey = PROXY_ID.get(value); + if (entityKey === undefined) return false; + const child = queryClient.entityMap.getEntity(entityKey); + if (child === undefined) return true; + let checked = visiting.get(child); + if (checked === undefined) visiting.set(child, (checked = new Map())); + const known = checked.get(def); + if (known !== undefined) return known === 'pending' || known; + checked.set(def, 'pending'); + const verdict = dataSatisfiesDef(child.data, def, queryClient, visiting); + checked.set(def, verdict); + return verdict; +} + +/** A shared-typename array is narrowed on read, so a non-matching member is not a mismatch. */ +function arraySatisfies(value: unknown[], itemDef: unknown, queryClient: QueryClient, visiting: TrustMemo): boolean { + let narrowed = false; + if (itemDef instanceof ValidatorDef && (itemDef.mask & Mask.ENTITY) !== 0 && (itemDef.mask & Mask.UNION) === 0) { + const typename = itemDef.typenameValue; + const defs = typename !== undefined ? queryClient.getEntityDefsForTypename(typename) : undefined; + narrowed = defs !== undefined && defs.length > 1; + } + for (const item of value) { + if (valueSatisfiesDef(item, itemDef, queryClient, visiting)) continue; + if (narrowed && typeof item === 'object' && item !== null && PROXY_ID.has(item)) continue; + return false; + } + return true; +} + export function entitySatisfiesShape(data: Record, def: ValidatorDef): boolean { return objectSatisfiesShape(data, def.shape as Record, def.typenameField); } diff --git a/packages/fetchium/src/query-types.ts b/packages/fetchium/src/query-types.ts index 07a53a3..5b70c45 100644 --- a/packages/fetchium/src/query-types.ts +++ b/packages/fetchium/src/query-types.ts @@ -99,14 +99,54 @@ export interface QueryStore { saveEntity(entityKey: number, value: unknown, refIds?: Set): void; + /** + * Writes the fields streamed events supplied, merged over the stored record + * if one exists (references derived from the merged value). Without it the + * client calls `saveEntity`, dropping fields the events did not carry. + * With `ifStored`, nothing is written when no record is stored, and a store + * with `onDelete` reports the key as dropped. + */ + mergeEntity?(entityKey: number, fields: unknown, refIds?: Set, ifStored?: boolean): void; + activateQuery(queryDef: QueryDefinition, storageKey: number): void; deleteQuery(queryKey: number): void; purgeStaleQueries?(): MaybePromise; - /** Called with the key of every record the store drops on its own (eviction, cascade, purge). */ - onDelete?(listener: (key: number) => void): void; + /** + * Called with the key of every record the store drops on its own (eviction, + * cascade, purge). May return an unsubscribe, called from `destroy()`. + * Without it, entities are written on every apply. + */ + onDelete?(listener: (key: number) => void): void | (() => void); + + /** + * Async stores: called with each entity key once its write is applied. The + * client treats an entity as persisted only after this acknowledgement. + */ + onPersisted?(listener: (key: number) => void): void | (() => void); + + /** + * Async stores: whether every queued operation has been processed. While + * `false` the client does not skip unchanged entity writes. Absent means + * always settled. + */ + isSettled?(): boolean; + + /** + * Async stores: a narrower `isSettled` that ignores the client's own entity + * writes, reporting only queued operations that could delete a record. + * Takes precedence over `isSettled` when present. + */ + hasQueuedDeletes?(): boolean; + + /** + * Whether the store holds a record for this key, or `undefined` if it can't + * tell yet. An event for an entity not in memory is written as an `ifStored` + * merge unless this returns `false`. Without `mergeEntity`, only when `true`. + */ + hasEntity?(key: number): boolean | undefined; } export type MaybePromise = T | Promise; diff --git a/packages/fetchium/src/stores/async.ts b/packages/fetchium/src/stores/async.ts index e8cf9e8..2c6518c 100644 --- a/packages/fetchium/src/stores/async.ts +++ b/packages/fetchium/src/stores/async.ts @@ -6,10 +6,13 @@ import { DEFAULT_MAX_COUNT, LAST_USED_PREFIX, lastUsedKeyFor, + mergeStoredRecord, + notifyListeners, queueKeyFor, refCountKeyFor, refIdsKeyFor, updatedAtKeyFor, + VALUE_PREFIX, valueKeyFor, } from './shared.js'; @@ -49,11 +52,21 @@ export type StoreMessage = value: unknown; updatedAt: number; cacheTime: number; - maxCount: number; + /** Absent from older readers. The writer then keeps the persisted queue's size. */ + maxCount?: number; refIds?: number[]; } - | { type: StoreMessageType.SaveEntity; entityKey: number; value: unknown; refIds?: number[] } - | { type: StoreMessageType.ActivateQuery; queryDefId: string; queryKey: number; cacheTime: number; maxCount: number } + | { + type: StoreMessageType.SaveEntity; + entityKey: number; + value: unknown; + refIds?: number[]; + /** `value` holds only some fields, merged over the stored record. */ + merge?: boolean; + /** With `merge`: dropped, and reported deleted, if no record is stored. */ + ifStored?: boolean; + } + | { type: StoreMessageType.ActivateQuery; queryDefId: string; queryKey: number; cacheTime: number; maxCount?: number } | { type: StoreMessageType.DeleteQuery; queryKey: number }; export interface AsyncQueryStoreConfig { @@ -63,6 +76,37 @@ export interface AsyncQueryStoreConfig { }; delegate?: AsyncPersistentStore; // Only provided for writer } + +/** Writer-internal work, serialized with wire messages. Never sent over the wire. */ +class InternalWork { + constructor( + readonly run: () => Promise, + readonly resolve: () => void, + readonly reject: (error: unknown) => void, + ) {} +} + +type QueuedWork = StoreMessage | InternalWork; + +function isStoreMessage(msg: unknown): msg is StoreMessage { + if (typeof msg !== 'object' || msg === null) return false; + const type = (msg as { type?: unknown }).type; + return ( + type === StoreMessageType.SaveQuery || + type === StoreMessageType.SaveEntity || + type === StoreMessageType.ActivateQuery || + type === StoreMessageType.DeleteQuery + ); +} + +function subscribe(listeners: Array<(key: number) => void>, listener: (key: number) => void): () => void { + listeners.push(listener); + return () => { + const idx = listeners.indexOf(listener); + if (idx !== -1) listeners.splice(idx, 1); + }; +} + // ----------------------------------------------------------------------------- // Async QueryStore Implementation // ----------------------------------------------------------------------------- @@ -71,18 +115,39 @@ export class AsyncQueryStore implements QueryStore { private readonly isWriter: boolean; private readonly delegate?: AsyncPersistentStore; private readonly sendMessage: (msg: StoreMessage) => void; - private readonly messageQueue: StoreMessage[] = []; + private readonly messageQueue: QueuedWork[] = []; private readonly queues: Map = new Map(); private queueProcessorPromise?: Promise; private resolveQueueWait?: () => void; private deleteListeners: Array<(key: number) => void> = []; - // Only the writer sees deletions. Readers leave this undefined, so their client writes every apply. - onDelete?: (listener: (key: number) => void) => void; + private persistedListeners: Array<(key: number) => void> = []; + private processing = false; + // Everything except this client's entity writes, whose dropped refs the client tracks. + private queuedDeletes = 0; + private readonly ownEntityWrites = new WeakSet(); + // Mirror of the delegate's value keys so `hasEntity` answers synchronously. + // Changes before the initial scan lands are noted aside and folded in. + private heldKeys: Set | undefined; + private heldSinceScan: Set | undefined = new Set(); + private droppedSinceScan: Set | undefined = new Set(); + // Writer only. A reader offers no hooks, so its client writes every apply. + onDelete?: (listener: (key: number) => void) => () => void; + onPersisted?: (listener: (key: number) => void) => () => void; + hasEntity?: (key: number) => boolean | undefined; + hasQueuedDeletes?: () => boolean; constructor(config: AsyncQueryStoreConfig) { this.isWriter = config.isWriter; this.delegate = config.delegate; - if (this.isWriter) this.onDelete = listener => this.deleteListeners.push(listener); + if (this.isWriter) { + this.onDelete = listener => subscribe(this.deleteListeners, listener); + this.onPersisted = listener => subscribe(this.persistedListeners, listener); + this.hasQueuedDeletes = () => this.queuedDeletes > 0; + this.hasEntity = key => { + if (this.heldKeys !== undefined) return this.heldKeys.has(key); + return this.heldSinceScan!.has(key) ? true : undefined; + }; + } // Connect and get sendMessage function const { sendMessage } = config.connect(this.handleMessage.bind(this)); @@ -93,19 +158,29 @@ export class AsyncQueryStore implements QueryStore { if (!this.delegate) { throw new Error('Writer must have a delegate'); } + void this.scanHeldKeys(); this.startQueueProcessor(); } } + private runInternal(run: () => Promise): Promise { + return new Promise((resolve, reject) => { + this.enqueueMessage(new InternalWork(run, resolve, reject)); + }); + } + private handleMessage(msg: StoreMessage): void { - if (this.isWriter) { - // Enqueue the message for serial processing - this.enqueueMessage(msg); + if (!this.isWriter) return; // Readers don't handle incoming messages + if (!isStoreMessage(msg)) { + console.error('Ignoring a message the store does not understand:', msg); + return; } - // Readers don't handle incoming messages + // Enqueue the message for serial processing + this.enqueueMessage(msg); } - private enqueueMessage(msg: StoreMessage): void { + private enqueueMessage(msg: QueuedWork): void { + if (!this.ownEntityWrites.has(msg)) this.queuedDeletes++; this.messageQueue.push(msg); // Wake up the queue processor if it's waiting if (this.resolveQueueWait) { @@ -116,12 +191,50 @@ export class AsyncQueryStore implements QueryStore { private dispatch(msg: StoreMessage): void { if (this.isWriter) { + if (msg.type === StoreMessageType.SaveEntity) { + this.noteHeld(msg.entityKey); + this.ownEntityWrites.add(msg); + } this.enqueueMessage(msg); } else { this.sendMessage(msg); } } + private noteHeld(id: number): void { + if (this.heldKeys !== undefined) this.heldKeys.add(id); + else { + this.heldSinceScan!.add(id); + this.droppedSinceScan!.delete(id); + } + } + + private noteDropped(id: number): void { + if (this.heldKeys !== undefined) this.heldKeys.delete(id); + else { + this.heldSinceScan!.delete(id); + this.droppedSinceScan!.add(id); + } + } + + /** The previous record may have survived a failed write. Held unless the delegate says otherwise. */ + private async recheckHeld(id: number): Promise { + for (let attempt = 0; ; attempt++) { + try { + if (await this.delegate!.has(valueKeyFor(id))) this.noteHeld(id); + else this.noteDropped(id); + return; + } catch (error) { + if (attempt >= 2) { + console.error('Could not check whether the store holds a record:', error); + this.noteHeld(id); + return; + } + await new Promise(resolve => setTimeout(resolve, 50 << attempt)); + } + } + } + private startQueueProcessor(): void { this.queueProcessorPromise = this.processQueue(); } @@ -138,12 +251,60 @@ export class AsyncQueryStore implements QueryStore { // Process one message at a time const msg = this.messageQueue.shift()!; + this.processing = true; try { - await this.processMessage(msg); + if (msg instanceof InternalWork) { + try { + await msg.run(); + msg.resolve(); + } catch (error) { + msg.reject(error); + } + } else { + await this.processMessage(msg); + } } catch (error) { console.error('Error processing message:', error); + // Report as dropped so the client rewrites it on its next apply. + if (!(msg instanceof InternalWork) && msg.type === StoreMessageType.SaveEntity) { + await this.recheckHeld(msg.entityKey); + notifyListeners(this.deleteListeners, msg.entityKey); + } + } finally { + this.processing = false; + if (!this.ownEntityWrites.has(msg)) this.queuedDeletes--; + } + } + } + + /** Runs alongside the queue, not ahead of it. On repeated failure nothing is assumed held. */ + private async scanHeldKeys(): Promise { + const held = new Set(); + for (let attempt = 0; ; attempt++) { + try { + for (const key of await this.delegate!.getAllKeys()) { + if (!key.startsWith(VALUE_PREFIX)) continue; + const id = Number(key.slice(VALUE_PREFIX.length)); + if (Number.isInteger(id)) held.add(id); + } + break; + } catch (error) { + if (attempt >= 2) { + console.error('Could not read the keys the store holds:', error); + held.clear(); + break; + } + await new Promise(resolve => setTimeout(resolve, 50 << attempt)); } } + for (const id of this.heldSinceScan!) held.add(id); + for (const id of this.droppedSinceScan!) held.delete(id); + this.heldKeys = held; + this.heldSinceScan = this.droppedSinceScan = undefined; + } + + isSettled(): boolean { + return !this.isWriter || (this.messageQueue.length === 0 && !this.processing); } private async processMessage(msg: StoreMessage): Promise { @@ -160,7 +321,11 @@ export class AsyncQueryStore implements QueryStore { ); break; case StoreMessageType.SaveEntity: - await this.writerSaveEntity(msg.entityKey, msg.value, msg.refIds); + if ( + await this.writerSaveEntity(msg.entityKey, msg.value, msg.refIds, msg.merge === true, msg.ifStored === true) + ) { + notifyListeners(this.persistedListeners, msg.entityKey); + } break; case StoreMessageType.ActivateQuery: await this.writerActivateQuery(msg.queryDefId, msg.queryKey, msg.cacheTime, msg.maxCount); @@ -213,6 +378,8 @@ export class AsyncQueryStore implements QueryStore { } for (const entityId of entityIds) { + // Records can reference each other in a cycle. + if (preloaded.has(entityId)) continue; const entityValue = await this.delegate.getString(valueKeyFor(entityId)); if (entityValue === undefined) { @@ -253,14 +420,23 @@ export class AsyncQueryStore implements QueryStore { } saveEntity(entityKey: number, value: unknown, refIds?: Set): void { - const message: StoreMessage = { + this.dispatch({ type: StoreMessageType.SaveEntity, entityKey, value, refIds: refIds ? Array.from(refIds) : undefined, - }; + }); + } - this.dispatch(message); + mergeEntity(entityKey: number, fields: unknown, refIds?: Set, ifStored?: boolean): void { + this.dispatch({ + type: StoreMessageType.SaveEntity, + entityKey, + value: fields, + refIds: refIds ? Array.from(refIds) : undefined, + merge: true, + ifStored, + }); } activateQuery(queryDef: QueryDefinition, queryKey: number): void { @@ -292,7 +468,7 @@ export class AsyncQueryStore implements QueryStore { value: unknown, updatedAt: number, cacheTime: number, - maxCount: number, + maxCount: number | undefined, refIds?: number[], ): Promise { await this.setValue(queryKey, value, refIds ? new Set(refIds) : undefined); @@ -300,15 +476,35 @@ export class AsyncQueryStore implements QueryStore { await this.writerActivateQuery(queryDefId, queryKey, cacheTime, maxCount); } - private async writerSaveEntity(entityKey: number, value: unknown, refIds?: number[]): Promise { + private async writerSaveEntity( + entityKey: number, + value: unknown, + refIds: number[] | undefined, + merge: boolean, + ifStored: boolean, + ): Promise { + if (merge) { + const stored = await this.delegate!.getString(valueKeyFor(entityKey)); + if (stored === undefined && ifStored) { + this.noteDropped(entityKey); + notifyListeners(this.deleteListeners, entityKey); + return false; + } + const merged = stored !== undefined ? mergeStoredRecord(stored, value) : undefined; + if (merged !== undefined) { + await this.setValue(entityKey, merged.value, merged.refIds); + return true; + } + } await this.setValue(entityKey, value, refIds ? new Set(refIds) : undefined); + return true; } private async writerActivateQuery( queryDefId: string, queryKey: number, cacheTime: number, - maxCount: number, + maxCount: number | undefined, ): Promise { if (!(await this.delegate!.has(valueKeyFor(queryKey)))) { return; @@ -321,10 +517,21 @@ export class AsyncQueryStore implements QueryStore { queue = await this.delegate!.getBuffer(queueKey); if (queue === undefined) { - queue = new Uint32Array(maxCount); + queue = new Uint32Array(maxCount ?? DEFAULT_MAX_COUNT); await this.delegate!.setBuffer(queueKey, queue); - } else if (queue.length !== maxCount) { - queue = new Uint32Array(queue.buffer, 0, maxCount); + } else if (maxCount !== undefined && queue.length !== maxCount) { + // A view over the old buffer can't grow, and a shorter one would strand + // the keys it drops. The activated key moves to the front, so keep it. + const resized = new Uint32Array(maxCount); + resized.set(queue.subarray(0, Math.min(queue.length, maxCount))); + for (let i = maxCount; i < queue.length; i++) { + const dropped = queue[i]; + if (dropped !== 0 && dropped !== queryKey) { + await this.writerDeleteValue(dropped); + await this.delegate!.delete(updatedAtKeyFor(dropped)); + } + } + queue = resized; await this.delegate!.setBuffer(queueKey, queue); } @@ -355,7 +562,14 @@ export class AsyncQueryStore implements QueryStore { } } - async purgeStaleQueries(): Promise { + /** Runs through the writer's queue, so a record it drops can't be reported as written afterwards. */ + purgeStaleQueries(): Promise { + if (!this.delegate) return Promise.resolve(); + if (!this.isWriter) return this.purgeStaleQueriesNow(); + return this.runInternal(() => this.purgeStaleQueriesNow()); + } + + private async purgeStaleQueriesNow(): Promise { if (!this.delegate) return; const allKeys = await this.delegate.getAllKeys(); @@ -393,6 +607,7 @@ export class AsyncQueryStore implements QueryStore { const delegate = this.delegate!; await delegate.setString(valueKeyFor(id), JSON.stringify(value)); + this.noteHeld(id); const refIdsKey = refIdsKeyFor(id); @@ -442,8 +657,9 @@ export class AsyncQueryStore implements QueryStore { const refIdsKey = refIdsKeyFor(id); await delegate.delete(valueKeyFor(id)); + this.noteDropped(id); await delegate.delete(refCountKeyFor(id)); - for (let i = 0; i < this.deleteListeners.length; i++) this.deleteListeners[i](id); + notifyListeners(this.deleteListeners, id); const refIds = await delegate.getBuffer(refIdsKey); await delegate.delete(refIdsKey); // Clean up the refIds key diff --git a/packages/fetchium/src/stores/shared.ts b/packages/fetchium/src/stores/shared.ts index 2a94976..93f6fcf 100644 --- a/packages/fetchium/src/stores/shared.ts +++ b/packages/fetchium/src/stores/shared.ts @@ -1,5 +1,6 @@ // Query Instance keys -export const valueKeyFor = (id: number) => `sq:doc:value:${id}`; +export const VALUE_PREFIX = 'sq:doc:value:'; +export const valueKeyFor = (id: number) => `${VALUE_PREFIX}${id}`; export const refCountKeyFor = (id: number) => `sq:doc:refCount:${id}`; export const refIdsKeyFor = (id: number) => `sq:doc:refIds:${id}`; export const updatedAtKeyFor = (id: number) => `sq:doc:updatedAt:${id}`; @@ -15,3 +16,49 @@ export const LAST_USED_PREFIX = 'sq:doc:lastUsed:'; export const DEFAULT_MAX_COUNT = 50; export const DEFAULT_CACHE_TIME = 60 * 24; // 24 hours in minutes export const DEFAULT_GC_TIME = 5; // 5 minutes - in-memory eviction default + +/** Merges at the JSON level and derives refs from the `__entityRef` markers. */ +export function mergeStoredRecord( + stored: string, + partial: unknown, +): { value: Record; refIds: Set | undefined } | undefined { + let record: unknown; + try { + record = JSON.parse(stored); + } catch { + return undefined; + } + if (typeof record !== 'object' || record === null || Array.isArray(record)) return undefined; + const value = { + ...(record as Record), + ...(JSON.parse(JSON.stringify(partial)) as object), + }; + const refIds = new Set(); + collectEntityRefs(value, refIds); + return { value, refIds: refIds.size > 0 ? refIds : undefined }; +} + +function collectEntityRefs(value: unknown, into: Set): void { + if (typeof value !== 'object' || value === null) return; + if (Array.isArray(value)) { + for (let i = 0; i < value.length; i++) collectEntityRefs(value[i], into); + return; + } + const ref = (value as { __entityRef?: unknown }).__entityRef; + if (typeof ref === 'number') { + into.add(ref); + return; + } + for (const key in value) collectEntityRefs((value as Record)[key], into); +} + +/** Notifies over a copy, since a listener may unsubscribe while being notified. */ +export function notifyListeners(listeners: Array<(key: number) => void>, key: number): void { + for (const listener of listeners.slice()) { + try { + listener(key); + } catch (error) { + console.error('A store listener threw:', error); + } + } +} diff --git a/packages/fetchium/src/stores/sync.ts b/packages/fetchium/src/stores/sync.ts index ea18399..f2df858 100644 --- a/packages/fetchium/src/stores/sync.ts +++ b/packages/fetchium/src/stores/sync.ts @@ -11,6 +11,8 @@ import { refIdsKeyFor, updatedAtKeyFor, valueKeyFor, + mergeStoredRecord, + notifyListeners, } from './shared.js'; export interface SyncPersistentStore { @@ -70,14 +72,32 @@ export class MemoryPersistentStore implements SyncPersistentStore { } } +/** Keyed by kv, not store: two stores over one kv delete each other's records. */ +const deleteListenersByKv = new WeakMap void>>(); + export class SyncQueryStore implements QueryStore { queues: Map = new Map(); - private deleteListeners: Array<(key: number) => void> = []; + private readonly deleteListeners: Array<(key: number) => void>; - constructor(private readonly kv: SyncPersistentStore) {} + constructor(private readonly kv: SyncPersistentStore) { + let listeners = deleteListenersByKv.get(kv); + if (listeners === undefined) { + listeners = []; + deleteListenersByKv.set(kv, listeners); + } + this.deleteListeners = listeners; + } - onDelete(listener: (key: number) => void): void { + onDelete(listener: (key: number) => void): () => void { this.deleteListeners.push(listener); + return () => { + const idx = this.deleteListeners.indexOf(listener); + if (idx !== -1) this.deleteListeners.splice(idx, 1); + }; + } + + hasEntity(key: number): boolean { + return this.kv.has(valueKeyFor(key)); } loadQuery(queryDef: QueryDefinition, queryKey: number): CachedQuery | undefined { @@ -114,6 +134,8 @@ export class SyncQueryStore implements QueryStore { private preloadEntities(entityIds: Uint32Array, preloaded: PreloadedEntityMap): void { for (const entityId of entityIds) { + // Records can reference each other in a cycle. + if (preloaded.has(entityId)) continue; const entityValue = this.kv.getString(valueKeyFor(entityId)); if (entityValue === undefined) { @@ -148,6 +170,13 @@ export class SyncQueryStore implements QueryStore { this.setValue(entityKey, value, refIds); } + mergeEntity(entityKey: number, fields: unknown, refIds?: Set): void { + const stored = this.kv.getString(valueKeyFor(entityKey)); + const merged = stored !== undefined ? mergeStoredRecord(stored, fields) : undefined; + if (merged !== undefined) this.setValue(entityKey, merged.value, merged.refIds); + else this.setValue(entityKey, fields, refIds); + } + activateQuery(queryDef: QueryDefinition, queryKey: number): void { if (!this.kv.has(valueKeyFor(queryKey))) { return; @@ -164,7 +193,18 @@ export class SyncQueryStore implements QueryStore { queue = new Uint32Array(maxCount); this.kv.setBuffer(queueKeyFor(queryDefId), queue); } else if (queue.length !== maxCount) { - queue = new Uint32Array(queue.buffer, 0, maxCount); + // A view over the old buffer can't grow, and a shorter one would strand + // the keys it drops. The activated key moves to the front, so keep it. + const resized = new Uint32Array(maxCount); + resized.set(queue.subarray(0, Math.min(queue.length, maxCount))); + for (let i = maxCount; i < queue.length; i++) { + const dropped = queue[i]; + if (dropped !== 0 && dropped !== queryKey) { + this.deleteQuery(dropped); + this.kv.delete(updatedAtKeyFor(dropped)); + } + } + queue = resized; this.kv.setBuffer(queueKeyFor(queryDefId), queue); } @@ -280,7 +320,7 @@ export class SyncQueryStore implements QueryStore { kv.delete(valueKeyFor(id)); kv.delete(refCountKeyFor(id)); - for (let i = 0; i < this.deleteListeners.length; i++) this.deleteListeners[i](id); + notifyListeners(this.deleteListeners, id); const refIds = kv.getBuffer(refIdsKeyFor(id)); kv.delete(refIdsKeyFor(id)); // Clean up the refIds key