From 7f784a82b396b32bcd26787268c7a606039b5e4c Mon Sep 17 00:00:00 2001 From: Adam Azbel-Bedell Date: Mon, 5 Oct 2026 21:29:56 -0400 Subject: [PATCH 01/15] fix(fetchium): optional store hooks for acknowledged, merged and checked entity writes Extends the QueryStore contract with optional methods the client uses to keep write skipping correct. A store without them keeps the pre-0.6 behavior (every apply writes the whole record). - onDelete and the new onPersisted return an unsubscribe. - isSettled(): whether every write handed to the store has been applied; while false the client does not skip entity writes. - hasEntity(key): whether the store holds a record, so a streamed event can refresh the record of an entity that is no longer in memory. - mergeEntity(key, fields, refIds): merges the fields streamed events carried over the stored record (mergeStoredRecord in stores/shared.ts), deriving the references from the merged value. SyncQueryStore shares delete listeners between instances over the same kv, answers hasEntity from the kv and implements mergeEntity. AsyncQueryStore writers acknowledge processed SaveEntity messages, report a failed one as a delete, answer hasEntity from the keys read at start, run purgeStaleQueries through the serial queue, drop malformed channel messages and keep a persisted LRU size for messages without maxCount. Both stores resize a persisted LRU queue into a new buffer instead of throwing (growing) or orphaning entries (shrinking), never evicting the key being activated, and stop loading forever on records that reference each other in a cycle. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/src/app/api/stores-async/page.md | 24 ++- docs/src/app/api/stores-sync/page.md | 19 +- docs/src/app/data/caching/page.md | 2 +- packages/fetchium/src/query-types.ts | 43 ++++- packages/fetchium/src/stores/async.ts | 256 ++++++++++++++++++++++--- packages/fetchium/src/stores/shared.ts | 44 ++++- packages/fetchium/src/stores/sync.ts | 55 +++++- 7 files changed, 395 insertions(+), 48 deletions(-) diff --git a/docs/src/app/api/stores-async/page.md b/docs/src/app/api/stores-async/page.md index d6e5f25..bbe2c6b 100644 --- a/docs/src/app/api/stores-async/page.md +++ b/docs/src/app/api/stores-async/page.md @@ -41,14 +41,19 @@ new AsyncQueryStore(config: AsyncQueryStoreConfig) #### Methods -| Method | Signature | Description | -| ------------------- | -------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `loadQuery` | `(queryDef: QueryDefinition, queryKey: number): Promise` | Loads a cached query. Only works when a `delegate` is available (writer mode). Returns `undefined` if no delegate, if the entry has expired, or if it does not exist. Preloads referenced entities. | -| `saveQuery` | `(queryDef: QueryDefinition, queryKey: number, value: unknown, updatedAt: number, refIds?: Set): void` | Dispatches a save-query message. In writer mode, enqueues for serial processing. In reader mode, sends via `sendMessage`. | -| `saveEntity` | `(entityKey: number, value: unknown, refIds?: Set): void` | Dispatches a save-entity message. | -| `activateQuery` | `(queryDef: QueryDefinition, queryKey: number): void` | Dispatches an activate-query message to update the LRU queue. | -| `deleteQuery` | `(queryKey: number): void` | Dispatches a delete-query message. | -| `purgeStaleQueries` | `(): Promise` | Scans all stored query classes and removes expired entries. Only operates when a `delegate` is available. | +| Method | Signature | Description | +| ------------------- | -------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `loadQuery` | `(queryDef: QueryDefinition, queryKey: number): Promise` | Loads a cached query. Only works when a `delegate` is available (writer mode). Returns `undefined` if no delegate, if the entry has expired, or if it does not exist. Preloads referenced entities. | +| `saveQuery` | `(queryDef: QueryDefinition, queryKey: number, value: unknown, updatedAt: number, refIds?: Set): void` | Dispatches a save-query message. In writer mode, enqueues for serial processing. In reader mode, sends via `sendMessage`. | +| `saveEntity` | `(entityKey: number, value: unknown, refIds?: Set): void` | Dispatches a save-entity message. | +| `mergeEntity` | `(entityKey: number, fields: unknown, refIds?: Set): void` | Dispatches a save-entity message with `merge: true`; the writer merges `fields` over the stored record, if there is one (see `SyncQueryStore.mergeEntity`). | +| `activateQuery` | `(queryDef: QueryDefinition, queryKey: number): void` | Dispatches an activate-query message to update the LRU queue. A queue persisted with a different `maxCount` is resized by the writer: growing reallocates it, shrinking evicts the keys that no longer fit. | +| `deleteQuery` | `(queryKey: number): void` | Dispatches a delete-query message. | +| `purgeStaleQueries` | `(): Promise` | Scans all stored query classes and removes expired entries. In writer mode it runs through the same serial queue as the messages, after everything queued before it; a reader with its own `delegate` runs it directly. Only operates when a `delegate` is available. | +| `onDelete` | `(listener: (key: number) => void): () => void` | Writer mode only. Registers a listener called with the key of every record the writer drops on its own (an LRU eviction, a reference-count cascade, a purge) or fails to write. Returns an unsubscribe function. | +| `onPersisted` | `(listener: (key: number) => void): () => void` | Writer mode only. Registers a listener called with the key of every entity record a queued write has stored. Returns an unsubscribe function. | +| `isSettled` | `(): boolean` | Whether every operation handed to the writer has been applied (nothing queued, nothing mid-flight). Always `true` in reader mode. | +| `hasEntity` | `(key: number): boolean \| undefined` | Writer mode only. Whether the delegate holds a record for the key, answered from a set the writer reads from the delegate when it starts (alongside its queue, not ahead of it) and keeps current with every write it queues and every deletion it performs. `undefined` for a key it has not written itself until that read lands; after a read that keeps failing, nothing is assumed held. | --- @@ -126,6 +131,7 @@ type StoreMessage = value: unknown; updatedAt: number; cacheTime: number; + maxCount?: number; // absent from readers built before 0.6; the writer then keeps the queue's size refIds?: number[]; } | { @@ -133,12 +139,14 @@ type StoreMessage = entityKey: number; value: unknown; refIds?: number[]; + merge?: boolean; // `value` holds only some fields: merge them over the stored record } | { type: 2; // ActivateQuery queryDefId: string; queryKey: number; cacheTime: number; + maxCount?: number; } | { type: 3; // DeleteQuery diff --git a/docs/src/app/api/stores-sync/page.md b/docs/src/app/api/stores-sync/page.md index 6135588..c4f4235 100644 --- a/docs/src/app/api/stores-sync/page.md +++ b/docs/src/app/api/stores-sync/page.md @@ -32,14 +32,17 @@ new SyncQueryStore(kv: SyncPersistentStore) #### Methods -| Method | Signature | Description | -| ------------------- | -------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `loadQuery` | `(queryDef: QueryDefinition, queryKey: number): CachedQuery \| undefined` | Loads a cached query by key. Returns `undefined` if the cache entry has expired (beyond `cacheTime`) or does not exist. Also preloads referenced entities. | -| `saveQuery` | `(queryDef: QueryDefinition, queryKey: number, value: unknown, updatedAt: number, refIds?: Set): void` | Persists a query result, its timestamp, and entity reference IDs. Activates the query in the LRU queue. | -| `saveEntity` | `(entityKey: number, value: unknown, refIds?: Set): void` | Persists an entity's serialized data and its child entity references. Manages reference counts for nested entities. | -| `activateQuery` | `(queryDef: QueryDefinition, queryKey: number): void` | Moves a query to the front of the LRU queue for its query class. If the queue is full, the oldest entry is evicted. | -| `deleteQuery` | `(queryKey: number): void` | Deletes a query's stored value, reference IDs, and decrements reference counts for all referenced entities. | -| `purgeStaleQueries` | `(): void` | Scans all query classes and removes those whose `lastUsedAt` timestamp exceeds their `cacheTime`. Called automatically on `QueryClient` construction. | +| Method | Signature | Description | +| ------------------- | -------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `loadQuery` | `(queryDef: QueryDefinition, queryKey: number): CachedQuery \| undefined` | Loads a cached query by key. Returns `undefined` if the cache entry has expired (beyond `cacheTime`) or does not exist. Also preloads referenced entities. | +| `saveQuery` | `(queryDef: QueryDefinition, queryKey: number, value: unknown, updatedAt: number, refIds?: Set): void` | Persists a query result, its timestamp, and entity reference IDs. Activates the query in the LRU queue. | +| `saveEntity` | `(entityKey: number, value: unknown, refIds?: Set): void` | Persists an entity's serialized data and its child entity references. Manages reference counts for nested entities. | +| `mergeEntity` | `(entityKey: number, fields: unknown, refIds?: Set): void` | Writes the fields streamed events supplied for an entity built from them: merged over the stored record when there is one (the record's references are then derived from the merged value), written as the record otherwise. A record that cannot be merged into is replaced. | +| `activateQuery` | `(queryDef: QueryDefinition, queryKey: number): void` | Moves a query to the front of the LRU queue for its query class. If the queue is full, the oldest entry is evicted. A queue persisted with a different `maxCount` is resized: growing reallocates it, shrinking evicts the keys that no longer fit. | +| `deleteQuery` | `(queryKey: number): void` | Deletes a query's stored value, reference IDs, and decrements reference counts for all referenced entities. | +| `purgeStaleQueries` | `(): void` | Scans all query classes and removes those whose `lastUsedAt` timestamp exceeds their `cacheTime`. Called automatically on `QueryClient` construction. | +| `onDelete` | `(listener: (key: number) => void): () => void` | Registers a listener called with the key of every record the store drops on its own (an LRU eviction, the reference-count cascade that follows it, a purge). Listeners are shared by every store over the same `kv`. Returns an unsubscribe function; `QueryClient.destroy()` calls it. | +| `hasEntity` | `(key: number): boolean` | Whether the store holds a record for the key. The client refreshes the record of an entity a streamed event updates while the entity is not in memory only when this answers `true`. | --- diff --git a/docs/src/app/data/caching/page.md b/docs/src/app/data/caching/page.md index feacbd1..c3582a0 100644 --- a/docs/src/app/data/caching/page.md +++ b/docs/src/app/data/caching/page.md @@ -133,7 +133,7 @@ class GetUser extends RESTQuery { } ``` -`maxCount` limits how many distinct parameter combinations are stored for this query class. When the limit is exceeded, the oldest entries are evicted. The default is `50`. +`maxCount` limits how many distinct parameter combinations are stored for this query class. When the limit is exceeded, the oldest entries are evicted. The default is `50`. Changing `maxCount` between releases resizes the queue already persisted for the class the first time the store touches that class after starting: growing keeps every entry, shrinking evicts the oldest entries that no longer fit. --- diff --git a/packages/fetchium/src/query-types.ts b/packages/fetchium/src/query-types.ts index a4246ea..4578cc9 100644 --- a/packages/fetchium/src/query-types.ts +++ b/packages/fetchium/src/query-types.ts @@ -111,14 +111,53 @@ export interface QueryStore { saveEntity(entityKey: number, value: unknown, refIds?: Set): void; + /** + * Writes the fields streamed events supplied for an entity built from them: + * a store that holds a record for the key merges `fields` over it (and + * derives the record's references from the merged value); one that does not + * writes `fields` as the record. A store without this method gets the + * entity's whole in-memory data through `saveEntity` instead, which drops + * from its record the fields the events did not carry. + */ + mergeEntity?(entityKey: number, fields: unknown, refIds?: Set): void; + activateQuery(queryDef: QueryDefinition, storageKey: number): void; deleteQuery(queryKey: number): void; purgeStaleQueries?(): MaybePromise; - /** Called with the key of every record the store drops on its own (eviction, cascade, purge). */ - onDelete?(listener: (key: number) => void): void; + /** + * Called with the key of every record the store drops on its own (eviction, + * cascade, purge). May return an unsubscribe function; the client calls it + * from `destroy()`. A store without this hook is written on every apply. + */ + onDelete?(listener: (key: number) => void): void | (() => void); + + /** + * For stores that process writes asynchronously: called with the key of + * every entity record once its write has actually been applied. While a + * store offers this, the client treats an entity as persisted only after + * the acknowledgement, so a write the store dropped is retried. + */ + onPersisted?(listener: (key: number) => void): void | (() => void); + + /** + * For stores that process writes asynchronously: whether every operation + * handed to the store has been processed. While it returns `false` the + * client does not skip entity writes, since an operation still queued could + * delete the record the skip relies on. Absent means always settled. + */ + isSettled?(): boolean; + + /** + * Whether the store currently holds a record for this entity key, or + * `undefined` while it cannot tell yet (a store that has not finished + * reading what it holds). A streamed event for an entity that is not in + * memory refreshes the record only when this answers `true`; a store + * without this hook is not written for such events. + */ + hasEntity?(key: number): boolean | undefined; } export type MaybePromise = T | Promise; diff --git a/packages/fetchium/src/stores/async.ts b/packages/fetchium/src/stores/async.ts index 614a536..3946c5d 100644 --- a/packages/fetchium/src/stores/async.ts +++ b/packages/fetchium/src/stores/async.ts @@ -6,10 +6,12 @@ import { DEFAULT_MAX_COUNT, LAST_USED_PREFIX, lastUsedKeyFor, + mergeStoredRecord, queueKeyFor, refCountKeyFor, refIdsKeyFor, updatedAtKeyFor, + VALUE_PREFIX, valueKeyFor, } from './shared.js'; @@ -49,11 +51,19 @@ export type StoreMessage = value: unknown; updatedAt: number; cacheTime: number; - maxCount: number; + /** Absent on the wire from readers built before 0.6; the writer then keeps the persisted queue's size. */ + maxCount?: number; refIds?: number[]; } - | { type: StoreMessageType.SaveEntity; entityKey: number; value: unknown; refIds?: number[] } - | { type: StoreMessageType.ActivateQuery; queryDefId: string; queryKey: number; cacheTime: number; maxCount: number } + | { + type: StoreMessageType.SaveEntity; + entityKey: number; + value: unknown; + refIds?: number[]; + /** `value` holds only some fields; the writer merges them over the stored record, if any. */ + merge?: boolean; + } + | { type: StoreMessageType.ActivateQuery; queryDefId: string; queryKey: number; cacheTime: number; maxCount?: number } | { type: StoreMessageType.DeleteQuery; queryKey: number }; export interface AsyncQueryStoreConfig { @@ -63,6 +73,42 @@ export interface AsyncQueryStoreConfig { }; delegate?: AsyncPersistentStore; // Only provided for writer } + +/** + * Writer-internal work that runs through the same serial queue as the wire + * messages, so it is ordered with respect to the writes and deletions around + * it. Never sent over the wire, and distinguishable from anything that is. + */ +class InternalWork { + constructor( + readonly run: () => Promise, + readonly resolve: () => void, + readonly reject: (error: unknown) => void, + ) {} +} + +type QueuedWork = StoreMessage | InternalWork; + +/** Whether something received on the channel is a message this writer handles. */ +function isStoreMessage(msg: unknown): msg is StoreMessage { + if (typeof msg !== 'object' || msg === null) return false; + const type = (msg as { type?: unknown }).type; + return ( + type === StoreMessageType.SaveQuery || + type === StoreMessageType.SaveEntity || + type === StoreMessageType.ActivateQuery || + type === StoreMessageType.DeleteQuery + ); +} + +function subscribe(listeners: Array<(key: number) => void>, listener: (key: number) => void): () => void { + listeners.push(listener); + return () => { + const idx = listeners.indexOf(listener); + if (idx !== -1) listeners.splice(idx, 1); + }; +} + // ----------------------------------------------------------------------------- // Async QueryStore Implementation // ----------------------------------------------------------------------------- @@ -71,18 +117,39 @@ export class AsyncQueryStore implements QueryStore { private readonly isWriter: boolean; private readonly delegate?: AsyncPersistentStore; private readonly sendMessage: (msg: StoreMessage) => void; - private readonly messageQueue: StoreMessage[] = []; + private readonly messageQueue: QueuedWork[] = []; private readonly queues: Map = new Map(); private queueProcessorPromise?: Promise; private resolveQueueWait?: () => void; private deleteListeners: Array<(key: number) => void> = []; - // Only the writer sees deletions; a reader offers no hook, so the client writes every apply. - onDelete?: (listener: (key: number) => void) => void; + private persistedListeners: Array<(key: number) => void> = []; + private processing = false; + // The ids whose value the delegate holds (or is about to: a queued write + // counts), mirrored by the writer so that `hasEntity` can answer + // synchronously. Read from the delegate once at start, without holding up + // the queue; until that read lands, writes and deletions are noted aside + // and folded in. + private heldKeys: Set | undefined; + private heldSinceScan: Set | undefined = new Set(); + private droppedSinceScan: Set | undefined = new Set(); + // Only the writer sees deletions and completed writes; a reader offers no + // hook, so the client writes every apply. + onDelete?: (listener: (key: number) => void) => () => void; + onPersisted?: (listener: (key: number) => void) => () => void; + hasEntity?: (key: number) => boolean | undefined; constructor(config: AsyncQueryStoreConfig) { this.isWriter = config.isWriter; this.delegate = config.delegate; - if (this.isWriter) this.onDelete = listener => this.deleteListeners.push(listener); + if (this.isWriter) { + this.onDelete = listener => subscribe(this.deleteListeners, listener); + this.onPersisted = listener => subscribe(this.persistedListeners, listener); + this.hasEntity = key => { + if (this.heldKeys !== undefined) return this.heldKeys.has(key); + // Not read yet: only what this writer itself wrote is known. + return this.heldSinceScan!.has(key) ? true : undefined; + }; + } // Connect and get sendMessage function const { sendMessage } = config.connect(this.handleMessage.bind(this)); @@ -93,19 +160,29 @@ export class AsyncQueryStore implements QueryStore { if (!this.delegate) { throw new Error('Writer must have a delegate'); } + void this.scanHeldKeys(); this.startQueueProcessor(); } } + /** Queues writer-internal work behind everything already queued. */ + private runInternal(run: () => Promise): Promise { + return new Promise((resolve, reject) => { + this.enqueueMessage(new InternalWork(run, resolve, reject)); + }); + } + private handleMessage(msg: StoreMessage): void { - if (this.isWriter) { - // Enqueue the message for serial processing - this.enqueueMessage(msg); + if (!this.isWriter) return; // Readers don't handle incoming messages + if (!isStoreMessage(msg)) { + console.error('Ignoring a message the store does not understand:', msg); + return; } - // Readers don't handle incoming messages + // Enqueue the message for serial processing + this.enqueueMessage(msg); } - private enqueueMessage(msg: StoreMessage): void { + private enqueueMessage(msg: QueuedWork): void { this.messageQueue.push(msg); // Wake up the queue processor if it's waiting if (this.resolveQueueWait) { @@ -116,12 +193,29 @@ export class AsyncQueryStore implements QueryStore { private dispatch(msg: StoreMessage): void { if (this.isWriter) { + if (msg.type === StoreMessageType.SaveEntity) this.noteHeld(msg.entityKey); this.enqueueMessage(msg); } else { this.sendMessage(msg); } } + private noteHeld(id: number): void { + if (this.heldKeys !== undefined) this.heldKeys.add(id); + else { + this.heldSinceScan!.add(id); + this.droppedSinceScan!.delete(id); + } + } + + private noteDropped(id: number): void { + if (this.heldKeys !== undefined) this.heldKeys.delete(id); + else { + this.heldSinceScan!.delete(id); + this.droppedSinceScan!.add(id); + } + } + private startQueueProcessor(): void { this.queueProcessorPromise = this.processQueue(); } @@ -138,16 +232,76 @@ export class AsyncQueryStore implements QueryStore { // Process one message at a time const msg = this.messageQueue.shift()!; + this.processing = true; try { - await this.processMessage(msg); + if (msg instanceof InternalWork) { + try { + await msg.run(); + msg.resolve(); + } catch (error) { + msg.reject(error); + } + } else { + await this.processMessage(msg); + } } catch (error) { console.error('Error processing message:', error); + // A failed entity write leaves the record missing or stale, which to + // the client is the same as the store having dropped it: it must + // write the entity again on its next apply. + if (!(msg instanceof InternalWork) && msg.type === StoreMessageType.SaveEntity) { + this.noteDropped(msg.entityKey); + for (let i = 0; i < this.deleteListeners.length; i++) this.deleteListeners[i](msg.entityKey); + } + } finally { + this.processing = false; } } } + /** + * Reads which records the delegate holds, alongside the queue rather than + * ahead of it. Writes and deletions processed meanwhile are folded in. + * A read that fails is retried; if it keeps failing, nothing is assumed + * held (an event for an entity not in memory is then not written). + */ + private async scanHeldKeys(): Promise { + const held = new Set(); + for (let attempt = 0; ; attempt++) { + try { + for (const key of await this.delegate!.getAllKeys()) { + if (!key.startsWith(VALUE_PREFIX)) continue; + const id = Number(key.slice(VALUE_PREFIX.length)); + if (Number.isInteger(id)) held.add(id); + } + break; + } catch (error) { + if (attempt >= 2) { + console.error('Could not read the keys the store holds:', error); + held.clear(); + break; + } + await new Promise(resolve => setTimeout(resolve, 50 << attempt)); + } + } + for (const id of this.heldSinceScan!) held.add(id); + for (const id of this.droppedSinceScan!) held.delete(id); + this.heldKeys = held; + this.heldSinceScan = this.droppedSinceScan = undefined; + } + + /** + * Whether every operation handed to this store has been applied. Readers + * hand their operations to a writer elsewhere and never skip writes anyway. + */ + isSettled(): boolean { + return !this.isWriter || (this.messageQueue.length === 0 && !this.processing); + } + private async processMessage(msg: StoreMessage): Promise { switch (msg.type) { + // `maxCount` is absent from messages sent by readers built before it + // was added; the writer then keeps whatever size the queue has. case StoreMessageType.SaveQuery: await this.writerSaveQuery( msg.queryDefId, @@ -160,7 +314,8 @@ export class AsyncQueryStore implements QueryStore { ); break; case StoreMessageType.SaveEntity: - await this.writerSaveEntity(msg.entityKey, msg.value, msg.refIds); + await this.writerSaveEntity(msg.entityKey, msg.value, msg.refIds, msg.merge === true); + for (let i = 0; i < this.persistedListeners.length; i++) this.persistedListeners[i](msg.entityKey); break; case StoreMessageType.ActivateQuery: await this.writerActivateQuery(msg.queryDefId, msg.queryKey, msg.cacheTime, msg.maxCount); @@ -213,6 +368,9 @@ export class AsyncQueryStore implements QueryStore { } for (const entityId of entityIds) { + // Records can reference each other in a cycle (a file whose record + // links back to its folder); one read per record. + if (preloaded.has(entityId)) continue; const entityValue = await this.delegate.getString(valueKeyFor(entityId)); if (entityValue === undefined) { @@ -253,14 +411,22 @@ export class AsyncQueryStore implements QueryStore { } saveEntity(entityKey: number, value: unknown, refIds?: Set): void { - const message: StoreMessage = { + this.dispatch({ type: StoreMessageType.SaveEntity, entityKey, value, refIds: refIds ? Array.from(refIds) : undefined, - }; + }); + } - this.dispatch(message); + mergeEntity(entityKey: number, fields: unknown, refIds?: Set): void { + this.dispatch({ + type: StoreMessageType.SaveEntity, + entityKey, + value: fields, + refIds: refIds ? Array.from(refIds) : undefined, + merge: true, + }); } activateQuery(queryDef: QueryDefinition, queryKey: number): void { @@ -292,7 +458,7 @@ export class AsyncQueryStore implements QueryStore { value: unknown, updatedAt: number, cacheTime: number, - maxCount: number, + maxCount: number | undefined, refIds?: number[], ): Promise { await this.setValue(queryKey, value, refIds ? new Set(refIds) : undefined); @@ -300,7 +466,20 @@ export class AsyncQueryStore implements QueryStore { await this.writerActivateQuery(queryDefId, queryKey, cacheTime, maxCount); } - private async writerSaveEntity(entityKey: number, value: unknown, refIds?: number[]): Promise { + private async writerSaveEntity( + entityKey: number, + value: unknown, + refIds: number[] | undefined, + merge: boolean, + ): Promise { + if (merge) { + const stored = await this.delegate!.getString(valueKeyFor(entityKey)); + const merged = stored !== undefined ? mergeStoredRecord(stored, value) : undefined; + if (merged !== undefined) { + await this.setValue(entityKey, merged.value, merged.refIds); + return; + } + } await this.setValue(entityKey, value, refIds ? new Set(refIds) : undefined); } @@ -308,7 +487,7 @@ export class AsyncQueryStore implements QueryStore { queryDefId: string, queryKey: number, cacheTime: number, - maxCount: number, + maxCount: number | undefined, ): Promise { if (!(await this.delegate!.has(valueKeyFor(queryKey)))) { return; @@ -321,10 +500,25 @@ export class AsyncQueryStore implements QueryStore { queue = await this.delegate!.getBuffer(queueKey); if (queue === undefined) { - queue = new Uint32Array(maxCount); + queue = new Uint32Array(maxCount ?? DEFAULT_MAX_COUNT); await this.delegate!.setBuffer(queueKey, queue); - } else if (queue.length !== maxCount) { - queue = new Uint32Array(queue.buffer, 0, maxCount); + } else if (maxCount !== undefined && queue.length !== maxCount) { + // `maxCount` changed since the queue was written (until 0.6 the writer + // always used the default, so every older install has a 50-slot queue). + // A view over the old buffer cannot grow past it, and a shorter view + // would silently strand the keys it drops: copy what fits and evict + // the rest properly. The key being activated is about to move to the + // front, so it is kept whichever slot it sits in. + const resized = new Uint32Array(maxCount); + resized.set(queue.subarray(0, Math.min(queue.length, maxCount))); + for (let i = maxCount; i < queue.length; i++) { + const dropped = queue[i]; + if (dropped !== 0 && dropped !== queryKey) { + await this.writerDeleteValue(dropped); + await this.delegate!.delete(updatedAtKeyFor(dropped)); + } + } + queue = resized; await this.delegate!.setBuffer(queueKey, queue); } @@ -355,7 +549,19 @@ export class AsyncQueryStore implements QueryStore { } } - async purgeStaleQueries(): Promise { + /** + * Drops every query definition's cached queries once the definition has not + * been used for its `cacheTime`. Runs through the writer's serial queue, so + * its deletions are ordered with the writes and acknowledgements around + * them: a record it drops cannot be reported as written afterwards. + */ + purgeStaleQueries(): Promise { + if (!this.delegate) return Promise.resolve(); + if (!this.isWriter) return this.purgeStaleQueriesNow(); + return this.runInternal(() => this.purgeStaleQueriesNow()); + } + + private async purgeStaleQueriesNow(): Promise { if (!this.delegate) return; const allKeys = await this.delegate.getAllKeys(); @@ -393,6 +599,7 @@ export class AsyncQueryStore implements QueryStore { const delegate = this.delegate!; await delegate.setString(valueKeyFor(id), JSON.stringify(value)); + this.noteHeld(id); const refIdsKey = refIdsKeyFor(id); @@ -442,6 +649,7 @@ export class AsyncQueryStore implements QueryStore { const refIdsKey = refIdsKeyFor(id); await delegate.delete(valueKeyFor(id)); + this.noteDropped(id); await delegate.delete(refCountKeyFor(id)); for (let i = 0; i < this.deleteListeners.length; i++) this.deleteListeners[i](id); diff --git a/packages/fetchium/src/stores/shared.ts b/packages/fetchium/src/stores/shared.ts index 2a94976..e2ed01c 100644 --- a/packages/fetchium/src/stores/shared.ts +++ b/packages/fetchium/src/stores/shared.ts @@ -1,5 +1,6 @@ // Query Instance keys -export const valueKeyFor = (id: number) => `sq:doc:value:${id}`; +export const VALUE_PREFIX = 'sq:doc:value:'; +export const valueKeyFor = (id: number) => `${VALUE_PREFIX}${id}`; export const refCountKeyFor = (id: number) => `sq:doc:refCount:${id}`; export const refIdsKeyFor = (id: number) => `sq:doc:refIds:${id}`; export const updatedAtKeyFor = (id: number) => `sq:doc:updatedAt:${id}`; @@ -15,3 +16,44 @@ export const LAST_USED_PREFIX = 'sq:doc:lastUsed:'; export const DEFAULT_MAX_COUNT = 50; export const DEFAULT_CACHE_TIME = 60 * 24; // 24 hours in minutes export const DEFAULT_GC_TIME = 5; // 5 minutes - in-memory eviction default + +/** + * Merges the fields of a partial record over a stored one, at the JSON level + * (proxies have already been serialized to `{ __entityRef }` markers), and + * derives the merged record's references from the markers it contains. + */ +export function mergeStoredRecord( + stored: string, + partial: unknown, +): { value: Record; refIds: Set | undefined } | undefined { + let record: unknown; + try { + record = JSON.parse(stored); + } catch { + return undefined; + } + // A record that is not an object is not one to merge into; the caller + // writes the fields as the record, which heals it. + if (typeof record !== 'object' || record === null || Array.isArray(record)) return undefined; + const value = { + ...(record as Record), + ...(JSON.parse(JSON.stringify(partial)) as object), + }; + const refIds = new Set(); + collectEntityRefs(value, refIds); + return { value, refIds: refIds.size > 0 ? refIds : undefined }; +} + +function collectEntityRefs(value: unknown, into: Set): void { + if (typeof value !== 'object' || value === null) return; + if (Array.isArray(value)) { + for (let i = 0; i < value.length; i++) collectEntityRefs(value[i], into); + return; + } + const ref = (value as { __entityRef?: unknown }).__entityRef; + if (typeof ref === 'number') { + into.add(ref); + return; + } + for (const key in value) collectEntityRefs((value as Record)[key], into); +} diff --git a/packages/fetchium/src/stores/sync.ts b/packages/fetchium/src/stores/sync.ts index ea18399..1f96303 100644 --- a/packages/fetchium/src/stores/sync.ts +++ b/packages/fetchium/src/stores/sync.ts @@ -11,6 +11,7 @@ import { refIdsKeyFor, updatedAtKeyFor, valueKeyFor, + mergeStoredRecord, } from './shared.js'; export interface SyncPersistentStore { @@ -70,14 +71,36 @@ export class MemoryPersistentStore implements SyncPersistentStore { } } +/** + * Delete listeners are keyed by the backing kv, not the store instance: two + * stores over one kv delete each other's records, and a client behind either + * must hear about it. + */ +const deleteListenersByKv = new WeakMap void>>(); + export class SyncQueryStore implements QueryStore { queues: Map = new Map(); - private deleteListeners: Array<(key: number) => void> = []; + private readonly deleteListeners: Array<(key: number) => void>; - constructor(private readonly kv: SyncPersistentStore) {} + constructor(private readonly kv: SyncPersistentStore) { + let listeners = deleteListenersByKv.get(kv); + if (listeners === undefined) { + listeners = []; + deleteListenersByKv.set(kv, listeners); + } + this.deleteListeners = listeners; + } - onDelete(listener: (key: number) => void): void { + onDelete(listener: (key: number) => void): () => void { this.deleteListeners.push(listener); + return () => { + const idx = this.deleteListeners.indexOf(listener); + if (idx !== -1) this.deleteListeners.splice(idx, 1); + }; + } + + hasEntity(key: number): boolean { + return this.kv.has(valueKeyFor(key)); } loadQuery(queryDef: QueryDefinition, queryKey: number): CachedQuery | undefined { @@ -114,6 +137,9 @@ export class SyncQueryStore implements QueryStore { private preloadEntities(entityIds: Uint32Array, preloaded: PreloadedEntityMap): void { for (const entityId of entityIds) { + // Records can reference each other in a cycle (a file whose record + // links back to its folder); one read per record. + if (preloaded.has(entityId)) continue; const entityValue = this.kv.getString(valueKeyFor(entityId)); if (entityValue === undefined) { @@ -148,6 +174,13 @@ export class SyncQueryStore implements QueryStore { this.setValue(entityKey, value, refIds); } + mergeEntity(entityKey: number, fields: unknown, refIds?: Set): void { + const stored = this.kv.getString(valueKeyFor(entityKey)); + const merged = stored !== undefined ? mergeStoredRecord(stored, fields) : undefined; + if (merged !== undefined) this.setValue(entityKey, merged.value, merged.refIds); + else this.setValue(entityKey, fields, refIds); + } + activateQuery(queryDef: QueryDefinition, queryKey: number): void { if (!this.kv.has(valueKeyFor(queryKey))) { return; @@ -164,7 +197,21 @@ export class SyncQueryStore implements QueryStore { queue = new Uint32Array(maxCount); this.kv.setBuffer(queueKeyFor(queryDefId), queue); } else if (queue.length !== maxCount) { - queue = new Uint32Array(queue.buffer, 0, maxCount); + // `maxCount` changed since the queue was written. A view over the old + // buffer cannot grow past it, and a shorter view would silently strand + // the keys it drops: copy what fits and evict the rest properly. The + // key being activated is about to move to the front, so it is kept + // whichever slot it sits in. + const resized = new Uint32Array(maxCount); + resized.set(queue.subarray(0, Math.min(queue.length, maxCount))); + for (let i = maxCount; i < queue.length; i++) { + const dropped = queue[i]; + if (dropped !== 0 && dropped !== queryKey) { + this.deleteQuery(dropped); + this.kv.delete(updatedAtKeyFor(dropped)); + } + } + queue = resized; this.kv.setBuffer(queueKeyFor(queryDefId), queue); } From c9b4dfe4e27b46cbde036be9e3a8a90fbf1c3ef7 Mon Sep 17 00:00:00 2001 From: Adam Azbel-Bedell Date: Mon, 5 Oct 2026 21:30:10 -0400 Subject: [PATCH 02/15] fix(fetchium): close the snapshot, hydration and event-persistence regressions A differential hunt against the tree before the snapshot fast path found 17 client-observable regressions from the snapshot fast path, unchanged- apply skipping and single-write events. This closes them on top of the synchronous hydration and fast delivery work: - Dev snapshots are frozen, so a consumer that mutates one fails at its own call site. A detected snapshot drift is served from the re-read and raised out of band (the client's log.error, then an uncaught error), never from inside a snapshot, where it escaped Signalium's watcher flush and stalled every consumer on the page. The fast path compares with Object.is, pairs rebuilt Sets in linear time, and splits static and dynamic fields per client so two classes sharing a typename stay fresh. - Hydrating a cached query never re-parses an entity already in memory. The in-memory data must structurally satisfy the query's shape; if not, a CachedEntityMismatchError lands in hydrate()'s catch, which drops the cached query, clears updatedAt and fetches. Entities built from streamed events are completed from their record. The parser no longer writes into the objects it is given. - Streamed events write the existing entities they changed and the entities they created once a written record references them (root before routing, children first, deferred writes for back references, unrouted roots evicted with everything they created). Event-built entities merge only the fields their events carried through the store's mergeEntity; records the store holds but that are not in memory are refreshed the same way. - _persisted is set only after a successful write (or the store's acknowledgement), so a write that threw is retried by the next apply, and no write is skipped while an async writer has work queued. - destroy() removes the store listeners it registered; late query extras notify a shared root entity; hot-reloaded proxies are walked; an entity listed twice in one payload is applied once; a shared-typename narrowed array consumes the members it leaves out. 55 tests added (skip-write-and-snapshot-fixes, event-persistence); the entity-snapshot, same-value, entity-write-counts and parsing tests follow the new contract (drift via the handler hook, frozen snapshots, NaN/-0, merges counted as writes). Co-Authored-By: Claude Opus 5.5 (1M context) --- .changeset/skip-write-and-snapshot-fixes.md | 17 + docs/src/app/api/fetchium-react/page.md | 2 +- docs/src/app/api/fetchium/page.md | 26 +- docs/src/app/core/entities/page.md | 2 +- docs/src/app/setup/project-setup/page.md | 2 + packages/fetchium/src/ConstraintMatcher.ts | 10 +- packages/fetchium/src/EntityInstance.ts | 423 ++++- packages/fetchium/src/EntityStore.ts | 26 +- packages/fetchium/src/LiveCollection.ts | 23 +- packages/fetchium/src/QueryClient.ts | 212 ++- packages/fetchium/src/QueryResult.ts | 4 + .../src/__tests__/entity-snapshot.test.ts | 51 +- .../src/__tests__/entity-write-counts.test.ts | 25 +- .../src/__tests__/event-persistence.test.ts | 1544 +++++++++++++++++ .../src/__tests__/parsing/entity.test.ts | 8 +- .../fetchium/src/__tests__/same-value.test.ts | 7 +- .../skip-write-and-snapshot-fixes.test.ts | 1247 +++++++++++++ packages/fetchium/src/applyEntities.ts | 166 +- packages/fetchium/src/errors.ts | 13 + packages/fetchium/src/parseEntities.ts | 295 +++- 20 files changed, 3918 insertions(+), 185 deletions(-) create mode 100644 .changeset/skip-write-and-snapshot-fixes.md create mode 100644 packages/fetchium/src/__tests__/event-persistence.test.ts create mode 100644 packages/fetchium/src/__tests__/skip-write-and-snapshot-fixes.test.ts diff --git a/.changeset/skip-write-and-snapshot-fixes.md b/.changeset/skip-write-and-snapshot-fixes.md new file mode 100644 index 0000000..fac9ebc --- /dev/null +++ b/.changeset/skip-write-and-snapshot-fixes.md @@ -0,0 +1,17 @@ +--- +'fetchium': patch +--- + +Fixes for the entity snapshot fast path, unchanged-apply skipping and single-write events introduced in 0.6.0. + +- Development builds hand out frozen snapshots, so a consumer that mutates one (a `sort()` in render, an assignment) fails at its own call site with a TypeError. Production snapshots stay mutable; treat them as read-only. The parser no longer writes into the objects it is given, so a snapshot (or part of one) handed back through a mutation event or a mutation's `effects` is read, not mutated, and two events sharing one nested object no longer make two entities share it. +- The dev-only stale-snapshot check no longer throws from inside a snapshot. A throw there escaped Signalium's watcher flush and stopped every consumer on the page from updating. It now serves the re-read value and raises the error out of band, once per field: through the client's `log.error` and then as an uncaught error (a red box in React Native, an uncaught error in the browser, a failed run under vitest). It compares with `Object.is` (so a `NaN` field no longer trips it, and `-0` over `0` is applied at every level), compares a rebuilt `Set` in linear time, and restores the key set of a snapshot a consumer altered. +- Hydrating a cached query no longer re-parses an entity that is already in memory. The re-parse ran already-parsed values through the parser again in place, wiping nested optional formats and parse results, and since the apply then saw no change it never notified; consumers of the first query kept stale snapshots. The in-memory data is used as-is once it structurally satisfies the query's shape, nested objects, arrays, records, unions and child entities included (members an array narrows out on read, and children that have left memory, are not held against it); otherwise the whole cached query is dropped and fetched, never served with an item missing. An entity built from streamed events is completed from its record instead. +- A streamed `create` (or an `update` for an entity not in memory) now writes the existing entities its payload changed, and the entities it created only once a written record references them: the root a live array retains is written before the event is routed (a write that fails is logged and nothing is routed, as a failed event always behaved), the rest follow from the records that reference them, and what nothing references is evicted at once, `gcTime` or not. A record's write puts its missing children first; a payload that links back to an ancestor defers both writes to the end of the apply. An entity built from events remembers the fields its events carried: its writes go through the store's new `mergeEntity`, which merges those fields over the record on disk, so the fields the events did not carry survive (a store without `mergeEntity` gets the whole in-memory data through `saveEntity`, as before 0.6). An event for an entity the store holds a record of (`hasEntity`) but that is not in memory refreshes that record the same way; a store that cannot say what it holds is not written, so no record nothing references is left behind. +- An entity that appears more than once in one payload (an author listed under `author` and again under `mentions`) is applied once: it keeps its child references and is written once. +- `_persisted` is only set once the store write succeeded, so a write that threw is retried by the next apply. `AsyncQueryStore` writers acknowledge processed entity writes (`onPersisted`), report a failed one, answer `hasEntity` from the keys they read from the delegate at start (alongside the queue, so writes do not wait for it; a queued write counts as held) and keep current, run `purgeStaleQueries` through the same serial queue as the writes (a reader with its own delegate still purges directly), ignore anything on the channel that is not one of their messages, and expose `isSettled()`; the client counts its own outstanding writes, so an acknowledgement of a write another instance queued does not mark a hydrated instance current, and it does not skip a write while the writer has operations queued, since one of them may delete the record the skip relies on. +- `AsyncQueryStore` resizes a persisted LRU queue by copying it (growing no longer throws `RangeError`, shrinking evicts the dropped keys instead of stranding them, never the key being activated), and keeps a queue's persisted size for a message without `maxCount` from an older reader. `SyncQueryStore` gets the same resize. +- `onDelete` returns an unsubscribe and `QueryClient.destroy()` calls it, so a store no longer retains every client created against it. `SyncQueryStore` shares its delete listeners between every instance over the same backing kv. +- Reading a shared-typename entity array consumes the notifiers of the members it currently leaves out, so a reactive consumer learns when a member becomes eligible without a data change, and a consumer that only reads the array is not recomputed when a member it holds changes. +- Two classes sharing a typename: fields that are entity-free in one class but hold an entity through the other are no longer classified static for the snapshot fast path. +- Query extras (`__refetch`, `__hasNext`, ...) attached to a shared entity notify existing consumers; a query whose cached value was rejected fetches even within `staleTime`; snapshots of proxies created by a previous (hot-reloaded) module instance are walked instead of returned as-is. diff --git a/docs/src/app/api/fetchium-react/page.md b/docs/src/app/api/fetchium-react/page.md index 98f2c51..3bc22c9 100644 --- a/docs/src/app/api/fetchium-react/page.md +++ b/docs/src/app/api/fetchium-react/page.md @@ -132,7 +132,7 @@ const SearchResults = component(() => { - `useQuery` is a thin wrapper around Signalium v3's `useReactive`, which is deep-by-default. It returns a **structurally-shared snapshot** of the query result, so memoized children that receive subtrees as props keep stable references when the underlying data is unchanged. - Fetchium registers a custom snapshot for entity proxies so the snapshot walks into entities (instead of returning them by reference), giving you correct re-rendering on entity field changes. -- The snapshot is a plain-object copy of the query result to prevent accidental mutation of the entity cache. Use `draft()` from `fetchium` if you need a mutable copy for mutations. +- The snapshot is a plain-object copy of the query result to prevent accidental mutation of the entity cache. Treat it as read-only: in development builds the snapshot and every nested object and array in it are frozen, so an in-place `sort()`, a `push()` or an assignment in render throws a `TypeError` at that line. Production snapshots are not frozen, and a mutation there is carried forward until the entity changes. Sort a copy (`[...items].sort()`) or use `draft()` from `fetchium` if you need a mutable copy for mutations. - `useQuery` never suspends. Reading `.value` while pending returns `undefined`. Use [`useSuspenseQuery`](#usesuspensequery) to suspend on a cold miss. --- diff --git a/docs/src/app/api/fetchium/page.md b/docs/src/app/api/fetchium/page.md index bcf3f80..3d6f76e 100644 --- a/docs/src/app/api/fetchium/page.md +++ b/docs/src/app/api/fetchium/page.md @@ -241,14 +241,14 @@ new QueryClient(config: QueryClientConfig) #### Methods -| Method | Signature | Description | -| -------------------- | -------------------------------------------------- | -------------------------------------------------------------------------------------------------------- | -| `getContext` | `(): QueryContext` | Returns the `QueryContext` passed at construction. | -| `applyMutationEvent` | `(event: MutationEvent): void` | Applies an external mutation event (create/update/delete) to the entity store. | -| `invalidateQueries` | `(targets: ReadonlyArray): void` | Marks matching query instances as stale. Accepts query classes and optional param subsets for filtering. | -| `prefetch` | `(QueryClass, params?, { ttl? }?): () => void` | Starts a query now and keeps it active for `ttl` ms (default `DEFAULT_PREFETCH_TTL`, 10 s). See below. | -| `retain` | `(fn: () => unknown, { ttl? }?): () => void` | Keeps the queries `fn` reads active until the returned `release` is called, or for `ttl` ms. See below. | -| `destroy` | `(): void` | Tears down the GC manager, network manager, and all caches, and releases outstanding leases. | +| Method | Signature | Description | +| -------------------- | -------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------- | +| `getContext` | `(): QueryContext` | Returns the `QueryContext` passed at construction. | +| `applyMutationEvent` | `(event: MutationEvent): void` | Applies an external mutation event (create/update/delete) to the entity store. | +| `invalidateQueries` | `(targets: ReadonlyArray): void` | Marks matching query instances as stale. Accepts query classes and optional param subsets for filtering. | +| `prefetch` | `(QueryClass, params?, { ttl? }?): () => void` | Starts a query now and keeps it active for `ttl` ms (default `DEFAULT_PREFETCH_TTL`, 10 s). See below. | +| `retain` | `(fn: () => unknown, { ttl? }?): () => void` | Keeps the queries `fn` reads active until the returned `release` is called, or for `ttl` ms. See below. | +| `destroy` | `(): void` | Tears down the GC manager, network manager, and all caches, releases outstanding leases, and unsubscribes the store listeners it registered. | #### `prefetch` and `retain` @@ -710,8 +710,8 @@ The `t` object provides a declarative type definition DSL for describing query p ### Store types -| Type | Definition | Description | -| -------------------- | ----------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | -| `QueryStore` | Interface | Storage backend interface. See [stores/sync](/api/stores-sync) and [stores/async](/api/stores-async). | -| `CachedQuery` | `{ value: unknown; refIds: Set \| undefined; updatedAt: number; preloadedEntities?: PreloadedEntityMap }` | Cached query data returned by `QueryStore.loadQuery()`. | -| `PreloadedEntityMap` | `Map>` | Pre-loaded entity data for hydration. | +| Type | Definition | Description | +| -------------------- | ----------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `QueryStore` | Interface | Storage backend interface. See [stores/sync](/api/stores-sync) and [stores/async](/api/stores-async). Besides the required methods, a store may implement `onDelete` (the client then skips writing entities whose records are current, and writes one again once the store reports its record dropped), `onPersisted` (for a store that applies writes later), `isSettled`, `hasEntity` (lets a streamed event refresh the record of an entity that is not in memory) and `mergeEntity` (merges the fields streamed events supplied for an entity built from them over its record; without it, such an entity is written whole through `saveEntity`). A store without `onDelete` is written on every apply, as before 0.6; without `hasEntity`, an event for an entity that is not in memory and that no live array retains is not written. | +| `CachedQuery` | `{ value: unknown; refIds: Set \| undefined; updatedAt: number; preloadedEntities?: PreloadedEntityMap }` | Cached query data returned by `QueryStore.loadQuery()`. | +| `PreloadedEntityMap` | `Map>` | Pre-loaded entity data for hydration. | diff --git a/docs/src/app/core/entities/page.md b/docs/src/app/core/entities/page.md index 71187c0..11745ae 100644 --- a/docs/src/app/core/entities/page.md +++ b/docs/src/app/core/entities/page.md @@ -58,7 +58,7 @@ class GetPost extends RESTQuery { ``` {% callout type="warning" %} -Entities are **read-only**. Attempting to set a property on an entity will throw an error in development mode. To update entity data, use mutations or streaming updates. +Entities are **read-only**, and so are the snapshots `useQuery` and `useReactive` take of them. Attempting to set a property on an entity, or on a snapshot in a development build, will throw an error. To update entity data, use mutations or streaming updates. {% /callout %} --- diff --git a/docs/src/app/setup/project-setup/page.md b/docs/src/app/setup/project-setup/page.md index d90077d..b7ca2b8 100644 --- a/docs/src/app/setup/project-setup/page.md +++ b/docs/src/app/setup/project-setup/page.md @@ -289,6 +289,8 @@ Fetchium uses a compile-time constant `IS_DEV` to enable additional runtime chec - **Parse validation warnings** --- detailed logs when response data doesn't match your type definitions - **Entity write protection** --- throws when you accidentally try to mutate an entity property directly (entities are read-only) +- **Frozen snapshots** --- the snapshots `useQuery` and `useReactive` hand out are frozen, so mutating one (an in-place `sort()`, an assignment in render) throws a `TypeError` at the offending line instead of silently corrupting what the next render shows +- **Stale-snapshot detection** --- a snapshot taken at an unchanged entity version re-reads only the fields whose values live elsewhere; development builds re-read the rest too and, if one changed, serve the re-read value, log the discrepancy through the client's `log.error`, and raise it once per typename and field as an uncaught error - **Reference validation** --- catches common mistakes like using conditional logic in class field definitions In production builds, all `IS_DEV` code paths are tree-shaken, so there is zero runtime cost. diff --git a/packages/fetchium/src/ConstraintMatcher.ts b/packages/fetchium/src/ConstraintMatcher.ts index 1bb40eb..e5610aa 100644 --- a/packages/fetchium/src/ConstraintMatcher.ts +++ b/packages/fetchium/src/ConstraintMatcher.ts @@ -201,13 +201,19 @@ export class ConstraintMatcher { } } + /** + * With `dryRun`, reports through `onMatch` what the event would do (whether + * a binding matches, and whether one would retain the entity) without + * touching any collection. + */ routeEvent( typename: string, entityData: Record, entityKey: number, eventType: 'create' | 'update' | 'delete', - onMatch?: () => void, + onMatch?: (willRetain: boolean) => void, deleteData?: Record, + dryRun: boolean = false, ): void { for (const group of this._groups.values()) { const hash = computeConstraintHash(entityData, group.fieldPaths); @@ -216,7 +222,7 @@ export class ConstraintMatcher { const bindings = group.getMatching(hash); if (bindings !== undefined) { for (const binding of bindings) { - binding.onEvent(typename, entityKey, eventType, onMatch, deleteData); + binding.onEvent(typename, entityKey, eventType, onMatch, deleteData, dryRun); } } } diff --git a/packages/fetchium/src/EntityInstance.ts b/packages/fetchium/src/EntityInstance.ts index 4857df3..ee948e2 100644 --- a/packages/fetchium/src/EntityInstance.ts +++ b/packages/fetchium/src/EntityInstance.ts @@ -159,6 +159,17 @@ function snapshotCameFrom(snapshotObj: unknown, sourceId: number): boolean { return snapshotOrigins.get(snapshotObj)?.sourceId === sourceId; } +/** + * Dev builds hand out frozen snapshots. A consumer that mutates one (a + * `sort()` in render, an assignment) then fails at its own call site with a + * TypeError and a stack trace, instead of the fast path quietly carrying the + * mutation forward. Production keeps the objects mutable: freezing costs a + * call per new container and the contract is documented instead. + */ +function freezeInDev(obj: T): T { + return IS_DEV ? Object.freeze(obj) : obj; +} + function asPrevObject(prev: unknown): Record | undefined { return prev !== null && typeof prev === 'object' && !Array.isArray(prev) ? (prev as Record) @@ -198,9 +209,11 @@ function snapshotRawValue(value: unknown, prev: unknown, snap: SnapshotFn): unkn } const next = snapshotRawValue(item, prevItem, snap); result[i] = next; - if (!changed && next !== prevArr![i]) changed = true; + // `Object.is`, not `!==`: a NaN slot must not mark the array changed on + // every walk, or the snapshot never reaches a stable identity. + if (!changed && !Object.is(next, prevArr![i])) changed = true; } - return changed ? result : prevArr!; + return changed ? freezeInDev(result) : prevArr!; } if (Object.getPrototypeOf(value) === ObjectProto) { @@ -213,14 +226,68 @@ function snapshotRawValue(value: unknown, prev: unknown, snap: SnapshotFn): unkn const key = keys[i]; const next = snapshotRawValue(obj[key], prevObj?.[key], snap); result[key] = next; - if (!changed && next !== prevObj![key]) changed = true; + if (!changed && !Object.is(next, prevObj![key])) changed = true; } - return changed ? result : prevObj!; + return changed ? freezeInDev(result) : prevObj!; } return snap(value, prev); } +/** + * Structural equality of two snapshot values (plain objects, arrays, and + * primitives compared with `Object.is`). Used by the dev guard to tell a real + * drift from a value that merely lost reference identity, e.g. a `Set` a + * format parser returned, which Signalium's `snapshotSet` rebuilds each walk. + */ +function sameSnapshotValue(a: unknown, b: unknown): boolean { + if (Object.is(a, b)) return true; + if (typeof a !== 'object' || typeof b !== 'object' || a === null || b === null) return false; + if (Array.isArray(a) || Array.isArray(b)) { + if (!Array.isArray(a) || !Array.isArray(b) || a.length !== b.length) return false; + for (let i = 0; i < a.length; i++) if (!sameSnapshotValue(a[i], b[i])) return false; + return true; + } + if (a instanceof Set || b instanceof Set) { + if (!(a instanceof Set) || !(b instanceof Set) || a.size !== b.size) return false; + // A rebuilt Set (Signalium's `snapshotSet` copies one every walk) keeps + // the insertion order, so pair members by position first and only search + // for a member that fails to line up. This keeps the common case linear. + const bItems = [...b]; + // Indices of `b` already paired, tracked from the first misalignment on. + let matched: Set | undefined; + let i = 0; + for (const item of a) { + if (matched === undefined && sameSnapshotValue(item, bItems[i])) { + i++; + continue; + } + if (matched === undefined) { + matched = new Set(); + for (let k = 0; k < i; k++) matched.add(k); + } + const j = bItems.findIndex((other, idx) => !matched!.has(idx) && sameSnapshotValue(item, other)); + if (j === -1) return false; + matched.add(j); + i++; + } + return true; + } + if (a instanceof Map || b instanceof Map) { + if (!(a instanceof Map) || !(b instanceof Map) || a.size !== b.size) return false; + for (const [k, v] of a) if (!b.has(k) || !sameSnapshotValue(v, b.get(k))) return false; + return true; + } + if (Object.getPrototypeOf(a) !== ObjectProto || Object.getPrototypeOf(b) !== ObjectProto) return false; + const aKeys = Object.keys(a); + if (aKeys.length !== Object.keys(b).length) return false; + for (const key of aKeys) { + if (!Object.hasOwn(b, key)) return false; + if (!sameSnapshotValue((a as Record)[key], (b as Record)[key])) return false; + } + return true; +} + /** Dev-only counters: losing the fast path is invisible to behavioral tests. */ export let __debug_snapshotFieldReads = 0; export let __debug_snapshotFullWalks = 0; @@ -245,12 +312,44 @@ function walkFields( } } +/** + * The pre-fast-path walk: `Object.keys` on the proxy and a read per key. Kept + * for a proxy this module instance did not create — after a hot reload + * re-evaluates this file, proxies made by the previous instance are still + * live but absent from the new `snapshotSources`. + */ +function snapshotProxyByWalking(current: object, prev: unknown, snap: SnapshotFn): unknown { + const obj = current as Record; + const keys = Object.keys(obj); + const prevObj = asPrevObject(prev); + let changed = prevObj === undefined || Object.keys(prevObj).length !== keys.length; + const result: Record = {}; + for (let i = 0; i < keys.length; i++) { + const key = keys[i]; + const val = obj[key]; + const next = typeof val === 'function' ? val : snap(val, prevObj?.[key]); + result[key] = next; + if (!changed && !Object.is(next, prevObj![key])) changed = true; + } + return changed ? freezeInDev(result) : prevObj; +} + const snapshotEntity = (current: object, prev: unknown, snap: SnapshotFn): unknown => { const source = snapshotSources.get(current); - // An `Entity` that never went through `createProxy` has no fields to read — - // its own properties are the shape's type defs. Hand it back untouched, the - // way Signalium treats any class it has no handler for. - if (source === undefined) return current; + if (source === undefined) { + // A proxy from another instance of this module (hot reload re-evaluated + // this file and its registries, but the proxies are still live): walk it + // the old way rather than handing the proxy back, which would stop React + // from ever re-rendering on its changes. Every entity proxy answers + // `toJSON`; a bare `Entity` instance does not. + if (PROXY_ID.has(current) || typeof (current as { toJSON?: unknown }).toJSON === 'function') { + return snapshotProxyByWalking(current, prev, snap); + } + // An `Entity` that never went through `createProxy` has no fields to read — + // its own properties are the shape's type defs. Hand it back untouched, the + // way Signalium treats any class it has no handler for. + return current; + } // eslint-disable-next-line @typescript-eslint/no-unused-expressions source.relay?.value; @@ -276,14 +375,33 @@ const snapshotEntity = (current: object, prev: unknown, snap: SnapshotFn): unkno let result: Record | undefined; for (let i = 0; i < keys.dynamic.length; i++) { const key = keys.dynamic[i]; - if (patch[key] !== before[key]) { + if (!Object.is(patch[key], before[key])) { if (result === undefined) result = { ...before }; result[key] = patch[key]; } } - if (IS_DEV) assertStaticFieldsUnchanged(source, keys, before, snap); - return rememberSnapshot(source.id, result ?? before, version, keys); + if (IS_DEV) { + // Dev builds re-read what the fast path skipped. A field that drifted is + // served from the re-read (the pre-fast-path behaviour) and reported, + // rather than thrown: a throw from inside a snapshot lands in Signalium's + // watcher flush, which has no catch, and stalls every consumer on the page. + const drifted = verifyStaticFields(source, keys, before, snap); + if (drifted !== undefined) { + if (result === undefined) result = { ...before }; + for (const key of Object.keys(drifted)) result[key] = drifted[key]; + } + // A consumer that added or deleted keys on the snapshot it was handed: + // rebuild with the shape's keys, in the shape's order, as a walk would. + if (!sameKeyList(Object.keys(before), keys.enumerable)) { + const source_ = result ?? before; + const rebuilt: Record = {}; + for (const key of keys.enumerable) rebuilt[key] = source_[key]; + result = rebuilt; + reportDrift(source, '(keys)', 'its key set was altered'); + } + } + return rememberSnapshot(source.id, result !== undefined ? freezeInDev(result) : before, version, keys); } if (IS_DEV) __debug_snapshotFullWalks++; @@ -294,47 +412,97 @@ const snapshotEntity = (current: object, prev: unknown, snap: SnapshotFn): unkno let changed = prevObj === undefined || Object.keys(prevObj).length !== enumerable.length; if (!changed) { for (let i = 0; i < enumerable.length; i++) { - if (result[enumerable[i]] !== prevObj![enumerable[i]]) { + if (!Object.is(result[enumerable[i]], prevObj![enumerable[i]])) { changed = true; break; } } } - return rememberSnapshot(source.id, changed ? result : prevObj!, version, keys); + return rememberSnapshot(source.id, changed ? freezeInDev(result) : prevObj!, version, keys); }; +function sameKeyList(a: string[], b: string[]): boolean { + if (a.length !== b.length) return false; + for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false; + return true; +} + +/** Dev-only: (typename, field) pairs already reported, so a drift is raised once. */ +const reportedDrift = new Set(); + +export type SnapshotDriftHandler = (error: Error, queryClient: QueryClient) => void; + /** - * Guards the fast path's assumptions: `data` only changes through a `notify()` + * Dev-only default: log through the client's logger, then raise the error + * outside the snapshot so it surfaces as an uncaught error (a red box in + * React Native, "Uncaught Error" in a browser, a failed run under vitest) + * without throwing from inside Signalium's watcher flush, which has no catch + * and would stop every consumer on the page from updating. + */ +const defaultDriftHandler: SnapshotDriftHandler = (error, queryClient) => { + queryClient.getContext().log?.error?.(error.message, error); + queueMicrotask(() => { + throw error; + }); +}; + +let snapshotDriftHandler: SnapshotDriftHandler = defaultDriftHandler; + +/** + * Dev-only test hook: replace how a stale snapshot is raised, and forget which + * fields were already raised. `undefined` restores the default handler. + */ +export function __setSnapshotDriftHandler(handler: SnapshotDriftHandler | undefined): void { + snapshotDriftHandler = handler ?? defaultDriftHandler; + reportedDrift.clear(); +} + +function reportDrift(source: EntitySnapshotSource, key: string, what: string): void { + const tag = `${source.instance.typename}.${key}`; + if (reportedDrift.has(tag)) return; + reportedDrift.add(tag); + const error = new Error( + `[fetchium] stale entity snapshot: ${source.instance.typename}:${source.instance.id} ${what} while version ` + + `stayed at ${source.instance.version}. The re-read value is being served, but a production build would keep ` + + `the stale one. Either the field was mutated without notify(), the shape's static/dynamic split is wrong ` + + `for it, or a consumer mutated a snapshot it received (dev snapshots are frozen so that fails at the call ` + + `site; in production they are not). Raised once per field.`, + ); + snapshotDriftHandler(error, source.instance._queryClient); +} + +/** + * Checks the fast path's assumptions: `data` only changes through a `notify()` * that bumps `version`, and every field outside `keys.dynamic` is a pure - * function of `data`. Either breaking strands a snapshot permanently, which no - * behavioral test catches. Skipped fields only — re-reading a dynamic field is - * not identity-stable, since snapshotting a child updates state it pairs on. + * function of `data`. Re-reads the skipped fields and returns the ones whose + * value differs structurally from the cached snapshot (identity alone is not + * enough: a `Set` from a format parser is rebuilt on every walk), or + * `undefined` when nothing drifted. Skipped fields only — re-reading a dynamic + * field is not identity-stable, since snapshotting a child updates state it + * pairs on. * * This costs dev builds the read the fast path saved, so the speedup shows up * in production builds only. */ -function assertStaticFieldsUnchanged( +function verifyStaticFields( source: EntitySnapshotSource, keys: EntityKeys, before: Record, snap: SnapshotFn, -): void { +): Record | undefined { const verified: Record = {}; const readsBefore = __debug_snapshotFieldReads; walkFields(source, keys.static, before, snap, verified); // This walk is verification, not work the fast path did. __debug_snapshotFieldReads = readsBefore; + let drifted: Record | undefined; for (const key of keys.static) { - if (verified[key] !== before[key]) { - throw new Error( - `[fetchium] stale entity snapshot: ${source.instance.typename}:${source.instance.id} field '${key}' ` + - `changed while version stayed at ${source.instance.version}. Either the field was mutated without ` + - `notify(), or the shape's static/dynamic split is wrong for it. This throws from inside a snapshot, ` + - `so the reactive computation holding it stays in an error state and will rethrow on every read — ` + - `reload rather than chasing the later throws, which are all this one.`, - ); - } + if (Object.is(verified[key], before[key]) || sameSnapshotValue(verified[key], before[key])) continue; + if (drifted === undefined) drifted = {}; + drifted[key] = verified[key]; + reportDrift(source, key, `field '${key}' changed`); } + return drifted; } // Register once on the `Entity` base class. Signalium 3.0.1+ resolves custom @@ -361,6 +529,23 @@ export class EntityInstance { refCount: number = 0; /** Whether this instance has been written to the store. */ _persisted: boolean = false; + /** Writes handed to a store that acknowledges them, not yet acknowledged. */ + _pendingWrites: number = 0; + /** Set while an apply is still reifying this instance's fields; its data is not yet a record. */ + _applying: boolean = false; + /** + * The data came from streamed events only (the entity was created by one), + * so it may lack fields a record of it on disk has: a write merges over + * that record. Cleared by the first full payload applied to the entity. + */ + _partial: boolean = false; + /** With `_partial`: the fields the events carried, i.e. the ones a write merges. */ + _partialKeys: Set | undefined = undefined; + /** A write requested while a child was still being reified; performed when that apply is done. */ + _deferredWrite: boolean = false; + /** The store is known to hold a record of this entity (hydrated from it, or written to it). */ + _recorded: boolean = false; + private _saving: boolean = false; entityRefs: Map | undefined; liveCollections: LiveCollectionBinding[] = []; satisfiedDefs: WeakSet> = new WeakSet(); @@ -484,8 +669,86 @@ export class EntityInstance { } save(): void { - this._persisted = true; - this._queryClient.entityMap.save(this); + const client = this._queryClient; + if (this._saving) return; + this._saving = true; + try { + // A record's references must point at records that exist: a child the + // store holds no record of (never written, dropped, or a failed write) + // is written first. While an apply is still reifying a child (a payload + // that links back to an ancestor), neither it nor this entity can be + // written yet: both are written, in that order, once the apply is done. + const refs = this.entityRefs; + if (refs !== undefined) { + let deferred = false; + for (const child of refs.keys()) { + if (child._recorded || child._persisted || child._pendingWrites > 0 || child._saving) continue; + if (client.entityMap.getEntity(child.key) !== child) continue; + if (child._applying) { + client.deferWrite(child); + deferred = true; + } else { + try { + child.save(); + } catch (e) { + // This entity's record (if any) still lacks the reference: the + // next apply must write it again. + this._persisted = false; + throw e; + } + if (child._deferredWrite) deferred = true; + } + } + if (deferred) { + this._deferredWrite = true; + client.deferWrite(this); + return; + } + } + this._deferredWrite = false; + // Mark after the store call: a store that throws leaves the record stale + // or missing, so the flag is cleared and the next apply writes again. A + // store that acknowledges writes later (`onPersisted`) marks it itself + // once the write has been processed. An entity built from events hands + // the store only the fields they carried, to merge over its record. + // Counted before the call: a store may acknowledge synchronously. + if (client.storeAcksWrites) this._pendingWrites++; + try { + client.entityMap.save(this, this._partial ? this._partialKeys : undefined); + } catch (e) { + if (client.storeAcksWrites && this._pendingWrites > 0) this._pendingWrites--; + this._persisted = false; + throw e; + } + if (!client.storeAcksWrites) this._persisted = this._recorded = true; + } finally { + this._saving = false; + } + } + + /** A live field of this entity gained or lost a member outside an apply. */ + liveFieldChanged(fieldKey: string): void { + if (this._partial) this._partialKeys?.add(fieldKey); + } + + /** + * The store processed a write of this entity's record. Only a write this + * instance dispatched counts: an acknowledgement can also belong to a write + * queued by a previous instance of the same entity (one that was collected + * and re-hydrated from the store while the write was in flight), and that + * write carried a value this instance never had. The record is current + * once every dispatched write has landed. + */ + acknowledgeWrite(): void { + if (this._pendingWrites === 0) return; + if (--this._pendingWrites === 0) this._persisted = this._recorded = true; + } + + /** The store dropped (or failed to write) this entity's record. */ + recordDropped(): void { + this._persisted = false; + this._recorded = false; + this._pendingWrites = 0; } notify(): void { @@ -504,18 +767,31 @@ function sameMembers(a: unknown[], b: unknown[]): boolean { return true; } -function filterEntityArray(array: unknown[], innerDef: ValidatorDef, queryClient: QueryClient): unknown[] { - const result: unknown[] = []; +interface NarrowedArray { + source: unknown[]; + filtered: unknown[]; + /** Members that do not (yet) satisfy the def; the only ones whose membership can change. */ + excluded: EntityInstance[]; + parseId: number; +} + +function filterEntityArray( + array: unknown[], + innerDef: ValidatorDef, + queryClient: QueryClient, +): { filtered: unknown[]; excluded: EntityInstance[] } { + const filtered: unknown[] = []; + const excluded: EntityInstance[] = []; for (const item of array) { if (typeof item !== 'object' || item === null) continue; const entityKey = PROXY_ID.get(item); if (entityKey === undefined) continue; const entityInstance = queryClient.entityMap.getEntity(entityKey); - if (entityInstance !== undefined && entityInstance.satisfiesDef(innerDef)) { - result.push(item); - } + if (entityInstance === undefined) continue; + if (entityInstance.satisfiesDef(innerDef)) filtered.push(item); + else excluded.push(entityInstance); } - return result; + return { filtered, excluded }; } // ====================================================== @@ -587,7 +863,7 @@ function computeIsStaticFieldDef(def: ValidatorDef): boolean { * The key lists a proxy reports, shared per shape. `own` is `ownKeys`; * `enumerable` drops the non-enumerable entity methods. */ -interface EntityKeys { +export interface EntityKeys { own: string[]; enumerable: string[]; dynamic: string[]; @@ -596,14 +872,29 @@ interface EntityKeys { enumerableSet: Set; } -const shapeKeyCache = new WeakMap, EntityKeys>(); +/** + * A field whose def carries no shape: a primitive, a format, `t.typename`, or + * a set of literals. Its value can only be a primitive or a format's parsed + * value, so no other class sharing the typename can put an entity under it. + */ +function isShapelessFieldDef(def: unknown): boolean { + if (typeof def === 'number' || typeof def === 'string') return true; + if (def instanceof Set) return true; + return def instanceof ValidatorDef && (def.shape === undefined || def.shape === null); +} +/** + * Cached per client (`queryClient.shapeKeyCache`): the split depends on which + * other classes share the typename, and that set grows as queries register. + */ function shapeKeys( validatorDef: ValidatorDef, shapeFields: Record, methods: Record unknown> | undefined, + queryClient: QueryClient, ): EntityKeys { - let keys = shapeKeyCache.get(validatorDef); + const cache = queryClient.shapeKeyCache; + let keys = cache.get(validatorDef); if (keys !== undefined) return keys; const own = Object.keys(shapeFields); @@ -614,7 +905,16 @@ function shapeKeys( if (!own.includes(methodKey)) own.push(methodKey); } } - const dynamic = enumerable.filter(key => key !== '__typename' && !isStaticFieldDef(shapeFields[key])); + // The data object is shared by every class with this typename, so a nested + // object that is entity-free in this class may hold a child entity written + // through another class's shape. Only shapeless fields stay static then. + const typename = validatorDef.typenameValue; + const shared = typename !== undefined && (queryClient.getEntityDefsForTypename(typename)?.length ?? 0) > 1; + const dynamic = enumerable.filter( + key => + key !== '__typename' && + (!isStaticFieldDef(shapeFields[key]) || (shared && !isShapelessFieldDef(shapeFields[key]))), + ); const dynamicSet = new Set(dynamic); keys = { @@ -624,7 +924,7 @@ function shapeKeys( static: enumerable.filter(key => !dynamicSet.has(key)), enumerableSet: new Set(enumerable), }; - shapeKeyCache.set(validatorDef, keys); + cache.set(validatorDef, keys); return keys; } @@ -675,12 +975,24 @@ function bindMethod( return bound; } +/** + * The members a narrowed array leaves out are dependencies of the read: one + * of them gaining the def's fields notifies itself, and that is what must + * recompute the consumer. Members already in the array are read through the + * array by any consumer that cares about their fields, and a member never + * loses eligibility, so consuming them here would only add recomputes (and + * cost a lookup per member on every read of the field). + */ +function consumeExcluded(excluded: EntityInstance[]): void { + for (let i = 0; i < excluded.length; i++) excluded[i].consume(); +} + /** Narrow a shared-typename array to this field's def, cached on array identity. */ function narrowEntityArray( prop: string, value: unknown[], shapeFields: Record, - filterCache: Map, + filterCache: Map, queryClient: QueryClient, ): unknown[] { const fieldDef = shapeFields[prop]; @@ -695,11 +1007,14 @@ function narrowEntityArray( const parseId = queryClient.currentParseId; const cached = filterCache.get(prop); if (cached !== undefined && cached.source === value && cached.parseId === parseId) { + consumeExcluded(cached.excluded); return cached.filtered; } - let filtered = filterEntityArray(value, innerDef, queryClient); + const narrowed = filterEntityArray(value, innerDef, queryClient); + let filtered = narrowed.filtered; if (cached !== undefined && sameMembers(cached.filtered, filtered)) filtered = cached.filtered; - filterCache.set(prop, { source: value, filtered, parseId }); + filterCache.set(prop, { source: value, filtered, excluded: narrowed.excluded, parseId }); + consumeExcluded(narrowed.excluded); return filtered; } } @@ -728,7 +1043,7 @@ function createProxy( const typenameField = shape.typenameField; const wrappedMethods = new Map unknown>(); - const filterCache = new Map(); + const filterCache = new Map(); const toJSON = () => ({ __entityRef: key }); @@ -755,22 +1070,24 @@ function createProxy( } // Shared per shape; only a query's root entity allocates its own lists. - const baseKeys = shapeKeys(validatorDef, shapeFields, methods); + let cachedBase: EntityKeys | undefined; let cachedMethods: Record | undefined; let cachedGetters: Record | undefined; - let keys = baseKeys; + let keys: EntityKeys | undefined; function entityKeys(): EntityKeys { - // Both slots: these lists decide what a snapshot walks and re-reads. + // The base is re-resolved on every call (a WeakMap lookup): the client + // resets its cache when another class registers for this typename, and + // the new split must reach snapshots already in flight. Both extras + // slots matter too: these lists decide what a snapshot walks and re-reads. + const base = shapeKeys(validatorDef, shapeFields, methods, queryClient); const methodsNow = instance._extraMethods; const gettersNow = instance._extraGetters; - if (methodsNow !== cachedMethods || gettersNow !== cachedGetters) { + if (keys === undefined || base !== cachedBase || methodsNow !== cachedMethods || gettersNow !== cachedGetters) { + cachedBase = base; cachedMethods = methodsNow; cachedGetters = gettersNow; - keys = - methodsNow === undefined && gettersNow === undefined - ? baseKeys - : withExtraKeys(baseKeys, methodsNow, gettersNow); + keys = methodsNow === undefined && gettersNow === undefined ? base : withExtraKeys(base, methodsNow, gettersNow); } return keys; } diff --git a/packages/fetchium/src/EntityStore.ts b/packages/fetchium/src/EntityStore.ts index 89e0b31..12b8fb8 100644 --- a/packages/fetchium/src/EntityStore.ts +++ b/packages/fetchium/src/EntityStore.ts @@ -5,9 +5,13 @@ import { ValidatorDef } from './typeDefs.js'; export class EntityStore { private instances = new Map(); - private persistEntity: (key: number, data: Record, refKeys?: Set) => void; + private persistEntity: (key: number, data: Record, refKeys?: Set, merge?: boolean) => void; + /** Whether the store can merge fields over a record it holds. */ + mergesEntities: boolean = false; - constructor(persistEntity: (key: number, data: Record, refKeys?: Set) => void) { + constructor( + persistEntity: (key: number, data: Record, refKeys?: Set, merge?: boolean) => void, + ) { this.persistEntity = persistEntity; } @@ -58,12 +62,24 @@ export class EntityStore { this.instances.clear(); } - save(instance: EntityInstance): void { + /** + * With `mergeKeys`, only those fields are handed to the store, to be merged + * over the record it holds (the rest of the instance's data is what streamed + * events left unset, not what the record says). + */ + save(instance: EntityInstance, mergeKeys?: Set): void { let refKeys: Set | undefined; if (instance.entityRefs) { refKeys = new Set(); - for (const e of instance.entityRefs.keys()) refKeys.add(e.key); + // A child evicted from memory has no record to reference. + for (const e of instance.entityRefs.keys()) if (this.instances.get(e.key) === e) refKeys.add(e.key); } - this.persistEntity(instance.key, instance.data, refKeys); + let value = instance.data; + const merge = mergeKeys !== undefined && this.mergesEntities; + if (merge) { + value = {}; + for (const k of mergeKeys) value[k] = instance.data[k]; + } + this.persistEntity(instance.key, value, refKeys, merge); } } diff --git a/packages/fetchium/src/LiveCollection.ts b/packages/fetchium/src/LiveCollection.ts index 673c54a..b85fbcd 100644 --- a/packages/fetchium/src/LiveCollection.ts +++ b/packages/fetchium/src/LiveCollection.ts @@ -66,6 +66,8 @@ export interface LiveCollectionParent { addChildRef(child: EntityInstance, persist?: boolean): void; removeChildRef(child: EntityInstance, persist?: boolean): void; save(): void; + /** A live field's membership changed outside an apply (an event was routed into it). */ + liveFieldChanged?(fieldKey: string): void; } // ====================================================== @@ -153,8 +155,9 @@ export class LiveCollectionBinding { typename: string, entityKey: number, eventType: 'create' | 'update' | 'delete', - onMatch?: () => void, + onMatch?: (willRetain: boolean) => void, deleteData?: Record, + dryRun: boolean = false, ): void { const defs = this._entityDefsByTypename.get(typename); if (defs === undefined) return; @@ -175,8 +178,8 @@ export class LiveCollectionBinding { } const entity = entityInstance !== undefined ? entityInstance.getProxy(def as unknown as EntityDef) : deleteData; if (entity !== undefined) { - this.instance.onEvent(entityKey, entity, deleteData ?? entityInstance?.data ?? {}, 'delete'); - onMatch?.(); + if (!dryRun) this.instance.onEvent(entityKey, entity, deleteData ?? entityInstance?.data ?? {}, 'delete'); + onMatch?.(false); } return; } @@ -189,7 +192,9 @@ export class LiveCollectionBinding { if (def === undefined) return; if (!entityInstance.satisfiesDef(def as unknown as ValidatorDef)) return; - onMatch?.(); + // A live array retains what it adds; a live value only reduces over it. + onMatch?.(this.instance instanceof LiveArrayInstance); + if (dryRun) return; const proxy = entityInstance.getProxy(def as unknown as EntityDef); this.instance.onEvent(entityKey, proxy, entityInstance.data, eventType); } @@ -214,6 +219,8 @@ export class LiveArrayInstance { _queryClient: QueryClient; _parent: LiveCollectionParent; + _fieldKey: string | undefined; + constructor( queryClient: QueryClient, parent: LiveCollectionParent, @@ -221,12 +228,14 @@ export class LiveArrayInstance { constraintFieldPaths?: FieldPath[], constraintHash?: number, sort?: (a: unknown, b: unknown) => number, + fieldKey?: string, ) { this._notifier = notifier(); this._items = items; this._keys = buildKeySet(items); this._queryClient = queryClient; this._parent = parent; + this._fieldKey = fieldKey; const needsFilter = constraintFieldPaths !== undefined && constraintHash !== undefined; const needsSort = sort !== undefined; @@ -309,6 +318,7 @@ export class LiveArrayInstance { this._keys.add(key); this._items.push(proxy); + if (this._fieldKey !== undefined) this._parent.liveFieldChanged?.(this._fieldKey); const child = this._queryClient.entityMap.getEntity(key); if (child !== undefined) { // Only the parent's ref set needs persisting; the child's record is current. @@ -325,6 +335,7 @@ export class LiveArrayInstance { const idx = this._findIndex(key); if (idx !== -1) this._items.splice(idx, 1); + if (this._fieldKey !== undefined) this._parent.liveFieldChanged?.(this._fieldKey); const child = this._queryClient.entityMap.getEntity(key); if (child !== undefined) { this._parent.removeChildRef(child); @@ -498,6 +509,7 @@ export function createLiveCollection( parent: LiveCollectionParent, parentData: Record, queryClient: QueryClient, + fieldKey?: string, ): LiveCollectionBinding { let constraintFieldRefs = config.constraintFieldRefs; @@ -540,6 +552,7 @@ export function createLiveCollection( arrayConstraintFieldPaths, arrayConstraintHash, config.sort, + fieldKey, ); } else { inner = new LiveValueInstance( @@ -597,7 +610,7 @@ export function initializeLiveFields( } data[fieldName] = existing; } else { - data[fieldName] = createLiveCollection(config, data[fieldName], parent, data, queryClient); + data[fieldName] = createLiveCollection(config, data[fieldName], parent, data, queryClient, fieldName); } } else if ( (fieldDef.mask & Mask.OBJECT) !== 0 && diff --git a/packages/fetchium/src/QueryClient.ts b/packages/fetchium/src/QueryClient.ts index 82dc8b0..f7a7626 100644 --- a/packages/fetchium/src/QueryClient.ts +++ b/packages/fetchium/src/QueryClient.ts @@ -11,7 +11,7 @@ import { } from './types.js'; import { PROXY_ID } from './proxyId.js'; import { EntityStore } from './EntityStore.js'; -import { EntityInstance } from './EntityInstance.js'; +import { EntityInstance, type EntityKeys } from './EntityInstance.js'; import { NetworkManager, NoOpNetworkManager } from './NetworkManager.js'; import { QueryInstance } from './QueryResult.js'; import { MutationResultImpl } from './MutationResult.js'; @@ -168,6 +168,14 @@ export class QueryClient { currentParseId: number = 0; /** Without `store.onDelete`, `_persisted` cannot be trusted. */ storeReportsDeletes: boolean = false; + /** With `store.onPersisted`, `_persisted` is set by the store's acknowledgement, not by `save()`. */ + storeAcksWrites: boolean = false; + /** + * Per-client static/dynamic key split for each entity shape (see + * `shapeKeys` in EntityInstance.ts). Replaced wholesale when a typename + * gains a second class, since the split depends on the other classes. + */ + shapeKeyCache = new WeakMap, EntityKeys>(); /** See `QueryClientConfig.reactivationGraceMs`. */ readonly reactivationGraceMs: number; @@ -203,6 +211,7 @@ export class QueryClient { private mergedDefCache = new Map>(); private adapters = new Map(); private networkUnsubscribe: (() => void) | undefined; + private storeUnsubscribes: Array<() => void> = []; private mutationParseContext = new ParseContext(); constructor(config: QueryClientConfig = {}) { @@ -229,13 +238,23 @@ export class QueryClient { config.gcManager ?? (this.isServer ? new NoOpGcManager() : new GcManager(this.handleEviction, evictionMultiplier)); this.networkManager = config.networkManager ?? new NetworkManager(); - this.entityMap = new EntityStore((key, data, refs) => this.store.saveEntity(key, data, refs)); + this.entityMap = new EntityStore((key, data, refs, merge) => { + if (merge) this.store.mergeEntity!(key, data, refs); + else this.store.saveEntity(key, data, refs); + }); + this.entityMap.mergesEntities = typeof this.store.mergeEntity === 'function'; // A record the store drops must be written again by the next apply. this.storeReportsDeletes = typeof this.store.onDelete === 'function'; - this.store.onDelete?.(key => { - const entity = this.entityMap.getEntity(key); - if (entity !== undefined) entity._persisted = false; + const offDelete = this.store.onDelete?.(key => { + this.entityMap.getEntity(key)?.recordDropped(); + }); + if (typeof offDelete === 'function') this.storeUnsubscribes.push(offDelete); + // A store that processes writes later says when a record is really there. + this.storeAcksWrites = typeof this.store.onPersisted === 'function'; + const offPersisted = this.store.onPersisted?.(key => { + this.entityMap.getEntity(key)?.acknowledgeWrite(); }); + if (typeof offPersisted === 'function') this.storeUnsubscribes.push(offPersisted); // Register user-supplied adapters for (const adapter of config.adapters ?? []) { @@ -322,6 +341,15 @@ export class QueryClient { // Typename Registry (per-client) // ====================================================== + /** + * Whether every write handed to the store has been processed, so a skipped + * write cannot be undone by a deletion the store has queued but not yet run. + * A synchronous store is always settled. + */ + storeIsSettled(): boolean { + return this.store.isSettled?.() ?? true; + } + private registerEntityDef(def: ValidatorDef): void { const typename = def.typenameValue; if (typename === undefined) return; @@ -335,6 +363,9 @@ export class QueryClient { existing.push(def); this.mergedDefCache.delete(typename); this.getMergedDef(typename); + // The snapshot fast path's static/dynamic split for this typename's + // classes now has to account for the new class's fields. + this.shapeKeyCache = new WeakMap(); } else { this.typenameRegistry.set(typename, [def]); } @@ -719,7 +750,8 @@ export class QueryClient { obj !== null && !('__entityRef' in (obj as Record)) ) { - (obj as Record)[QUERY_ID] = queryId; + // On a copy: the payload belongs to the adapter (it may be frozen). + obj = { ...(obj as Record), [QUERY_ID]: queryId }; } const parseResult = this.parseData(obj, rootEntityShape as unknown as InternalTypeDef, preloadedEntities); @@ -770,13 +802,16 @@ export class QueryClient { return; } + // Entities the event creates are not written by the apply; entities it + // merely updates are. A created entity is written once a written record + // references it, or below, once the root is known to be retained. + const created = new Set(); try { const warn = this.context.log?.warn ?? (() => {}); const parseCtx = this.mutationParseContext; parseCtx.reset(this, undefined, warn, /* isPartialEvent */ true); const parsedData = parseEntity(data, mergedDef as unknown as EntityDef, parseCtx); - // A new entity is written only once something routes it. - applyEntityRefs(parseCtx, parsedData, /* persist */ existing !== undefined); + applyEntityRefs(parseCtx, parsedData, existing !== undefined ? true : 'existing', false, created); } catch (e) { // Unknown required-union variant: surface as an error (not a silent warn) // so the dropped update is visible. Optional unions degrade during parse. @@ -786,8 +821,9 @@ export class QueryClient { this.context.log?.warn?.('Failed to apply mutation event', e); } if (existing === undefined) { - const created = this.entityMap.getEntity(key); - if (created !== undefined) created.evict(); + // Half applied: nothing built from it can stay. + const createdRoot = this.entityMap.getEntity(key); + if (createdRoot !== undefined) this.evictCreated(createdRoot, created); } return; } @@ -795,19 +831,138 @@ export class QueryClient { const entity = this.entityMap.getEntity(key); if (entity === undefined) return; - const wasNew = existing === undefined; + if (existing !== undefined) { + this.routeEvent(typename, entity.data, key, type, eventSource); + return; + } + + // The root the event created is written, with the entities it created + // under it, when a live array is about to retain it (the array's owner + // is written with the reference) or when the store already holds a record + // of it: a query that was collected from memory can still hold this + // entity in its cache, and the event is that record's only chance to stay + // fresh. That write merges the event's fields over the record, since an + // event is a partial update and the fields it leaves out are not known + // here. The write comes before the routing: if it fails, nothing is + // routed, the failure is logged, and memory stays as it was, as a failed + // event always did. A store that cannot say what it holds (no + // `hasEntity`, or not yet) is not written, so it never accumulates + // records nothing references. Anything else the event created was only + // ever referenced by an unwritten root and stays unwritten. let matched = false; + let retains = false; + this.routeEvent( + typename, + entity.data, + key, + type, + eventSource, + willRetain => { + matched = true; + if (willRetain) retains = true; + }, + undefined, + /* dryRun */ true, + ); + const held = this.store.hasEntity?.(key) === true; + if ((retains || held) && !entity._persisted && entity._pendingWrites === 0) { + try { + entity.save(); + } catch (e) { + this.context.log?.warn?.('Failed to apply mutation event', e); + this.evictUnlessAdopted(entity, created); + return; + } + } - this.routeEvent(typename, entity.data, key, type, eventSource, () => { - matched = true; - }); + this.routeEvent(typename, entity.data, key, type, eventSource); - if (wasNew) { - if (matched) { - // The parent persisted a ref to it, so its record must exist. - persistUnwritten(entity); - } else { - entity.evict(); + if (!matched) this.evictUnlessAdopted(entity, created); + } + + /** + * Evicts the root an event created unless an entity that existed before + * the event now references it (its owner's data holds the proxy, so it is + * live). References from entities the same event created, the root itself + * included, do not count: they were only ever reachable through the root. + */ + private evictUnlessAdopted(root: EntityInstance, created: Set): void { + let createdHolders = 0; + for (const c of created) if (c.entityRefs?.has(root)) createdHolders++; + if (root.refCount <= createdHolders) this.evictCreated(root, created); + } + + /** + * Evicts the root an event created, and with it every entity the event + * created that nothing references any more: they were never written, so + * lingering until a `gcTime` runs out would let a later event write them + * as records nothing references. + */ + private evictCreated(root: EntityInstance, created: Set): void { + root.evict(); + // Evicting one releases what it held; repeat until nothing more is free. + let evicted = true; + while (evicted) { + evicted = false; + for (const c of created) { + if (c !== root && c.refCount === 0 && this.entityMap.getEntity(c.key) === c) { + c.evict(); + evicted = true; + } + } + } + } + + /** + * Entities whose write was requested while an apply was still reifying + * them; written once that apply is done. + */ + private deferredWrites: Set | undefined; + + /** @internal */ + deferWrite(entity: EntityInstance): void { + (this.deferredWrites ??= new Set()).add(entity); + } + + /** + * Writes the deferred entities in the order they were deferred: children + * before their parents. A write that fails leaves the rest unwritten, and + * marked so that the next apply writes them. + */ + /** @internal */ + flushDeferredWrites(): void { + const deferred = this.deferredWrites; + if (deferred === undefined || deferred.size === 0) return; + this.deferredWrites = undefined; + try { + for (const entity of deferred) { + if (entity._applying || this.entityMap.getEntity(entity.key) !== entity) continue; + if (entity._deferredWrite || (!entity._persisted && entity._pendingWrites === 0)) entity.save(); + } + } finally { + for (const entity of deferred) { + if (entity._deferredWrite) { + entity._deferredWrite = false; + entity._persisted = false; + } + } + } + } + + /** + * An apply that failed leaves nothing to write: what it deferred may be half + * reified. The records that were waiting are stale, so the next apply + * writes them. + */ + /** @internal */ + discardDeferredWrites(): void { + const deferred = this.deferredWrites; + if (deferred === undefined) return; + this.deferredWrites = undefined; + for (const entity of deferred) { + if (entity._deferredWrite) { + entity._deferredWrite = false; + entity._persisted = false; } } } @@ -931,14 +1086,15 @@ export class QueryClient { entityKey: number, eventType: 'create' | 'update' | 'delete', eventSource: number | undefined, - onMatch?: () => void, + onMatch?: (willRetain: boolean) => void, deleteData?: Record, + dryRun: boolean = false, ): void { const matcher = this.constraintRegistry.get(typename); if (matcher === undefined) return; const data = eventSource !== undefined ? { ...entityData, [EVENT_SOURCE_FIELD]: eventSource } : entityData; - matcher.routeEvent(typename, data, entityKey, eventType, onMatch, deleteData); + matcher.routeEvent(typename, data, entityKey, eventType, onMatch, deleteData, dryRun); } destroy(): void { @@ -948,6 +1104,8 @@ export class QueryClient { this.staggerTimer = undefined; this.staggerQueue.clear(); this.networkUnsubscribe?.(); + for (const off of this.storeUnsubscribes) off(); + this.storeUnsubscribes = []; this.gcManager.destroy(); this.networkManager.destroy(); for (const adapter of this.adapters.values()) { @@ -1014,13 +1172,3 @@ function paramsMatch(instanceParams: Record | undefined, subset } return true; } - -/** Writes an entity and its not-yet-written descendants. */ -function persistUnwritten(entity: EntityInstance): void { - if (!entity._persisted) entity.save(); - const refs = entity.entityRefs; - if (refs === undefined) return; - for (const child of refs.keys()) { - if (!child._persisted) persistUnwritten(child); - } -} diff --git a/packages/fetchium/src/QueryResult.ts b/packages/fetchium/src/QueryResult.ts index cf232e4..3c2ea52 100644 --- a/packages/fetchium/src/QueryResult.ts +++ b/packages/fetchium/src/QueryResult.ts @@ -271,6 +271,10 @@ export class QueryInstance { __hasNext: () => this.hasNext, __isFetchingNext: () => this._fetchNextPromise !== undefined, }; + // The entity's key set just changed. A shared entity (a query whose + // result is an entity) may already have consumers, and an apply that + // found its data unchanged will not have notified them. + this.rootEntity.notify(); } return this.rootEntity.getProxy(def.statics.shape as unknown as EntityDef) as QueryResult; diff --git a/packages/fetchium/src/__tests__/entity-snapshot.test.ts b/packages/fetchium/src/__tests__/entity-snapshot.test.ts index 9b52c99..04ced66 100644 --- a/packages/fetchium/src/__tests__/entity-snapshot.test.ts +++ b/packages/fetchium/src/__tests__/entity-snapshot.test.ts @@ -2,6 +2,7 @@ import { describe, it, expect } from 'vitest'; import { watcher, withContexts } from 'signalium'; import { hashValue, snapshot } from 'signalium/utils'; import { + __setSnapshotDriftHandler, __debug_resetSnapshotCounters, __debug_snapshotFieldReads, __debug_snapshotFullWalks, @@ -615,7 +616,7 @@ describe('Entity Snapshots', () => { expect(after.board.slots[1]).toBe(before.board.slots[2]); }); - it('throws in dev when a field changes without the version moving', async () => { + it('raises in dev when a field changes without the version moving, and serves the re-read value', async () => { const { client, mockFetch } = getClient(); mockFetch.get('/portfolio', portfolio(2)); @@ -630,15 +631,49 @@ describe('Entity Snapshots', () => { const instance = client.entityMap.getEntity(hashValue(['Token', 'tok-0']))!; const original = instance.data.symbol; instance.data.symbol = 'MUTATED'; - client.applyMutationEvent({ type: 'update', typename: 'Token', data: { id: 'tok-1', price: 5 } }); + const raised: Error[] = []; + __setSnapshotDriftHandler(error => raised.push(error)); + try { + client.applyMutationEvent({ type: 'update', typename: 'Token', data: { id: 'tok-1', price: 5 } }); + + // The guard must not throw from inside the snapshot: that escapes + // Signalium's watcher flush and stalls every consumer on the page. It + // raises the error out of band (by default as an uncaught error, through + // the client's logger) and hands out what a full walk would have produced. + const snap = read() as unknown as { tokens: { symbol: string }[] }; + expect(snap.tokens[0].symbol).toBe('MUTATED'); + expect(raised).toHaveLength(1); + expect(raised[0].message).toMatch(/stale entity snapshot: Token:tok-0 field 'symbol'/); + + // Raised once per field: a second recompute stays quiet. + client.applyMutationEvent({ type: 'update', typename: 'Token', data: { id: 'tok-1', price: 6 } }); + read(); + expect(raised).toHaveLength(1); + } finally { + __setSnapshotDriftHandler(undefined); + instance.data.symbol = original; + instance.notify(); + read(); + } + }); - expect(() => read()).toThrow(/stale entity snapshot/); + it('hands out frozen snapshots in dev, so a consumer mutation fails at its call site', async () => { + const { client, mockFetch } = getClient(); + mockFetch.get('/portfolio', portfolio(2)); - // Put the entity back and let the version move, so the watcher's own - // recompute doesn't rethrow after the test has finished. - instance.data.symbol = original; - instance.notify(); - read(); + const { query, read } = snapshotHarness(client, () => fetchQuery(GetPortfolio)); + await query; + const snap = read() as unknown as { tokens: { symbol: string }[] }; + + expect(Object.isFrozen(snap)).toBe(true); + expect(Object.isFrozen(snap.tokens)).toBe(true); + expect(Object.isFrozen(snap.tokens[0])).toBe(true); + // The classic "sort in render" bug throws where it happens, not later. + expect(() => snap.tokens.sort()).toThrow(TypeError); + expect(() => { + snap.tokens[0].symbol = 'X'; + }).toThrow(TypeError); + expect(snap.tokens[0].symbol).not.toBe('X'); }); it('re-snapshots a union field whose only entity-bearing member is an array', async () => { diff --git a/packages/fetchium/src/__tests__/entity-write-counts.test.ts b/packages/fetchium/src/__tests__/entity-write-counts.test.ts index f9c93c2..8d67268 100644 --- a/packages/fetchium/src/__tests__/entity-write-counts.test.ts +++ b/packages/fetchium/src/__tests__/entity-write-counts.test.ts @@ -40,8 +40,19 @@ class GetList extends RESTQuery { result = { list: t.entity(List) }; } -function itemSaves(spy: ReturnType, typename = 'Item') { - return spy.mock.calls.filter(([, value]) => (value as Record)?.__typename === typename); +/** Every entity write, whether it replaced the record or merged into it. */ +function spyEntityWrites(store: { saveEntity: (...args: any[]) => void; mergeEntity: (...args: any[]) => void }) { + const save = vi.spyOn(store, 'saveEntity'); + const merge = vi.spyOn(store, 'mergeEntity'); + return { + get calls() { + return [...save.mock.calls, ...merge.mock.calls]; + }, + }; +} + +function itemSaves(spy: { calls: unknown[][] }, typename = 'Item') { + return spy.calls.filter(([, value]) => (value as Record)?.__typename === typename); } describe('Entity Write Counts', () => { @@ -56,7 +67,7 @@ describe('Entity Write Counts', () => { await query; }); - const saveEntity = vi.spyOn(store, 'saveEntity'); + const saveEntity = spyEntityWrites(store); client.applyMutationEvent({ type: 'update', typename: 'Item', data: { id: 'i-1', name: 'B' } }); expect(itemSaves(saveEntity)).toHaveLength(1); @@ -72,7 +83,7 @@ describe('Entity Write Counts', () => { await query; }); - const saveEntity = vi.spyOn(store, 'saveEntity'); + const saveEntity = spyEntityWrites(store); // A subscription re-broadcasting what the store already holds: the apply // declines the write, and nothing writes on its behalf afterwards. @@ -95,7 +106,7 @@ describe('Entity Write Counts', () => { await query; }); - const saveEntity = vi.spyOn(store, 'saveEntity'); + const saveEntity = spyEntityWrites(store); client.applyMutationEvent({ type: 'create', typename: 'Item', @@ -123,7 +134,7 @@ describe('Entity Write Counts', () => { await query; }); - const saveEntity = vi.spyOn(store, 'saveEntity'); + const saveEntity = spyEntityWrites(store); client.applyMutationEvent({ type: 'create', typename: 'Item', @@ -152,7 +163,7 @@ describe('Entity Write Counts', () => { listQuery = query as unknown as typeof listQuery; }); - const saveEntity = vi.spyOn(store, 'saveEntity'); + const saveEntity = spyEntityWrites(store); // The item is already current in the store from the first query, so the // apply declines its write. The insert must not put one back. client.applyMutationEvent({ diff --git a/packages/fetchium/src/__tests__/event-persistence.test.ts b/packages/fetchium/src/__tests__/event-persistence.test.ts new file mode 100644 index 0000000..bc18d56 --- /dev/null +++ b/packages/fetchium/src/__tests__/event-persistence.test.ts @@ -0,0 +1,1544 @@ +import { describe, it, expect, vi, afterEach } from 'vitest'; +import { watcher, withContexts } from 'signalium'; +import { hashValue } from 'signalium/utils'; +import { t } from '../typeDefs.js'; +import { Entity } from '../proxy.js'; +import { RESTQuery } from '../rest/index.js'; +import { RESTQueryAdapter } from '../rest/RESTQueryAdapter.js'; +import { fetchQuery } from '../query.js'; +import { QueryClient, QueryClientContext } from '../QueryClient.js'; +import { SyncQueryStore, MemoryPersistentStore } from '../stores/sync.js'; +import { AsyncQueryStore, type AsyncPersistentStore } from '../stores/async.js'; +import { valueKeyFor, refCountKeyFor, refIdsKeyFor, queueKeyFor, updatedAtKeyFor } from '../stores/shared.js'; +import { createMockFetch, setupTestClient, sleep } from './utils.js'; + +/** + * What a streamed event leaves in the store: which records are written, which + * are not, and whether every reference on disk points at a record that exists. + * Complements `skip-write-and-snapshot-fixes.test.ts`. + */ + +function getDoc(kv: MemoryPersistentStore, key: number): Record | undefined { + const value = kv.getString(valueKeyFor(key)); + return value ? (JSON.parse(value) as Record) : undefined; +} +function refIds(kv: MemoryPersistentStore, key: number): number[] { + return Array.from(kv.getBuffer(refIdsKeyFor(key)) ?? []); +} + +/** + * Every record must be referenced (a refcount) unless it is a query, and every + * reference must point at a record that exists. + */ +function audit(kv: MemoryPersistentStore): { orphans: number[]; dangling: string[] } { + const ids = new Set(); + for (const k of kv.getAllKeys()) { + const m = /^sq:doc:(value|refCount|refIds|updatedAt):(\d+)$/.exec(k); + if (m) ids.add(Number(m[2])); + } + const orphans: number[] = []; + const dangling: string[] = []; + for (const id of ids) { + const hasValue = kv.getString(valueKeyFor(id)) !== undefined; + const isQuery = kv.getNumber(updatedAtKeyFor(id)) !== undefined; + if (hasValue && !isQuery && kv.getNumber(refCountKeyFor(id)) === undefined) orphans.push(id); + for (const ref of kv.getBuffer(refIdsKeyFor(id)) ?? []) { + if (kv.getString(valueKeyFor(ref)) === undefined) dangling.push(`${id} -> ${ref}`); + } + } + return { orphans, dangling }; +} + +function holdQuery(client: QueryClient, start: () => T): T { + return withContexts([[QueryClientContext, client]], () => { + const query = start(); + const w = watcher(() => (query as unknown as { value: unknown }).value); + w.addListener(() => {}); + return query; + }); +} + +const K = (typename: string, id: string | number) => hashValue([typename, id]); + +/** The keys of every entity write, whichever store method carried it. */ +function spyWrites(store: SyncQueryStore): () => number[] { + const saves = vi.spyOn(store, 'saveEntity'); + const merges = vi.spyOn(store, 'mergeEntity'); + return () => [...saves.mock.calls.map(c => c[0]), ...merges.mock.calls.map(c => c[0])]; +} + +/** Makes every write of the given key throw, whichever store method carries it. */ +function failWritesOf(store: SyncQueryStore, key: number, shouldFail: () => boolean = () => true): void { + const save = store.saveEntity.bind(store); + const merge = store.mergeEntity.bind(store); + vi.spyOn(store, 'saveEntity').mockImplementation((k, value, refs) => { + if (shouldFail() && k === key) throw new Error('quota exceeded'); + save(k, value, refs); + }); + vi.spyOn(store, 'mergeEntity').mockImplementation((k, fields, refs) => { + if (shouldFail() && k === key) throw new Error('quota exceeded'); + merge(k, fields, refs); + }); +} + +// ====================================================== +// Which records an event writes +// ====================================================== + +describe('records a streamed event writes', () => { + class Badge extends Entity { + __typename = t.typename('Badge'); + id = t.id; + label = t.string; + } + class User extends Entity { + __typename = t.typename('User'); + id = t.id; + name = t.string; + karma = t.number; + badge = t.optional(t.entity(Badge)); + } + class Comment extends Entity { + __typename = t.typename('Comment'); + id = t.id; + postId = t.string; + body = t.string; + author = t.entity(User); + mentions = t.optional(t.array(t.entity(User))); + } + class Post extends Entity { + __typename = t.typename('Post'); + id = t.id; + title = t.string; + comments = t.liveArray(Comment, { constraints: { postId: (this as any).id } }); + commentCount = t.liveValue(t.number, Comment, { + constraints: { postId: (this as any).id }, + onCreate: (count: number) => count + 1, + onUpdate: (count: number) => count, + onDelete: (count: number) => count - 1, + }); + } + class Thread extends Entity { + __typename = t.typename('Thread'); + id = t.id; + latest = t.liveValue(t.optional(t.string), Comment, { + constraints: { postId: (this as any).id }, + onCreate: (_prev: string | undefined, comment: Comment) => comment.body, + onUpdate: (prev: string | undefined) => prev, + onDelete: (prev: string | undefined) => prev, + }); + } + class GetPost extends RESTQuery { + params = { id: t.id }; + path = `/post/${this.params.id}`; + result = { post: t.entity(Post) }; + } + class GetThread extends RESTQuery { + params = { id: t.id }; + path = `/thread/${this.params.id}`; + result = { thread: t.entity(Thread) }; + } + class GetUser extends RESTQuery { + params = { id: t.id }; + path = `/user/${this.params.id}`; + result = { user: t.entity(User) }; + } + + const getClient = setupTestClient(); + const newAuthor = () => ({ + __typename: 'User', + id: 'u7', + name: 'Newbie', + karma: 1, + badge: { __typename: 'Badge', id: 'b7', label: 'new' }, + }); + + it('the same new entity twice in one payload is referenced, counted and written once', async () => { + const { client, mockFetch, kv, store } = getClient(); + mockFetch.get('/post/p1', { post: { __typename: 'Post', id: 'p1', title: 'T', comments: [], commentCount: 0 } }); + const postQ = holdQuery(client, () => fetchQuery(GetPost, { id: 'p1' })); + await postQ; + const writes = spyWrites(store); + + client.applyMutationEvent({ + type: 'create', + typename: 'Comment', + data: { __typename: 'Comment', id: 'c9', postId: 'p1', body: 'hi', author: newAuthor(), mentions: [newAuthor()] }, + }); + await sleep(5); + + const u7 = client.entityMap.getEntity(K('User', 'u7'))!; + // The comment references u7 twice (author and mentions[0]) and u7 keeps its badge. + expect(client.entityMap.getEntity(K('Comment', 'c9'))!.entityRefs!.get(u7)).toBe(2); + expect([...u7.entityRefs!.keys()].map(e => e.key)).toEqual([K('Badge', 'b7')]); + expect(writes().filter(k => k === K('User', 'u7'))).toHaveLength(1); + expect(refIds(kv, K('User', 'u7'))).toEqual([K('Badge', 'b7')]); + expect(getDoc(kv, K('Badge', 'b7'))).toMatchObject({ label: 'new' }); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + expect((postQ.value as any).post.comments[0].mentions[0].badge.label).toBe('new'); + }); + + it('the same entity twice in a fetched payload keeps its child references', async () => { + const { client, mockFetch, kv } = getClient(); + class GetComments extends RESTQuery { + path = '/comments'; + result = { comments: t.array(t.entity(Comment)) }; + } + const author = () => ({ + __typename: 'User', + id: 'u1', + name: 'Alice', + karma: 3, + badge: { __typename: 'Badge', id: 'b1', label: 'gold' }, + }); + const payload = () => ({ + comments: [ + { __typename: 'Comment', id: 'c1', postId: 'p1', body: 'a', author: author() }, + { __typename: 'Comment', id: 'c2', postId: 'p1', body: 'b', author: author() }, + ], + }); + mockFetch.get('/comments', payload()); + const q = holdQuery(client, () => fetchQuery(GetComments)); + await q; + + const u1 = client.entityMap.getEntity(K('User', 'u1'))!; + expect(u1.refCount).toBe(2); + expect([...u1.entityRefs!.keys()].map(e => e.key)).toEqual([K('Badge', 'b1')]); + expect(client.entityMap.getEntity(K('Badge', 'b1'))!.refCount).toBe(1); + expect(refIds(kv, K('User', 'u1'))).toEqual([K('Badge', 'b1')]); + expect(getDoc(kv, K('Badge', 'b1'))).toMatchObject({ label: 'gold' }); + expect((q.value as any).comments.map((c: any) => c.author.badge.label)).toEqual(['gold', 'gold']); + + // A refetch with the same payload changes nothing. + mockFetch.get('/comments', payload()); + await (q.value as any).__refetch(); + expect(u1.refCount).toBe(2); + expect(client.entityMap.getEntity(K('Badge', 'b1'))!.refCount).toBe(1); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + }); + + it('a create matched only by a liveValue reducer writes nothing, so no orphan record is left', async () => { + const { client, mockFetch, kv, store } = getClient(); + mockFetch.get('/thread/p1', { thread: { __typename: 'Thread', id: 'p1', latest: null } }); + const threadQ = holdQuery(client, () => fetchQuery(GetThread, { id: 'p1' })); + await threadQ; + const writes = spyWrites(store); + + client.applyMutationEvent({ + type: 'create', + typename: 'Comment', + data: { __typename: 'Comment', id: 'c9', postId: 'p1', body: 'newest', author: newAuthor() }, + }); + await sleep(5); + + expect((threadQ.value as any).thread.latest).toBe('newest'); + expect(writes()).toEqual([]); + expect(getDoc(kv, K('User', 'u7'))).toBeUndefined(); + expect(getDoc(kv, K('Badge', 'b7'))).toBeUndefined(); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + }); + + it('a partial update for an entity on disk but not in memory merges into its record', async () => { + const { client, mockFetch, kv } = getClient(); + mockFetch.get('/user/u1', { + user: { + __typename: 'User', + id: 'u1', + name: 'Alice', + karma: 10, + badge: { __typename: 'Badge', id: 'b1', label: 'gold' }, + }, + }); + await holdQuery(client, () => fetchQuery(GetUser, { id: 'u1' })); + client.entityMap.getEntity(K('User', 'u1'))!.evict(); + expect(client.entityMap.getEntity(K('User', 'u1'))).toBeUndefined(); + + // The payload carries the required fields (an event for an entity that is + // not in memory is parsed as a full record) but not the optional badge. + client.applyMutationEvent({ + type: 'update', + typename: 'User', + data: { __typename: 'User', id: 'u1', name: 'Alicia', karma: 10 }, + }); + await sleep(5); + + // The badge reference the event did not carry survives. + expect(getDoc(kv, K('User', 'u1'))).toMatchObject({ name: 'Alicia', karma: 10 }); + expect(refIds(kv, K('User', 'u1'))).toEqual([K('Badge', 'b1')]); + expect(getDoc(kv, K('Badge', 'b1'))).toMatchObject({ label: 'gold' }); + expect(kv.getNumber(refCountKeyFor(K('Badge', 'b1')))).toBe(1); + expect(client.entityMap.getEntity(K('User', 'u1'))).toBeUndefined(); + + // A replaced child is written and the old one released. + client.applyMutationEvent({ + type: 'update', + typename: 'User', + data: { + __typename: 'User', + id: 'u1', + name: 'Alicia', + karma: 10, + badge: { __typename: 'Badge', id: 'b9', label: 'platinum' }, + }, + }); + await sleep(5); + expect(getDoc(kv, K('User', 'u1'))).toMatchObject({ name: 'Alicia', karma: 10 }); + expect(refIds(kv, K('User', 'u1'))).toEqual([K('Badge', 'b9')]); + expect(getDoc(kv, K('Badge', 'b9'))).toMatchObject({ label: 'platinum' }); + expect(getDoc(kv, K('Badge', 'b1'))).toBeUndefined(); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + + // The next session hydrates the merged record. + client.destroy(); + const warn = vi.fn(); + const second = new QueryClient({ + store: new SyncQueryStore(kv), + adapters: [new RESTQueryAdapter({ fetch: mockFetch as never, baseUrl: 'http://localhost' })], + log: { warn, error: () => {} }, + } as never); + mockFetch.get('/user/u1', { user: { __typename: 'User', id: 'u1', name: 'FRESH', karma: 0 } }, { delay: 5000 }); + const q = holdQuery(second, () => fetchQuery(GetUser, { id: 'u1' })); + await sleep(10); + expect((q.value as any).user.name).toBe('Alicia'); + expect((q.value as any).user.karma).toBe(10); + expect((q.value as any).user.badge.label).toBe('platinum'); + expect(warn).not.toHaveBeenCalled(); + second.destroy(); + }); + + it('a store write that fails after routing is reverted, and the failure is logged rather than thrown', async () => { + const { client, mockFetch, kv, store } = getClient(); + mockFetch.get('/post/p1', { post: { __typename: 'Post', id: 'p1', title: 'T', comments: [], commentCount: 0 } }); + const postQ = holdQuery(client, () => fetchQuery(GetPost, { id: 'p1' })); + await postQ; + const warn = vi + .spyOn(client.getContext().log as { warn: (...args: unknown[]) => void }, 'warn') + .mockImplementation(() => {}); + failWritesOf(store, K('Comment', 'c9')); + + expect(() => + client.applyMutationEvent({ + type: 'create', + typename: 'Comment', + data: { __typename: 'Comment', id: 'c9', postId: 'p1', body: 'hi', author: newAuthor() }, + }), + ).not.toThrow(); + await sleep(5); + + expect(warn).toHaveBeenCalledWith('Failed to apply mutation event', expect.any(Error)); + // Memory and the store agree: the comment is in neither, and no record + // references it. (The author written before the failing write stays + // behind as an unreferenced record, as any write that fails mid-walk has + // always left its predecessors.) + expect((postQ.value as any).post.comments).toEqual([]); + expect((postQ.value as any).post.commentCount).toBe(0); + expect(client.entityMap.getEntity(K('Comment', 'c9'))).toBeUndefined(); + expect(refIds(kv, K('Post', 'p1'))).toEqual([]); + expect(getDoc(kv, K('Comment', 'c9'))).toBeUndefined(); + expect(audit(kv).dangling).toEqual([]); + }); + + it('a store write that fails for an unrouted root does not leak the root', async () => { + const { client, mockFetch, store } = getClient(); + mockFetch.get('/user/u1', { user: { __typename: 'User', id: 'u1', name: 'Alice', karma: 10 } }); + await holdQuery(client, () => fetchQuery(GetUser, { id: 'u1' })); + client.entityMap.getEntity(K('User', 'u1'))!.evict(); + const warn = vi + .spyOn(client.getContext().log as { warn: (...args: unknown[]) => void }, 'warn') + .mockImplementation(() => {}); + failWritesOf(store, K('User', 'u1')); + + client.applyMutationEvent({ + type: 'update', + typename: 'User', + data: { __typename: 'User', id: 'u1', name: 'Alicia', karma: 10 }, + }); + await sleep(5); + + expect(warn).toHaveBeenCalledWith('Failed to apply mutation event', expect.any(Error)); + expect(client.entityMap.getEntity(K('User', 'u1'))).toBeUndefined(); + }); + + it('a failed create delivered again is routed once: the array and the reducer agree', async () => { + const { client, mockFetch, kv, store } = getClient(); + mockFetch.get('/post/p1', { post: { __typename: 'Post', id: 'p1', title: 'T', comments: [], commentCount: 0 } }); + const postQ = holdQuery(client, () => fetchQuery(GetPost, { id: 'p1' })); + await postQ; + vi.spyOn(client.getContext().log as { warn: (...args: unknown[]) => void }, 'warn').mockImplementation(() => {}); + let fail = true; + failWritesOf(store, K('Comment', 'c9'), () => fail); + const event = () => ({ + type: 'create' as const, + typename: 'Comment', + data: { __typename: 'Comment', id: 'c9', postId: 'p1', body: 'hi', author: newAuthor() }, + }); + + client.applyMutationEvent(event()); + await sleep(5); + expect((postQ.value as any).post.comments).toEqual([]); + expect((postQ.value as any).post.commentCount).toBe(0); + + // The transport retries once the store works again. + fail = false; + client.applyMutationEvent(event()); + await sleep(5); + expect((postQ.value as any).post.comments.map((c: any) => c.id)).toEqual(['c9']); + expect((postQ.value as any).post.commentCount).toBe(1); + expect(getDoc(kv, K('Comment', 'c9'))).toMatchObject({ body: 'hi' }); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + }); + + it('an update matched only by a liveValue reducer refreshes the record of an entity on disk but not in memory', async () => { + const { client, mockFetch, kv } = getClient(); + class GetComments extends RESTQuery { + path = '/comments'; + result = { comments: t.array(t.entity(Comment)) }; + } + mockFetch.get('/thread/p1', { thread: { __typename: 'Thread', id: 'p1', latest: null } }); + mockFetch.get('/comments', { + comments: [ + { + __typename: 'Comment', + id: 'c1', + postId: 'p1', + body: 'orig', + author: { __typename: 'User', id: 'u1', name: 'Alice', karma: 1 }, + }, + ], + }); + const threadQ = holdQuery(client, () => fetchQuery(GetThread, { id: 'p1' })); + await threadQ; + await holdQuery(client, () => fetchQuery(GetComments)); + // The comments query is collected from memory; its record stays on disk. + client.entityMap.getEntity(K('Comment', 'c1'))!.evict(); + + client.applyMutationEvent({ + type: 'update', + typename: 'Comment', + data: { + __typename: 'Comment', + id: 'c1', + postId: 'p1', + body: 'edited', + author: { __typename: 'User', id: 'u1', name: 'Alice', karma: 1 }, + }, + }); + await sleep(5); + + expect((threadQ.value as any).thread.latest).toBeUndefined(); + expect(getDoc(kv, K('Comment', 'c1'))).toMatchObject({ body: 'edited' }); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + }); + + it('a create adopted by an existing entity in its own payload is kept and written, even when nothing routes it', async () => { + const { client, mockFetch, kv } = getClient(); + class Note extends Entity { + __typename = t.typename('Note'); + id = t.id; + body = t.string; + article = t.optional(t.entity(ArticleRef)); + } + class NoteRef extends Entity { + __typename = t.typename('Note'); + id = t.id; + body = t.string; + } + class ArticleRef extends Entity { + __typename = t.typename('Article'); + id = t.id; + latestNote = t.optional(t.entity(NoteRef)); + } + class Article extends Entity { + __typename = t.typename('Article'); + id = t.id; + title = t.string; + latestNote = t.optional(t.entity(NoteRef)); + notes = t.array(t.entity(Note)); + } + class GetArticle extends RESTQuery { + params = { id: t.id }; + path = `/article/${this.params.id}`; + result = { article: t.entity(Article) }; + } + mockFetch.get('/article/a1', { + article: { + __typename: 'Article', + id: 'a1', + title: 'A', + notes: [{ __typename: 'Note', id: 'n1', body: 'first' }], + }, + }); + const artQ = holdQuery(client, () => fetchQuery(GetArticle, { id: 'a1' })); + await artQ; + + // No live collection routes notes; the payload's article points back at the note. + client.applyMutationEvent({ + type: 'create', + typename: 'Note', + data: { + __typename: 'Note', + id: 'n9', + body: 'hi', + article: { __typename: 'Article', id: 'a1', latestNote: { __typename: 'Note', id: 'n9', body: 'hi' } }, + }, + }); + await sleep(5); + + const n9 = client.entityMap.getEntity(K('Note', 'n9'))!; + expect(n9).toBeDefined(); + expect(n9.refCount).toBe(1); + expect((artQ.value as any).article.latestNote.body).toBe('hi'); + expect(refIds(kv, K('Article', 'a1')).sort()).toEqual([K('Note', 'n1'), K('Note', 'n9')].sort()); + expect(getDoc(kv, K('Note', 'n9'))).toMatchObject({ body: 'hi' }); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + + // A later event for the note reaches the article's consumer. + client.applyMutationEvent({ type: 'update', typename: 'Note', data: { id: 'n9', body: 'edited' } }); + expect((artQ.value as any).article.latestNote.body).toBe('edited'); + expect(getDoc(kv, K('Note', 'n9'))).toMatchObject({ body: 'edited' }); + + client.destroy(); + const second = new QueryClient({ + store: new SyncQueryStore(kv), + adapters: [new RESTQueryAdapter({ fetch: mockFetch as never, baseUrl: 'http://localhost' })], + log: { warn: () => {}, error: () => {} }, + } as never); + mockFetch.get( + '/article/a1', + { article: { __typename: 'Article', id: 'a1', title: 'FRESH', notes: [] } }, + { delay: 5000 }, + ); + const q = holdQuery(second, () => fetchQuery(GetArticle, { id: 'a1' })); + await sleep(10); + expect((q.value as any).article.latestNote.body).toBe('edited'); + second.destroy(); + }); + + it('created entities that only reference each other are evicted with the unrouted root', async () => { + const { client, mockFetch, kv } = getClient(); + class Author extends Entity { + __typename = t.typename('User'); + id = t.id; + name = t.string; + karma = t.number; + bestComment = t.optional(t.entity(CommentRef)); + } + class CommentRef extends Entity { + __typename = t.typename('Comment'); + id = t.id; + body = t.string; + } + class Remark extends Entity { + __typename = t.typename('Comment'); + id = t.id; + postId = t.string; + body = t.string; + author = t.entity(Author); + } + class GetRemark extends RESTQuery { + params = { id: t.id }; + path = `/remark/${this.params.id}`; + result = { remark: t.entity(Remark) }; + } + mockFetch.get('/remark/c0', { + remark: { + __typename: 'Comment', + id: 'c0', + postId: 'p0', + body: 'seed', + author: { __typename: 'User', id: 'u0', name: 'Zero', karma: 0 }, + }, + }); + await holdQuery(client, () => fetchQuery(GetRemark, { id: 'c0' })); + + client.applyMutationEvent({ + type: 'create', + typename: 'Comment', + data: { + __typename: 'Comment', + id: 'c9', + postId: 'nowhere', + body: 'hi', + author: { + __typename: 'User', + id: 'u7', + name: 'Newbie', + karma: 1, + bestComment: { __typename: 'Comment', id: 'c9', body: 'hi' }, + }, + }, + }); + await sleep(5); + + expect(client.entityMap.getEntity(K('Comment', 'c9'))).toBeUndefined(); + expect(client.entityMap.getEntity(K('User', 'u7'))).toBeUndefined(); + expect(getDoc(kv, K('Comment', 'c9'))).toBeUndefined(); + expect(getDoc(kv, K('User', 'u7'))).toBeUndefined(); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + }); + + it('a root write that fails while an existing entity adopted the root leaves no dangling reference', async () => { + const { client, mockFetch, kv, store } = getClient(); + class PostRef extends Entity { + __typename = t.typename('Post'); + id = t.id; + latestComment = t.optional(t.entity(CommentRef)); + } + class CommentRef extends Entity { + __typename = t.typename('Comment'); + id = t.id; + body = t.string; + } + class Reply extends Entity { + __typename = t.typename('Comment'); + id = t.id; + postId = t.string; + body = t.string; + author = t.entity(User); + post = t.optional(t.entity(PostRef)); + } + class Topic extends Entity { + __typename = t.typename('Post'); + id = t.id; + title = t.string; + latestComment = t.optional(t.entity(CommentRef)); + replies = t.liveArray(Reply, { constraints: { postId: (this as any).id } }); + } + class GetTopic extends RESTQuery { + params = { id: t.id }; + path = `/topic/${this.params.id}`; + result = { topic: t.entity(Topic) }; + } + mockFetch.get('/topic/t1', { topic: { __typename: 'Post', id: 't1', title: 'T', replies: [] } }); + const topicQ = holdQuery(client, () => fetchQuery(GetTopic, { id: 't1' })); + await topicQ; + vi.spyOn(client.getContext().log as { warn: (...args: unknown[]) => void }, 'warn').mockImplementation(() => {}); + failWritesOf(store, K('Comment', 'c9')); + + client.applyMutationEvent({ + type: 'create', + typename: 'Comment', + data: { + __typename: 'Comment', + id: 'c9', + postId: 't1', + body: 'hi', + author: newAuthor(), + post: { __typename: 'Post', id: 't1', latestComment: { __typename: 'Comment', id: 'c9', body: 'hi' } }, + }, + }); + await sleep(5); + + // The topic's record was not written with a reference to a record that + // does not exist; memory holds the adoption and the next write heals it. + expect(audit(kv).dangling).toEqual([]); + expect((topicQ.value as any).topic.replies).toEqual([]); + expect((topicQ.value as any).topic.latestComment.body).toBe('hi'); + expect(refIds(kv, K('Post', 't1'))).toEqual([]); + }); + + it('a payload that links back to its parent is applied, persisted and hydrated', async () => { + const { client, mockFetch, kv } = getClient(); + class Folder extends Entity { + __typename = t.typename('Folder'); + id = t.id; + name = t.string; + files = t.array(t.entity(File)); + } + class FolderRef extends Entity { + __typename = t.typename('Folder'); + id = t.id; + name = t.string; + } + class File extends Entity { + __typename = t.typename('File'); + id = t.id; + name = t.string; + folder = t.entity(FolderRef); + } + class GetFolder extends RESTQuery { + params = { id: t.id }; + path = `/folder/${this.params.id}`; + result = { folder: t.entity(Folder) }; + } + mockFetch.get('/folder/f1', { + folder: { + __typename: 'Folder', + id: 'f1', + name: 'root', + files: [ + { __typename: 'File', id: 'x1', name: 'a.txt', folder: { __typename: 'Folder', id: 'f1', name: 'root' } }, + ], + }, + }); + const q = holdQuery(client, () => fetchQuery(GetFolder, { id: 'f1' })); + await q; + expect((q.value as any).folder.files[0].folder.name).toBe('root'); + expect(refIds(kv, K('Folder', 'f1'))).toEqual([K('File', 'x1')]); + expect(refIds(kv, K('File', 'x1'))).toEqual([K('Folder', 'f1')]); + + client.destroy(); + const warn = vi.fn(); + const second = new QueryClient({ + store: new SyncQueryStore(kv), + adapters: [new RESTQueryAdapter({ fetch: mockFetch as never, baseUrl: 'http://localhost' })], + log: { warn, error: () => {} }, + } as never); + mockFetch.get( + '/folder/f1', + { folder: { __typename: 'Folder', id: 'f1', name: 'FRESH', files: [] } }, + { delay: 5000 }, + ); + const q2 = holdQuery(second, () => fetchQuery(GetFolder, { id: 'f1' })); + await sleep(10); + expect(warn).not.toHaveBeenCalled(); + expect((q2.value as any).folder.files.map((f: any) => f.folder.name)).toEqual(['root']); + second.destroy(); + }); + + it('merging keeps the fields an event did not carry, including a parse result, and merges created descendants too', async () => { + const { client, mockFetch, kv } = getClient(); + class Icon extends Entity { + __typename = t.typename('Icon'); + id = t.id; + url = t.string; + } + class Award extends Entity { + __typename = t.typename('Badge'); + id = t.id; + label = t.string; + icon = t.optional(t.entity(Icon)); + } + class Profile extends Entity { + __typename = t.typename('Profile'); + id = t.id; + name = t.string; + bio = t.optional(t.string); + badge = t.optional(t.entity(Award)); + parsed = t.optional(t.result(t.entity(Award))); + } + class GetProfile extends RESTQuery { + params = { id: t.id }; + path = `/profile/${this.params.id}`; + result = { profile: t.entity(Profile) }; + } + mockFetch.get('/profile/pr1', { + profile: { + __typename: 'Profile', + id: 'pr1', + name: 'Alice', + bio: 'hello', + badge: { + __typename: 'Badge', + id: 'b1', + label: 'gold', + icon: { __typename: 'Icon', id: 'i1', url: '/gold.png' }, + }, + parsed: { __typename: 'Badge', id: 'b5', label: 'parsed' }, + }, + }); + await holdQuery(client, () => fetchQuery(GetProfile, { id: 'pr1' })); + for (const key of ['pr1']) client.entityMap.getEntity(K('Profile', key))!.evict(); + client.entityMap.getEntity(K('Badge', 'b1'))?.evict(); + client.entityMap.getEntity(K('Icon', 'i1'))?.evict(); + + // The badge under the event is a created entity with a record on disk: + // its own merge keeps the icon the event did not mention. + client.applyMutationEvent({ + type: 'update', + typename: 'Profile', + data: { + __typename: 'Profile', + id: 'pr1', + name: 'Alicia', + badge: { __typename: 'Badge', id: 'b1', label: 'gold+' }, + }, + }); + await sleep(5); + + const doc = getDoc(kv, K('Profile', 'pr1'))!; + expect(doc).toMatchObject({ name: 'Alicia', bio: 'hello' }); + expect(doc.parsed).toEqual({ success: true, value: { __entityRef: K('Badge', 'b5') } }); + expect(getDoc(kv, K('Badge', 'b1'))).toMatchObject({ label: 'gold+', icon: { __entityRef: K('Icon', 'i1') } }); + expect(refIds(kv, K('Badge', 'b1'))).toEqual([K('Icon', 'i1')]); + expect(getDoc(kv, K('Icon', 'i1'))).toMatchObject({ url: '/gold.png' }); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + }); + + it('a write after a cold start does not rewrite the hydrated entities under it', async () => { + const { client, mockFetch, kv } = getClient(); + const comments = Array.from({ length: 5 }, (_, i) => ({ + __typename: 'Comment', + id: `c${i}`, + postId: 'p1', + body: `b${i}`, + author: { + __typename: 'User', + id: `u${i}`, + name: `n${i}`, + karma: i, + badge: { __typename: 'Badge', id: `b${i}`, label: `l${i}` }, + }, + })); + mockFetch.get('/post/p1', { post: { __typename: 'Post', id: 'p1', title: 'T', comments, commentCount: 5 } }); + await holdQuery(client, () => fetchQuery(GetPost, { id: 'p1' })); + client.destroy(); + + const store = new SyncQueryStore(kv); + const second = new QueryClient({ + store, + adapters: [new RESTQueryAdapter({ fetch: mockFetch as never, baseUrl: 'http://localhost' })], + log: { warn: () => {}, error: () => {} }, + } as never); + mockFetch.get( + '/post/p1', + { post: { __typename: 'Post', id: 'p1', title: 'FRESH', comments: [], commentCount: 0 } }, + { delay: 5000 }, + ); + const q = holdQuery(second, () => fetchQuery(GetPost, { id: 'p1' })); + await sleep(10); + expect((q.value as any).post.comments).toHaveLength(5); + const writes = spyWrites(store); + + second.applyMutationEvent({ type: 'update', typename: 'Post', data: { id: 'p1', title: 'T2' } }); + await sleep(5); + expect(writes()).toEqual([K('Post', 'p1')]); + expect(getDoc(kv, K('Post', 'p1'))).toMatchObject({ title: 'T2' }); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + second.destroy(); + }); + + it('a member routed into the live array of an entity built from events reaches its record', async () => { + const { client, mockFetch, kv } = getClient(); + // A second class for Post without the live fields makes them optional in + // the merged definition, so an event for a post can omit them. + class PostPreview extends Entity { + __typename = t.typename('Post'); + id = t.id; + title = t.string; + blogId = t.optional(t.string); + } + class Blog extends Entity { + __typename = t.typename('Blog'); + id = t.id; + posts = t.liveArray(PostPreview, { constraints: { blogId: (this as any).id } }); + } + class GetBlog extends RESTQuery { + params = { id: t.id }; + path = `/blog/${this.params.id}`; + result = { blog: t.entity(Blog) }; + } + mockFetch.get('/post/p9', { + post: { + __typename: 'Post', + id: 'p9', + title: 'T', + comments: [ + { + __typename: 'Comment', + id: 'c1', + postId: 'p9', + body: 'first', + author: { __typename: 'User', id: 'u1', name: 'A', karma: 1 }, + }, + ], + commentCount: 1, + }, + }); + mockFetch.get('/blog/b1', { blog: { __typename: 'Blog', id: 'b1', posts: [] } }); + const postQ = holdQuery(client, () => fetchQuery(GetPost, { id: 'p9' })); + await postQ; + const blogQ = holdQuery(client, () => fetchQuery(GetBlog, { id: 'b1' })); + await blogQ; + expect(refIds(kv, K('Post', 'p9'))).toEqual([K('Comment', 'c1')]); + // The post is collected from memory; its record stays on disk. + client.entityMap.getEntity(K('Post', 'p9'))!.evict(); + + // An event brings the post back into memory, retained by the blog's live + // array and built from the event alone: no comments, no count. + client.applyMutationEvent({ + type: 'update', + typename: 'Post', + data: { __typename: 'Post', id: 'p9', title: 'T2', blogId: 'b1', commentCount: 1 }, + }); + await sleep(5); + expect((blogQ.value as any).blog.posts.map((p: any) => p.title)).toEqual(['T2']); + const p9 = client.entityMap.getEntity(K('Post', 'p9'))!; + expect(p9._partial).toBe(true); + expect(getDoc(kv, K('Post', 'p9'))).toMatchObject({ title: 'T2', comments: [{ __entityRef: K('Comment', 'c1') }] }); + + // A comment routed into its (empty in memory) live array reaches the record. + client.applyMutationEvent({ + type: 'create', + typename: 'Comment', + data: { __typename: 'Comment', id: 'c9', postId: 'p9', body: 'second', author: newAuthor() }, + }); + await sleep(5); + expect(getDoc(kv, K('Post', 'p9'))).toMatchObject({ title: 'T2', comments: [{ __entityRef: K('Comment', 'c9') }] }); + expect(refIds(kv, K('Post', 'p9'))).toEqual([K('Comment', 'c9')]); + expect(getDoc(kv, K('Comment', 'c9'))).toMatchObject({ body: 'second' }); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + }); + + it('an unrouted root that references itself is evicted', async () => { + const { client, mockFetch, kv } = getClient(); + class MessageRef extends Entity { + __typename = t.typename('Message'); + id = t.id; + body = t.string; + } + class Message extends Entity { + __typename = t.typename('Message'); + id = t.id; + roomId = t.string; + body = t.string; + threadRoot = t.optional(t.entity(MessageRef)); + } + class Room extends Entity { + __typename = t.typename('Room'); + id = t.id; + messages = t.liveArray(Message, { constraints: { roomId: (this as any).id } }); + } + class GetRoom extends RESTQuery { + params = { id: t.id }; + path = `/room/${this.params.id}`; + result = { room: t.entity(Room) }; + } + mockFetch.get('/room/r1', { room: { __typename: 'Room', id: 'r1', messages: [] } }); + await holdQuery(client, () => fetchQuery(GetRoom, { id: 'r1' })); + + client.applyMutationEvent({ + type: 'create', + typename: 'Message', + data: { + __typename: 'Message', + id: 'm9', + roomId: 'OTHER', + body: 'hi', + threadRoot: { __typename: 'Message', id: 'm9', body: 'hi' }, + }, + }); + await sleep(5); + expect(client.entityMap.getEntity(K('Message', 'm9'))).toBeUndefined(); + expect(getDoc(kv, K('Message', 'm9'))).toBeUndefined(); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + }); + + it('an entity an event creates under a root in memory merges over its record too', async () => { + const { client, mockFetch, kv } = getClient(); + mockFetch.get('/post/p1', { post: { __typename: 'Post', id: 'p1', title: 'T', comments: [], commentCount: 0 } }); + mockFetch.get('/user/u7', { user: { ...newAuthor(), name: 'Old' } }); + const postQ = holdQuery(client, () => fetchQuery(GetPost, { id: 'p1' })); + await postQ; + await holdQuery(client, () => fetchQuery(GetUser, { id: 'u7' })); + // The author's record is on disk with its badge; the author is collected from memory. + client.entityMap.getEntity(K('User', 'u7'))!.evict(); + client.entityMap.getEntity(K('Badge', 'b7'))?.evict(); + + client.applyMutationEvent({ + type: 'create', + typename: 'Comment', + data: { + __typename: 'Comment', + id: 'c9', + postId: 'p1', + body: 'hi', + author: { __typename: 'User', id: 'u7', name: 'Newbie', karma: 2 }, + }, + }); + await sleep(5); + expect((postQ.value as any).post.comments.map((c: any) => c.author.name)).toEqual(['Newbie']); + // The event did not mention the badge: the record keeps it. + expect(getDoc(kv, K('User', 'u7'))).toMatchObject({ + name: 'Newbie', + karma: 2, + badge: { __entityRef: K('Badge', 'b7') }, + }); + expect(getDoc(kv, K('Badge', 'b7'))).toMatchObject({ label: 'new' }); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + }); +}); + +// ====================================================== +// Entities with an in-memory gcTime linger after release +// ====================================================== + +describe('records a streamed event writes when released entities linger (gcTime)', () => { + class Badge extends Entity { + static cache = { gcTime: 1 }; + __typename = t.typename('Badge'); + id = t.id; + label = t.string; + } + class User extends Entity { + static cache = { gcTime: 1 }; + __typename = t.typename('User'); + id = t.id; + name = t.string; + badge = t.optional(t.entity(Badge)); + } + class Comment extends Entity { + static cache = { gcTime: 1 }; + __typename = t.typename('Comment'); + id = t.id; + postId = t.string; + body = t.string; + author = t.entity(User); + } + class Post extends Entity { + __typename = t.typename('Post'); + id = t.id; + title = t.string; + comments = t.liveArray(Comment, { constraints: { postId: (this as any).id } }); + } + class GetPost extends RESTQuery { + params = { id: t.id }; + path = `/post/${this.params.id}`; + result = { post: t.entity(Post) }; + } + const getClient = setupTestClient({ evictionMultiplier: 0.001 }); + const newAuthor = () => ({ + __typename: 'User', + id: 'u7', + name: 'Newbie', + badge: { __typename: 'Badge', id: 'b7', label: 'new' }, + }); + + it('an unrouted create evicts everything it created at once, gcTime or not, and writes none of it', async () => { + const { client, mockFetch, kv, store } = getClient(); + mockFetch.get('/post/p1', { post: { __typename: 'Post', id: 'p1', title: 'T', comments: [] } }); + const postQ = holdQuery(client, () => fetchQuery(GetPost, { id: 'p1' })); + await postQ; + const writes = spyWrites(store); + + client.applyMutationEvent({ + type: 'create', + typename: 'Comment', + data: { __typename: 'Comment', id: 'c9', postId: 'OTHER', body: 'x', author: newAuthor() }, + }); + await sleep(5); + + // Nothing references what the event created; had the author lingered + // until its gcTime, a later event could have written it as a record + // nothing references. + expect(client.entityMap.getEntity(K('Comment', 'c9'))).toBeUndefined(); + expect(client.entityMap.getEntity(K('User', 'u7'))).toBeUndefined(); + expect(client.entityMap.getEntity(K('Badge', 'b7'))).toBeUndefined(); + expect(writes()).toEqual([]); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + + // A second unrouted event for the same author still writes nothing. + client.applyMutationEvent({ + type: 'create', + typename: 'Comment', + data: { __typename: 'Comment', id: 'c9b', postId: 'OTHER', body: 'x', author: newAuthor() }, + }); + await sleep(5); + expect(writes()).toEqual([]); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + + // A routed create with the same author writes it, with its badge. + client.applyMutationEvent({ + type: 'create', + typename: 'Comment', + data: { __typename: 'Comment', id: 'c10', postId: 'p1', body: 'y', author: newAuthor() }, + }); + await sleep(5); + expect((postQ.value as any).post.comments.map((c: any) => c.author.badge.label)).toEqual(['new']); + expect(getDoc(kv, K('Comment', 'c10'))).toMatchObject({ body: 'y' }); + expect(getDoc(kv, K('User', 'u7'))).toMatchObject({ name: 'Newbie' }); + expect(getDoc(kv, K('Badge', 'b7'))).toMatchObject({ label: 'new' }); + expect(refIds(kv, K('User', 'u7'))).toEqual([K('Badge', 'b7')]); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + + // GC time passes; the entities are retained and survive. + await sleep(300); + expect(client.entityMap.getEntity(K('User', 'u7'))).toBeDefined(); + expect(client.entityMap.getEntity(K('Badge', 'b7'))).toBeDefined(); + }); + + it('an unrouted create followed by GC leaves nothing in memory or on disk', async () => { + const { client, mockFetch, kv } = getClient(); + mockFetch.get('/post/p1', { post: { __typename: 'Post', id: 'p1', title: 'T', comments: [] } }); + await holdQuery(client, () => fetchQuery(GetPost, { id: 'p1' })); + + client.applyMutationEvent({ + type: 'create', + typename: 'Comment', + data: { __typename: 'Comment', id: 'c9', postId: 'OTHER', body: 'x', author: newAuthor() }, + }); + await sleep(300); + + expect(client.entityMap.getEntity(K('User', 'u7'))).toBeUndefined(); + expect(client.entityMap.getEntity(K('Badge', 'b7'))).toBeUndefined(); + expect(getDoc(kv, K('User', 'u7'))).toBeUndefined(); + expect(getDoc(kv, K('Badge', 'b7'))).toBeUndefined(); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + }); +}); + +// ====================================================== +// AsyncQueryStore writer +// ====================================================== + +class AsyncDelegate implements AsyncPersistentStore { + kv: Record = {}; + constructor(public delayMs: number = 1) {} + private tick() { + return new Promise(resolve => setTimeout(resolve, this.delayMs)); + } + async has(key: string) { + await this.tick(); + return key in this.kv; + } + async getString(key: string) { + await this.tick(); + return this.kv[key] as string | undefined; + } + async setString(key: string, value: string) { + await this.tick(); + this.kv[key] = value; + } + async getNumber(key: string) { + await this.tick(); + return this.kv[key] as number | undefined; + } + async setNumber(key: string, value: number) { + await this.tick(); + this.kv[key] = value; + } + async getBuffer(key: string) { + await this.tick(); + return this.kv[key] as Uint32Array | undefined; + } + async setBuffer(key: string, value: Uint32Array) { + await this.tick(); + this.kv[key] = value; + } + async delete(key: string) { + await this.tick(); + delete this.kv[key]; + } + async getAllKeys() { + await this.tick(); + return Object.keys(this.kv); + } +} + +function writerStore(delegate: AsyncPersistentStore): AsyncQueryStore { + return new AsyncQueryStore({ + isWriter: true, + delegate, + connect: handleMessage => ({ sendMessage: msg => handleMessage(msg) }), + }); +} + +/** Waits for the writer's read of the keys it holds. */ +async function scanned(store: AsyncQueryStore): Promise { + for (let i = 0; i < 5000; i++) { + if (store.hasEntity!(0) !== undefined) return; + await sleep(2); + } + throw new Error('writer never read its keys'); +} + +async function drain(store: AsyncQueryStore): Promise { + for (let i = 0; i < 5000; i++) { + if (store.isSettled()) return; + await sleep(2); + } + throw new Error('writer queue did not drain'); +} + +function asyncDoc(delegate: AsyncDelegate, key: number): Record | undefined { + const value = delegate.kv[valueKeyFor(key)] as string | undefined; + return value ? (JSON.parse(value) as Record) : undefined; +} + +describe('AsyncQueryStore writer', () => { + const clients: QueryClient[] = []; + afterEach(() => { + for (const client of clients.splice(0)) client.destroy(); + }); + + class Badge extends Entity { + __typename = t.typename('Badge'); + id = t.id; + label = t.string; + } + class User extends Entity { + __typename = t.typename('User'); + id = t.id; + name = t.string; + karma = t.number; + badge = t.optional(t.entity(Badge)); + } + class GetUser extends RESTQuery { + params = { id: t.id }; + path = `/user/${this.params.id}`; + result = { user: t.entity(User) }; + } + + function makeClient(store: AsyncQueryStore, mockFetch: ReturnType) { + const client = new QueryClient({ + store, + adapters: [new RESTQueryAdapter({ fetch: mockFetch as never, baseUrl: 'http://localhost' })], + log: { warn: () => {}, error: () => {} }, + } as never); + clients.push(client); + return client; + } + + it('answers hasEntity from the records it holds, seeded from the delegate', async () => { + const delegate = new AsyncDelegate(); + delegate.kv[valueKeyFor(42)] = JSON.stringify({ name: 'on disk' }); + const store = writerStore(delegate); + await scanned(store); + expect(store.hasEntity!(42)).toBe(true); + expect(store.hasEntity!(43)).toBe(false); + + store.saveEntity(43, { name: 'written' }); + await drain(store); + expect(store.hasEntity!(43)).toBe(true); + + // A query whose record references 43 is deleted: the cascade drops it. + store.saveQuery( + { statics: { id: 'def', cache: {} } } as never, + 7, + { user: { __entityRef: 43 } }, + Date.now(), + new Set([43]), + ); + await drain(store); + store.deleteQuery(7); + await drain(store); + expect(store.hasEntity!(43)).toBe(false); + expect(store.hasEntity!(42)).toBe(true); + }); + + it('refreshes the record of an entity on disk but not in memory by merging the event into it', async () => { + const delegate = new AsyncDelegate(); + const store = writerStore(delegate); + const mockFetch = createMockFetch(); + mockFetch.get('/user/u1', { + user: { + __typename: 'User', + id: 'u1', + name: 'Alice', + karma: 10, + badge: { __typename: 'Badge', id: 'b1', label: 'gold' }, + }, + }); + const client = makeClient(store, mockFetch); + await holdQuery(client, () => fetchQuery(GetUser, { id: 'u1' })); + await drain(store); + await scanned(store); + client.entityMap.getEntity(K('User', 'u1'))!.evict(); + + client.applyMutationEvent({ + type: 'update', + typename: 'User', + data: { __typename: 'User', id: 'u1', name: 'Alicia', karma: 10 }, + }); + await drain(store); + + expect(asyncDoc(delegate, K('User', 'u1'))).toMatchObject({ name: 'Alicia', karma: 10 }); + expect(Array.from(delegate.kv[refIdsKeyFor(K('User', 'u1'))] as Uint32Array)).toEqual([K('Badge', 'b1')]); + expect(asyncDoc(delegate, K('Badge', 'b1'))).toMatchObject({ label: 'gold' }); + expect(client.entityMap.getEntity(K('User', 'u1'))).toBeUndefined(); + + // An entity the store has never seen leaves nothing behind. + client.applyMutationEvent({ + type: 'update', + typename: 'User', + data: { __typename: 'User', id: 'u404', name: 'Nobody', karma: 0 }, + }); + await drain(store); + expect(asyncDoc(delegate, K('User', 'u404'))).toBeUndefined(); + }); + + it('does not write an event for an entity it cannot yet say it holds, and refreshes it once it can', async () => { + const delegate = new AsyncDelegate(1); + const mockFetch = createMockFetch(); + mockFetch.get('/user/u1', { + user: { + __typename: 'User', + id: 'u1', + name: 'Alice', + karma: 10, + badge: { __typename: 'Badge', id: 'b1', label: 'gold' }, + }, + }); + mockFetch.get('/user/u2', { user: { __typename: 'User', id: 'u2', name: 'Bob', karma: 1 } }); + const first = writerStore(delegate); + const client1 = makeClient(first, mockFetch); + await holdQuery(client1, () => fetchQuery(GetUser, { id: 'u1' })); + await drain(first); + client1.destroy(); + + // A fresh writer has not read its keys yet when the event lands. + let releaseScan = () => {}; + const scanGate = new Promise(resolve => (releaseScan = resolve)); + const originalGetAllKeys = delegate.getAllKeys.bind(delegate); + delegate.getAllKeys = async () => { + await scanGate; + return originalGetAllKeys(); + }; + const store = writerStore(delegate); + const client = makeClient(store, mockFetch); + await holdQuery(client, () => fetchQuery(GetUser, { id: 'u2' })); + expect(store.hasEntity!(K('User', 'u1'))).toBeUndefined(); + // What this writer wrote itself is known even now. + expect(store.hasEntity!(K('User', 'u2'))).toBe(true); + const event = () => ({ + type: 'update' as const, + typename: 'User', + data: { __typename: 'User', id: 'u1', name: 'Alicia', karma: 10 }, + }); + client.applyMutationEvent(event()); + expect(client.entityMap.getEntity(K('User', 'u1'))).toBeUndefined(); + releaseScan(); + await scanned(store); + await drain(store); + // Not written: nothing referenced it and the store could not say it held it. + expect(asyncDoc(delegate, K('User', 'u1'))).toMatchObject({ name: 'Alice' }); + expect(store.hasEntity!(K('User', 'u1'))).toBe(true); + + client.applyMutationEvent(event()); + await drain(store); + expect(asyncDoc(delegate, K('User', 'u1'))).toMatchObject({ + name: 'Alicia', + karma: 10, + badge: { __entityRef: K('Badge', 'b1') }, + }); + expect(asyncDoc(delegate, K('Badge', 'b1'))).toMatchObject({ label: 'gold' }); + }); + + it('ignores what it does not understand on the channel and keeps processing', async () => { + const delegate = new AsyncDelegate(); + const store = writerStore(delegate); + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {}); + try { + const handle = (store as unknown as { handleMessage(msg: unknown): void }).handleMessage.bind(store); + handle(null); + handle(undefined); + handle('hello'); + handle({ type: -1 }); + handle({ type: 9, entityKey: 1 }); + handle({ type: 1.5 }); + store.saveEntity(7, { name: 'still written' }); + await drain(store); + expect(asyncDoc(delegate, 7)).toMatchObject({ name: 'still written' }); + expect(store.isSettled()).toBe(true); + expect(consoleError).toHaveBeenCalled(); + } finally { + consoleError.mockRestore(); + } + }); + + it('writes the fields as the record when the stored record cannot be merged into', async () => { + const delegate = new AsyncDelegate(); + delegate.kv[valueKeyFor(5)] = '{not json'; + delegate.kv[valueKeyFor(6)] = JSON.stringify(['an', 'array']); + const store = writerStore(delegate); + store.mergeEntity(5, { name: 'healed' }); + store.mergeEntity(6, { name: 'healed too' }); + await drain(store); + expect(asyncDoc(delegate, 5)).toEqual({ name: 'healed' }); + expect(asyncDoc(delegate, 6)).toEqual({ name: 'healed too' }); + + const kv = new MemoryPersistentStore(); + kv.setString(valueKeyFor(5), '{not json'); + new SyncQueryStore(kv).mergeEntity(5, { name: 'healed' }); + expect(getDoc(kv, 5)).toEqual({ name: 'healed' }); + }); + + it('counts a queued write as held, so an event for it is refreshed before the write lands', async () => { + const delegate = new AsyncDelegate(5); + const store = writerStore(delegate); + await scanned(store); + const mockFetch = createMockFetch(); + mockFetch.get('/user/u1', { user: { __typename: 'User', id: 'u1', name: 'Alice', karma: 10 } }); + const client = makeClient(store, mockFetch); + await holdQuery(client, () => fetchQuery(GetUser, { id: 'u1' })); + // The write is queued, not processed; the entity is collected meanwhile. + expect(store.isSettled()).toBe(false); + expect(store.hasEntity!(K('User', 'u1'))).toBe(true); + client.entityMap.getEntity(K('User', 'u1'))!.evict(); + client.applyMutationEvent({ + type: 'update', + typename: 'User', + data: { __typename: 'User', id: 'u1', name: 'Alicia', karma: 10 }, + }); + await drain(store); + expect(asyncDoc(delegate, K('User', 'u1'))).toMatchObject({ name: 'Alicia' }); + }); + + it('a reader with its own delegate still purges stale queries directly', async () => { + const delegate = new AsyncDelegate(); + delegate.kv['sq:doc:lastUsed:old-def'] = 1; + delegate.kv['sq:doc:cacheTime:old-def'] = 1; + delegate.kv[queueKeyFor('old-def')] = new Uint32Array([9, 0]); + delegate.kv[valueKeyFor(9)] = JSON.stringify({ stale: true }); + delegate.kv[updatedAtKeyFor(9)] = 1; + const reader = new AsyncQueryStore({ + isWriter: false, + delegate, + connect: () => ({ sendMessage: () => {} }), + }); + await reader.purgeStaleQueries(); + expect(delegate.kv[valueKeyFor(9)]).toBeUndefined(); + expect(delegate.kv[queueKeyFor('old-def')]).toBeUndefined(); + }); + + it('a store without mergeEntity gets the whole in-memory data of an event-built entity', async () => { + const kv = new MemoryPersistentStore(); + const inner = new SyncQueryStore(kv); + // A custom store forwarding the pre-0.6 surface only. + const custom = { + loadQuery: inner.loadQuery.bind(inner), + saveQuery: inner.saveQuery.bind(inner), + saveEntity: (key: number, value: unknown, refIds?: Set) => inner.saveEntity(key, value, refIds), + activateQuery: inner.activateQuery.bind(inner), + deleteQuery: inner.deleteQuery.bind(inner), + hasEntity: inner.hasEntity.bind(inner), + }; + const mockFetch = createMockFetch(); + mockFetch.get('/user/u1', { + user: { + __typename: 'User', + id: 'u1', + name: 'Alice', + karma: 10, + badge: { __typename: 'Badge', id: 'b1', label: 'gold' }, + }, + }); + const client = new QueryClient({ + store: custom, + adapters: [new RESTQueryAdapter({ fetch: mockFetch as never, baseUrl: 'http://localhost' })], + log: { warn: () => {}, error: () => {} }, + } as never); + clients.push(client); + await holdQuery(client, () => fetchQuery(GetUser, { id: 'u1' })); + client.entityMap.getEntity(K('User', 'u1'))!.evict(); + client.applyMutationEvent({ + type: 'update', + typename: 'User', + data: { __typename: 'User', id: 'u1', name: 'Alicia', karma: 10 }, + }); + await sleep(5); + // Written whole (the badge the event did not carry is gone, as before + // 0.6), never as a record holding only the event's fields. + expect(getDoc(kv, K('User', 'u1'))).toMatchObject({ __typename: 'User', id: 'u1', name: 'Alicia', karma: 10 }); + expect(getDoc(kv, K('User', 'u1'))!.badge).toBeUndefined(); + }); + + it('runs purgeStaleQueries through its queue, after the writes queued before it', async () => { + const delegate = new AsyncDelegate(); + // A definition last used long ago, with one query whose record references entity 5. + delegate.kv['sq:doc:lastUsed:old-def'] = 1; + delegate.kv['sq:doc:cacheTime:old-def'] = 1; + delegate.kv[queueKeyFor('old-def')] = new Uint32Array([9, 0, 0]); + delegate.kv[valueKeyFor(9)] = JSON.stringify({ user: { __entityRef: 5 } }); + delegate.kv[updatedAtKeyFor(9)] = 1; + delegate.kv[refIdsKeyFor(9)] = new Uint32Array([5]); + delegate.kv[valueKeyFor(5)] = JSON.stringify({ name: 'old' }); + delegate.kv[refCountKeyFor(5)] = 1; + const store = writerStore(delegate); + const log: string[] = []; + store.onPersisted!(key => log.push(`ack ${key}`)); + store.onDelete!(key => log.push(`delete ${key}`)); + + store.saveEntity(5, { name: 'new' }); + const purged = store.purgeStaleQueries(); + expect(store.isSettled()).toBe(false); + await purged; + + // The write was acknowledged before the purge dropped the record, so the + // client that heard both ends up with the record marked missing. + expect(log).toEqual(['ack 5', 'delete 9', 'delete 5']); + expect(delegate.kv[valueKeyFor(5)]).toBeUndefined(); + expect(store.hasEntity!(5)).toBe(false); + }); + + it('shrinking the LRU while activating a key among the dropped slots keeps that key', async () => { + const delegate = new AsyncDelegate(); + const defId = 'shrink-def'; + const def = (maxCount: number) => ({ statics: { id: defId, cache: { maxCount } } }) as never; + delegate.kv[queueKeyFor(defId)] = new Uint32Array([4, 3, 2, 1]); + for (const key of [4, 3, 2, 1]) { + delegate.kv[valueKeyFor(key)] = JSON.stringify({ __entityRef: key }); + delegate.kv[updatedAtKeyFor(key)] = 1; + } + const store = writerStore(delegate); + store.activateQuery(def(3), 1); + await drain(store); + + expect(Array.from(delegate.kv[queueKeyFor(defId)] as Uint32Array)).toEqual([1, 4, 3]); + expect(delegate.kv[valueKeyFor(1)]).toBeDefined(); + expect(delegate.kv[valueKeyFor(2)]).toBeUndefined(); + + // The same for SyncQueryStore. + const kv = new MemoryPersistentStore(); + kv.setBuffer(queueKeyFor(defId), new Uint32Array([4, 3, 2, 1])); + for (const key of [4, 3, 2, 1]) { + kv.setString(valueKeyFor(key), JSON.stringify({ __entityRef: key })); + kv.setNumber(updatedAtKeyFor(key), 1); + } + new SyncQueryStore(kv).activateQuery(def(3), 1); + expect(Array.from(kv.getBuffer(queueKeyFor(defId))!)).toEqual([1, 4, 3]); + expect(kv.getString(valueKeyFor(1))).toBeDefined(); + expect(kv.getString(valueKeyFor(2))).toBeUndefined(); + }); + + it('a message without maxCount keeps the persisted queue at its size', async () => { + const delegate = new AsyncDelegate(); + delegate.kv[queueKeyFor('legacy-def')] = new Uint32Array([4, 3, 2, 1]); + for (const key of [4, 3, 2, 1]) { + delegate.kv[valueKeyFor(key)] = JSON.stringify({ __entityRef: key }); + delegate.kv[updatedAtKeyFor(key)] = 1; + } + delegate.kv[valueKeyFor(7)] = JSON.stringify({ __entityRef: 7 }); + const store = writerStore(delegate); + (store as unknown as { handleMessage(msg: unknown): void }).handleMessage({ + type: 2, + queryDefId: 'legacy-def', + queryKey: 7, + cacheTime: 1440, + }); + await drain(store); + + const queue = delegate.kv[queueKeyFor('legacy-def')] as Uint32Array; + expect(Array.from(queue)).toEqual([7, 4, 3, 2]); + expect(delegate.kv[valueKeyFor(1)]).toBeUndefined(); + expect(delegate.kv[valueKeyFor(2)]).toBeDefined(); + }); + + it('an acknowledgement only counts against a write the instance dispatched itself', async () => { + const delegate = new AsyncDelegate(); + const store = writerStore(delegate); + const mockFetch = createMockFetch(); + mockFetch.get('/user/u1', { user: { __typename: 'User', id: 'u1', name: 'Alice', karma: 10 } }); + const client = makeClient(store, mockFetch); + await holdQuery(client, () => fetchQuery(GetUser, { id: 'u1' })); + const first = client.entityMap.getEntity(K('User', 'u1'))!; + expect(first._pendingWrites).toBe(1); + expect(first._persisted).toBe(false); + await drain(store); + expect(first._pendingWrites).toBe(0); + expect(first._persisted).toBe(true); + + // An instance that never wrote (hydrated from the store) ignores a stray acknowledgement. + first.recordDropped(); + first.acknowledgeWrite(); + expect(first._persisted).toBe(false); + + // Two writes in flight: the record is current once both have landed. + first.save(); + first.save(); + expect(first._pendingWrites).toBe(2); + await drain(store); + expect(first._pendingWrites).toBe(0); + expect(first._persisted).toBe(true); + }); +}); diff --git a/packages/fetchium/src/__tests__/parsing/entity.test.ts b/packages/fetchium/src/__tests__/parsing/entity.test.ts index 11fb3e8..0a0cea0 100644 --- a/packages/fetchium/src/__tests__/parsing/entity.test.ts +++ b/packages/fetchium/src/__tests__/parsing/entity.test.ts @@ -39,11 +39,11 @@ describe('Entity parsing', () => { }, }; - await parseEntities(result, QueryResult, client, new Map()); + const parsed = (await parseEntities(result, QueryResult, client, new Map())) as { data: unknown }; - expect(result.data).toBeInstanceOf(Entity); - expect(result.data).toBeInstanceOf(User); - expect(Object.getPrototypeOf(result.data)).toBe(User.prototype); + expect(parsed.data).toBeInstanceOf(Entity); + expect(parsed.data).toBeInstanceOf(User); + expect(Object.getPrototypeOf(parsed.data)).toBe(User.prototype); }); describe('nested entities', () => { diff --git a/packages/fetchium/src/__tests__/same-value.test.ts b/packages/fetchium/src/__tests__/same-value.test.ts index 89f6286..3fadb13 100644 --- a/packages/fetchium/src/__tests__/same-value.test.ts +++ b/packages/fetchium/src/__tests__/same-value.test.ts @@ -42,8 +42,11 @@ describe('sameValue', () => { expect(sameValue(proxy, proxy)).toBe(true); }); - it('reports NaN as changed, since it is not equal to itself', () => { - expect(sameValue(Number.NaN, Number.NaN)).toBe(false); + it('treats NaN as unchanged and -0 as a change (Object.is semantics)', () => { + // Re-applying a NaN field must not notify on every refetch, and the sign + // of zero is observable through `Object.is` on a proxy read. + expect(sameValue(Number.NaN, Number.NaN)).toBe(true); + expect(sameValue(0, -0)).toBe(false); }); const different: [string, unknown, unknown][] = [ diff --git a/packages/fetchium/src/__tests__/skip-write-and-snapshot-fixes.test.ts b/packages/fetchium/src/__tests__/skip-write-and-snapshot-fixes.test.ts new file mode 100644 index 0000000..c9664f3 --- /dev/null +++ b/packages/fetchium/src/__tests__/skip-write-and-snapshot-fixes.test.ts @@ -0,0 +1,1247 @@ +import { describe, it, expect, vi, afterEach } from 'vitest'; +import { watcher, withContexts } from 'signalium'; +import { hashValue, snapshot } from 'signalium/utils'; +import { t, registerFormat } from '../typeDefs.js'; +import { Mask } from '../types.js'; +import { Entity } from '../proxy.js'; +import { RESTQuery } from '../rest/index.js'; +import { RESTQueryAdapter } from '../rest/RESTQueryAdapter.js'; +import { fetchQuery } from '../query.js'; +import { QueryClient, QueryClientContext } from '../QueryClient.js'; +import { SyncQueryStore, MemoryPersistentStore } from '../stores/sync.js'; +import { AsyncQueryStore, type AsyncPersistentStore } from '../stores/async.js'; +import { valueKeyFor, refIdsKeyFor, queueKeyFor, DEFAULT_MAX_COUNT } from '../stores/shared.js'; +import { createMockFetch, setupTestClient, testWithClient, sleep } from './utils.js'; + +/** + * Regression coverage for the follow-ups to the snapshot fast path (#59), + * unchanged-apply skipping (#60), single-write events (#61) and store + * deletion reporting (#64). Each case failed on the tree with those PRs and + * passed on the tree before them. + */ + +function getDoc(kv: MemoryPersistentStore, key: number): Record | undefined { + const value = kv.getString(valueKeyFor(key)); + return value ? (JSON.parse(value) as Record) : undefined; +} + +/** Holds a query active and exposes `useReactive`-style snapshots of it. */ +function snapshotHarness(client: QueryClient, start: () => T) { + return withContexts([[QueryClientContext, client]], () => { + const query = start(); + let prev: unknown; + let computes = 0; + const snapshots = watcher(() => { + computes++; + return (prev = snapshot(query, prev)); + }); + snapshots.addListener(() => {}); + return { + query, + read: () => (snapshots.value as { value: any }).value, + computes: () => computes, + }; + }); +} + +function holdQuery(client: QueryClient, start: () => T): T { + return withContexts([[QueryClientContext, client]], () => { + const query = start(); + const w = watcher(() => (query as unknown as { value: unknown }).value); + w.addListener(() => {}); + return query; + }); +} + +function makeClient(store: SyncQueryStore | AsyncQueryStore, mockFetch: ReturnType) { + const warn = vi.fn(); + const client = new QueryClient({ + store, + adapters: [new RESTQueryAdapter({ fetch: mockFetch as never, baseUrl: 'http://localhost' })], + log: { warn, error: () => {} }, + } as never); + return { client, warn }; +} + +// ====================================================== +// Cache hydration must not re-parse an entity that is already in memory +// ====================================================== + +describe('hydration of a cached query whose entity is already in memory', () => { + const getClient = setupTestClient(); + class User extends Entity { + __typename = t.typename('User'); + id = t.id; + name = t.string; + profile = t.object({ city: t.string, since: t.optional(t.format('date')) }); + } + class GetA extends RESTQuery { + path = '/a'; + result = { user: t.entity(User) }; + config = { staleTime: 60_000 }; + } + class GetB extends RESTQuery { + path = '/b'; + result = { user: t.entity(User), extra: t.string }; + config = { staleTime: 60_000 }; + } + const payload = { __typename: 'User', id: 1, name: 'Alice', profile: { city: 'Rome', since: '2020-01-01' } }; + + it('keeps nested formatted values, notifies nothing, and leaves every consumer consistent', async () => { + const kv = new MemoryPersistentStore(); + const store = new SyncQueryStore(kv); + const mockFetch = createMockFetch(); + mockFetch.get('/a', { user: payload }); + mockFetch.get('/b', { user: payload, extra: 'e' }); + + // Session 1 persists both queries. + const { client: first } = makeClient(store, mockFetch); + await holdQuery(first, () => fetchQuery(GetA)); + await holdQuery(first, () => fetchQuery(GetB)); + first.destroy(); + + // Session 2: the network is unreachable; A hydrates, then B hydrates the same entity. + mockFetch.get('/a', { user: payload }, { delay: 10_000 }); + mockFetch.get('/b', { user: payload, extra: 'e' }, { delay: 10_000 }); + const { client, warn } = makeClient(store, mockFetch); + const error = vi.spyOn(console, 'error').mockImplementation(() => {}); + try { + const hA = snapshotHarness(client, () => fetchQuery(GetA)); + await sleep(20); + const a1 = hA.read(); + expect(a1.user.profile.since).toBeInstanceOf(Date); + const computesBefore = hA.computes(); + + const hB = snapshotHarness(client, () => fetchQuery(GetB)); + await sleep(20); + + // The in-memory data was not re-parsed: the proxy, the store record and + // both consumers agree, and A's consumer had no reason to recompute. + const instance = client.entityMap.getEntity(hashValue(['User', 1]))!; + expect((instance.data.profile as { since: unknown }).since).toBeDefined(); + expect((hA.query.value as any).user.profile.since).toBeInstanceOf(Date); + expect(hB.read().user.profile.since).toBeInstanceOf(Date); + expect(hA.read().user.profile.since).toBe(a1.user.profile.since); + expect(hA.computes()).toBe(computesBefore); + expect(warn).not.toHaveBeenCalled(); + expect(error).not.toHaveBeenCalled(); + } finally { + error.mockRestore(); + client.destroy(); + } + }); + + it('rejects the cached query when the in-memory entity lacks a field the query requires', async () => { + class Narrow extends Entity { + __typename = t.typename('Doc'); + id = t.id; + title = t.string; + } + class Wide extends Entity { + __typename = t.typename('Doc'); + id = t.id; + title = t.string; + body = t.string; + } + class GetNarrow extends RESTQuery { + path = '/narrow'; + result = { doc: t.entity(Narrow) }; + } + class GetWide extends RESTQuery { + path = '/wide'; + result = { doc: t.entity(Wide) }; + config = { staleTime: 60_000 }; + } + const kv = new MemoryPersistentStore(); + const store = new SyncQueryStore(kv); + const mockFetch = createMockFetch(); + mockFetch.get('/wide', { doc: { __typename: 'Doc', id: 1, title: 'T', body: 'B' } }); + mockFetch.get('/narrow', { doc: { __typename: 'Doc', id: 1, title: 'T' } }); + + const { client: first } = makeClient(store, mockFetch); + await holdQuery(first, () => fetchQuery(GetWide)); + first.destroy(); + + // The narrow query fetches fresh, so Doc:1 is in memory without `body`. + const { client, warn } = makeClient(store, mockFetch); + await holdQuery(client, () => fetchQuery(GetNarrow)); + mockFetch.get('/wide', { doc: { __typename: 'Doc', id: 1, title: 'T', body: 'B2' } }); + const wide = holdQuery(client, () => fetchQuery(GetWide)); + await wide; + // The cache was dropped with a warning and the network answered instead. + expect(warn).toHaveBeenCalledWith(expect.stringContaining('query cache may be corrupted'), expect.anything()); + expect((wide.value as any).doc.body).toBe('B2'); + client.destroy(); + }); + + it('rejects the cached query when the in-memory entity lacks a nested entity the query requires', async () => { + class Org extends Entity { + __typename = t.typename('Org'); + id = t.id; + name = t.string; + } + class UserSummary extends Entity { + __typename = t.typename('User'); + id = t.id; + meta = t.object({ x: t.number }); + } + class UserFull extends Entity { + __typename = t.typename('User'); + id = t.id; + meta = t.object({ x: t.number, owner: t.entity(Org) }); + } + class GetSummary extends RESTQuery { + path = '/summary'; + result = { user: t.entity(UserSummary) }; + config = { staleTime: 60_000 }; + } + class GetFull extends RESTQuery { + path = '/full'; + result = { user: t.entity(UserFull) }; + config = { staleTime: 60_000 }; + } + const full = { __typename: 'User', id: 1, meta: { x: 1, owner: { __typename: 'Org', id: 'o1', name: 'Acme' } } }; + const kv = new MemoryPersistentStore(); + const store = new SyncQueryStore(kv); + const mockFetch = createMockFetch(); + mockFetch.get('/summary', { user: { __typename: 'User', id: 1, meta: { x: 1 } } }); + mockFetch.get('/full', { user: full }); + + const { client: first } = makeClient(store, mockFetch); + await holdQuery(first, () => fetchQuery(GetSummary)); + await holdQuery(first, () => fetchQuery(GetFull)); + first.destroy(); + + // Session 2: the summary hydrates User:1 without `meta.owner`; the full + // query must not resolve from the cache with the owner missing. + mockFetch.get('/full', { user: { ...full, meta: { ...full.meta, x: 2 } } }, { delay: 50 }); + const { client, warn } = makeClient(store, mockFetch); + const summary = holdQuery(client, () => fetchQuery(GetSummary)); + await sleep(10); + expect((summary.value as any).user.meta.x).toBe(1); + + const wide = holdQuery(client, () => fetchQuery(GetFull)); + await sleep(10); + expect(wide.value).toBeUndefined(); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('query cache may be corrupted'), expect.anything()); + await wide; + expect((wide.value as any).user.meta.owner.name).toBe('Acme'); + expect((wide.value as any).user.meta.x).toBe(2); + client.destroy(); + }); + + it('a cached list holding an in-memory entity the query cannot use is refetched whole, not served with a hole', async () => { + class Narrow extends Entity { + __typename = t.typename('Doc'); + id = t.id; + title = t.string; + } + class Wide extends Entity { + __typename = t.typename('Doc'); + id = t.id; + title = t.string; + body = t.string; + } + class GetNarrow extends RESTQuery { + params = { id: t.id }; + path = `/narrow/${this.params.id}`; + result = { doc: t.entity(Narrow) }; + } + class GetDocs extends RESTQuery { + path = '/docs'; + result = { docs: t.array(t.entity(Wide)) }; + config = { staleTime: 60_000 }; + } + const kv = new MemoryPersistentStore(); + const store = new SyncQueryStore(kv); + const mockFetch = createMockFetch(); + const docs = [ + { __typename: 'Doc', id: 1, title: 'T1', body: 'B1' }, + { __typename: 'Doc', id: 2, title: 'T2', body: 'B2' }, + ]; + mockFetch.get('/docs', { docs }); + mockFetch.get('/narrow/1', { doc: { __typename: 'Doc', id: 1, title: 'T1' } }); + + const { client: first } = makeClient(store, mockFetch); + await holdQuery(first, () => fetchQuery(GetDocs)); + first.destroy(); + + // Doc:1 comes into memory through the narrow query, without `body`. + const { client, warn } = makeClient(store, mockFetch); + await holdQuery(client, () => fetchQuery(GetNarrow, { id: 1 })); + mockFetch.get('/docs', { docs: docs.map(d => ({ ...d, body: `${d.body}!` })) }); + const list = holdQuery(client, () => fetchQuery(GetDocs)); + await list; + expect(warn).toHaveBeenCalledWith(expect.stringContaining('query cache may be corrupted'), expect.anything()); + expect((list.value as any).docs.map((d: any) => d.body)).toEqual(['B1!', 'B2!']); + client.destroy(); + }); + + it('a member the query narrows out anyway does not reject the cached query', async () => { + class UserPreview extends Entity { + __typename = t.typename('User'); + id = t.id; + name = t.string; + } + class UserFull extends Entity { + __typename = t.typename('User'); + id = t.id; + name = t.string; + email = t.string; + } + class Team extends Entity { + __typename = t.typename('Team'); + id = t.id; + owner = t.entity(UserPreview); + members = t.array(t.entity(UserFull)); + } + class GetTeam extends RESTQuery { + path = '/team'; + result = { team: t.entity(Team) }; + config = { staleTime: 60_000 }; + } + class GetOwner extends RESTQuery { + path = '/owner'; + result = { user: t.entity(UserPreview) }; + } + const kv = new MemoryPersistentStore(); + const store = new SyncQueryStore(kv); + const mockFetch = createMockFetch(); + mockFetch.get('/team', { + team: { + __typename: 'Team', + id: 't-1', + owner: { __typename: 'User', id: 'u-1', name: 'Alice' }, + members: [ + { __typename: 'User', id: 'u-1', name: 'Alice' }, + { __typename: 'User', id: 'u-2', name: 'Bob', email: 'b@x' }, + ], + }, + }); + mockFetch.get('/owner', { user: { __typename: 'User', id: 'u-1', name: 'Alice' } }); + const { client: first } = makeClient(store, mockFetch); + await holdQuery(first, () => fetchQuery(GetTeam)); + first.destroy(); + + // u-1 is in memory without `email`; the members array narrows it out on + // read, so the cached team is still usable as it is. + const { client, warn } = makeClient(store, mockFetch); + await holdQuery(client, () => fetchQuery(GetOwner)); + mockFetch.get( + '/team', + { team: { __typename: 'Team', id: 't-1', owner: { __typename: 'User', id: 'u-1', name: 'FRESH' }, members: [] } }, + { delay: 5000 }, + ); + const team = holdQuery(client, () => fetchQuery(GetTeam)); + await sleep(20); + expect(warn).not.toHaveBeenCalled(); + expect((team.value as any).team.members.map((m: any) => m.id)).toEqual(['u-2']); + expect((team.value as any).team.owner.name).toBe('Alice'); + client.destroy(); + }); + + it('an entity built from events is completed from its record when a cached query hydrates it', async () => { + class Post extends Entity { + __typename = t.typename('Post'); + id = t.id; + feedId = t.string; + title = t.string; + body = t.optional(t.string); + tags = t.optional(t.array(t.string)); + } + class GetPost extends RESTQuery { + params = { id: t.id }; + path = `/post/${this.params.id}`; + result = { post: t.entity(Post) }; + config = { staleTime: 60_000 }; + } + class Feed extends Entity { + __typename = t.typename('Feed'); + id = t.id; + posts = t.liveArray(Post, { constraints: { feedId: (this as any).id } }); + } + class GetFeed extends RESTQuery { + params = { id: t.id }; + path = `/feed/${this.params.id}`; + result = { feed: t.entity(Feed) }; + } + const kv = new MemoryPersistentStore(); + const store = new SyncQueryStore(kv); + const mockFetch = createMockFetch(); + mockFetch.get('/post/p1', { + post: { __typename: 'Post', id: 'p1', feedId: 'f1', title: 'T', body: 'hello', tags: ['a', 'b'] }, + }); + mockFetch.get('/feed/f1', { feed: { __typename: 'Feed', id: 'f1', posts: [] } }); + const { client: first } = makeClient(store, mockFetch); + await holdQuery(first, () => fetchQuery(GetPost, { id: 'p1' })); + first.destroy(); + + // The post comes into memory through an event routed into the feed: + // title only. Then the cached post query hydrates the same entity. + const { client, warn } = makeClient(store, mockFetch); + const feed = holdQuery(client, () => fetchQuery(GetFeed, { id: 'f1' })); + await feed; + client.applyMutationEvent({ + type: 'create', + typename: 'Post', + data: { __typename: 'Post', id: 'p1', feedId: 'f1', title: 'T2' }, + }); + await sleep(5); + expect((feed.value as any).feed.posts.map((p: any) => p.title)).toEqual(['T2']); + const instance = client.entityMap.getEntity(hashValue(['Post', 'p1']))!; + expect(instance._partial).toBe(true); + expect(instance.data.body).toBeUndefined(); + + mockFetch.get( + '/post/p1', + { post: { __typename: 'Post', id: 'p1', title: 'FRESH', body: 'fresh' } }, + { delay: 5000 }, + ); + const post = holdQuery(client, () => fetchQuery(GetPost, { id: 'p1' })); + await sleep(20); + expect(warn).not.toHaveBeenCalled(); + // The event's title stays; the fields it did not carry come from the record. + expect((post.value as any).post.title).toBe('T2'); + expect((post.value as any).post.body).toBe('hello'); + expect((post.value as any).post.tags).toEqual(['a', 'b']); + expect((feed.value as any).feed.posts[0].body).toBe('hello'); + expect(instance._partial).toBe(false); + client.destroy(); + }); + + it('a frozen payload with undeclared nested values is applied without being written to', async () => { + const { client, mockFetch } = getClient(); + class Doc extends Entity { + __typename = t.typename('Doc'); + id = t.id; + title = t.string; + meta = t.object({ views: t.number }); + } + class GetDoc extends RESTQuery { + path = '/doc'; + result = { doc: t.entity(Doc) }; + } + mockFetch.get('/doc', { doc: { __typename: 'Doc', id: 'd', title: 'T', meta: { views: 1, deep: { x: [1] } } } }); + const h = snapshotHarness(client, () => fetchQuery(GetDoc)); + await h.query; + const snap = h.read().doc; + const warn = vi.spyOn(client.getContext().log as { warn: (...args: unknown[]) => void }, 'warn'); + + const frozen = Object.freeze({ + id: 'd', + title: 'T2', + meta: Object.freeze({ views: 2, deep: Object.freeze({ x: Object.freeze([2]) }) }), + }); + client.applyMutationEvent({ type: 'update', typename: 'Doc', data: frozen }); + expect(warn).not.toHaveBeenCalled(); + expect(h.read().doc.title).toBe('T2'); + expect(h.read().doc.meta.views).toBe(2); + expect(snap.meta.views).toBe(1); + }); +}); + +// ====================================================== +// Streamed events: what an event changes in an existing entity is written +// ====================================================== + +describe('mutation events and the store', () => { + const getClient = setupTestClient(); + + class User extends Entity { + __typename = t.typename('User'); + id = t.id; + name = t.string; + karma = t.number; + badge = t.optional(t.entity(Badge)); + } + class Badge extends Entity { + __typename = t.typename('Badge'); + id = t.id; + label = t.string; + } + class Comment extends Entity { + __typename = t.typename('Comment'); + id = t.id; + postId = t.string; + body = t.string; + author = t.entity(User); + } + class Post extends Entity { + __typename = t.typename('Post'); + id = t.id; + title = t.string; + comments = t.liveArray(Comment, { constraints: { postId: (this as any).id } }); + } + class GetPost extends RESTQuery { + params = { id: t.id }; + path = `/post/${this.params.id}`; + result = { post: t.entity(Post) }; + } + class GetUser extends RESTQuery { + params = { id: t.id }; + path = `/user/${this.params.id}`; + result = { user: t.entity(User) }; + } + const u1Key = hashValue(['User', 'u1']); + const badgeKey = hashValue(['Badge', 'b1']); + + async function seed(client: QueryClient, mockFetch: ReturnType, withPost: boolean) { + mockFetch.get('/user/u1', { user: { __typename: 'User', id: 'u1', name: 'Alice', karma: 10 } }); + mockFetch.get('/post/p1', { post: { __typename: 'Post', id: 'p1', title: 'T', comments: [] } }); + const userQ = holdQuery(client, () => fetchQuery(GetUser, { id: 'u1' })); + await userQ; + const postQ = withPost ? holdQuery(client, () => fetchQuery(GetPost, { id: 'p1' })) : undefined; + if (postQ) await postQ; + return { userQ, postQ }; + } + + const createEvent = (withBadge: boolean) => ({ + type: 'create' as const, + typename: 'Comment', + data: { + __typename: 'Comment', + id: 'c9', + postId: 'p1', + body: 'hi', + author: { + __typename: 'User', + id: 'u1', + name: 'Alice', + karma: 11, + ...(withBadge ? { badge: { __typename: 'Badge', id: 'b1', label: 'gold' } } : {}), + }, + }, + }); + + it('a create routed into a live array writes the existing author it changed, and the new badge under it', async () => { + const { client, mockFetch, kv } = getClient(); + await seed(client, mockFetch, true); + expect(getDoc(kv, u1Key)!.karma).toBe(10); + + client.applyMutationEvent(createEvent(true)); + await sleep(5); + + expect(getDoc(kv, u1Key)!.karma).toBe(11); + expect(getDoc(kv, badgeKey)).toMatchObject({ label: 'gold' }); + expect(Array.from(kv.getBuffer(refIdsKeyFor(u1Key)) ?? [])).toEqual([badgeKey]); + expect(getDoc(kv, hashValue(['Comment', 'c9']))).toMatchObject({ body: 'hi' }); + }); + + it('a create nothing routes still writes the existing author it changed, and writes no orphan', async () => { + const { client, mockFetch, kv } = getClient(); + await seed(client, mockFetch, true); + + // The post's live array is constrained to postId p1; this comment is for another post. + const event = createEvent(false); + client.applyMutationEvent({ ...event, data: { ...event.data, postId: 'p2' } }); + await sleep(5); + + expect(getDoc(kv, u1Key)!.karma).toBe(11); + expect(getDoc(kv, hashValue(['Comment', 'c9']))).toBeUndefined(); + expect(client.entityMap.getEntity(hashValue(['Comment', 'c9']))).toBeUndefined(); + }); + + it('a full-payload update for an entity on disk but not in memory refreshes its record', async () => { + const { client, mockFetch, kv, store } = getClient(); + await seed(client, mockFetch, false); + // Evict User u1 from memory while its record stays referenced by the cached query. + client.entityMap.getEntity(u1Key)!.evict(); + expect(client.entityMap.getEntity(u1Key)).toBeUndefined(); + expect(getDoc(kv, u1Key)!.name).toBe('Alice'); + + client.applyMutationEvent({ + type: 'update', + typename: 'User', + data: { __typename: 'User', id: 'u1', name: 'Alicia', karma: 12 }, + }); + await sleep(5); + + expect(client.entityMap.getEntity(u1Key)).toBeUndefined(); + expect(getDoc(kv, u1Key)).toMatchObject({ name: 'Alicia', karma: 12 }); + expect(store.hasEntity(u1Key)).toBe(true); + + // An update for an entity the store has never seen leaves nothing behind. + client.applyMutationEvent({ + type: 'update', + typename: 'User', + data: { __typename: 'User', id: 'u404', name: 'Nobody', karma: 0 }, + }); + await sleep(5); + expect(getDoc(kv, hashValue(['User', 'u404']))).toBeUndefined(); + }); +}); + +// ====================================================== +// `_persisted` is only trusted while the store agrees +// ====================================================== + +describe('write skipping stays consistent with the store', () => { + class User extends Entity { + __typename = t.typename('User'); + id = t.id; + name = t.string; + } + class GetUser extends RESTQuery { + path = '/user'; + result = { user: t.entity(User) }; + } + const userKey = hashValue(['User', 1]); + + it('a store write that throws is retried by the next unchanged apply', async () => { + const kv = new MemoryPersistentStore(); + const store = new SyncQueryStore(kv); + const mockFetch = createMockFetch(); + mockFetch.get('/user', { user: { __typename: 'User', id: 1, name: 'Alice' } }); + const { client } = makeClient(store, mockFetch); + + const relay = holdQuery(client, () => fetchQuery(GetUser)); + await relay; + expect(getDoc(kv, userKey)).toMatchObject({ name: 'Alice' }); + + // The next refetch changes the name, and the store rejects that write. + const setString = kv.setString.bind(kv); + let failing = true; + vi.spyOn(kv, 'setString').mockImplementation((key, value) => { + if (failing && key === valueKeyFor(userKey)) throw new Error('quota exceeded'); + setString(key, value); + }); + mockFetch.get('/user', { user: { __typename: 'User', id: 1, name: 'Alicia' } }); + await expect((relay.value as any).__refetch()).rejects.toThrow('quota exceeded'); + expect((relay.value as any)?.user?.name ?? client.entityMap.getEntity(userKey)!.data.name).toBe('Alicia'); + expect(getDoc(kv, userKey)).toMatchObject({ name: 'Alice' }); + + // The store recovers; the refetch returns the same data the entity already + // holds, so nothing changed in memory, yet the record must be rewritten. + failing = false; + await withContexts([[QueryClientContext, client]], () => fetchQuery(GetUser).value ?? undefined); + await (client as any).queryInstances.values().next().value.refetch(); + await sleep(20); + expect(getDoc(kv, userKey)).toMatchObject({ name: 'Alicia' }); + client.destroy(); + }); + + it('QueryClient.destroy() removes its store listeners', () => { + const store = new SyncQueryStore(new MemoryPersistentStore()); + const listeners = () => (store as unknown as { deleteListeners: unknown[] }).deleteListeners.length; + const clients = Array.from({ length: 5 }, () => new QueryClient({ store })); + expect(listeners()).toBe(5); + for (const client of clients) client.destroy(); + expect(listeners()).toBe(0); + }); +}); + +// ====================================================== +// AsyncQueryStore writer +// ====================================================== + +class AsyncDelegate implements AsyncPersistentStore { + readonly kv: Record = Object.create(null); + failNextSetString: string | undefined; + private async tick(): Promise { + await new Promise(resolve => setTimeout(resolve, 0)); + } + async has(key: string) { + await this.tick(); + return key in this.kv; + } + async getString(key: string) { + await this.tick(); + return this.kv[key] as string | undefined; + } + async setString(key: string, value: string) { + await this.tick(); + if (this.failNextSetString === key) { + this.failNextSetString = undefined; + throw new Error('injected failure'); + } + this.kv[key] = value; + } + async getNumber(key: string) { + await this.tick(); + return this.kv[key] as number | undefined; + } + async setNumber(key: string, value: number) { + await this.tick(); + this.kv[key] = value; + } + async getBuffer(key: string) { + await this.tick(); + return this.kv[key] as Uint32Array | undefined; + } + async setBuffer(key: string, value: Uint32Array) { + await this.tick(); + this.kv[key] = value; + } + async delete(key: string) { + await this.tick(); + delete this.kv[key]; + } + async getAllKeys() { + await this.tick(); + return Object.keys(this.kv); + } +} + +function writerStore(delegate: AsyncPersistentStore): AsyncQueryStore { + return new AsyncQueryStore({ + isWriter: true, + delegate, + connect: handleMessage => ({ sendMessage: msg => handleMessage(msg) }), + }); +} + +async function drain(store: AsyncQueryStore): Promise { + for (let i = 0; i < 5000; i++) { + if (store.isSettled()) return; + await sleep(2); + } + throw new Error('writer queue did not drain'); +} + +describe('AsyncQueryStore writer used in-process', () => { + const clients: QueryClient[] = []; + afterEach(() => { + for (const client of clients.splice(0)) client.destroy(); + }); + + class User extends Entity { + __typename = t.typename('User'); + id = t.id; + name = t.string; + } + class GetProfile extends RESTQuery { + static cache = { maxCount: 2 }; + params = { id: t.id }; + path = `/user/profile/${this.params.id}`; + result = { user: t.entity(User) }; + } + const userKey = (id: string | number) => hashValue(['User', id]); + + it('an unchanged refetch racing a queued eviction still leaves the query hydratable', async () => { + const delegate = new AsyncDelegate(); + const store = writerStore(delegate); + const mockFetch = createMockFetch(); + for (const id of ['1', '2', '3']) { + mockFetch.get(`/user/profile/${id}`, { user: { __typename: 'User', id, name: `User ${id}` } }); + } + const { client, warn } = makeClient(store, mockFetch); + clients.push(client); + + const q1 = holdQuery(client, () => fetchQuery(GetProfile, { id: '1' })); + await q1; + await holdQuery(client, () => fetchQuery(GetProfile, { id: '2' })); + await drain(store); + expect(delegate.kv[valueKeyFor(userKey('1'))]).toBeDefined(); + + // Q3's SaveQuery will evict Q1 and cascade to User 1 once processed. Before + // the writer gets there, Q1 refetches identical data. + await holdQuery(client, () => fetchQuery(GetProfile, { id: '3' })); + expect(store.isSettled()).toBe(false); + await (q1.value as any).__refetch(); + await drain(store); + + expect(delegate.kv[valueKeyFor(userKey('1'))]).toBeDefined(); + expect(warn).not.toHaveBeenCalled(); + + // Cold start on the same store: Q1 hydrates from cache. + client.destroy(); + clients.pop(); + mockFetch.reset(); + mockFetch.get('/user/profile/1', { user: { __typename: 'User', id: '1', name: 'Refetched' } }, { delay: 2000 }); + const cold = makeClient(store, mockFetch); + clients.push(cold.client); + const relay = holdQuery(cold.client, () => fetchQuery(GetProfile, { id: '1' })); + await sleep(150); + expect(cold.warn).not.toHaveBeenCalled(); + expect((relay.value as any)?.user?.name).toBe('User 1'); + }); + + it('a SaveEntity the delegate rejected is written again by the next unchanged apply', async () => { + const delegate = new AsyncDelegate(); + const store = writerStore(delegate); + const mockFetch = createMockFetch(); + mockFetch.get('/user/profile/1', { user: { __typename: 'User', id: '1', name: 'Alice' } }); + const { client } = makeClient(store, mockFetch); + clients.push(client); + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {}); + try { + delegate.failNextSetString = valueKeyFor(userKey('1')); + const q1 = holdQuery(client, () => fetchQuery(GetProfile, { id: '1' })); + await q1; + await drain(store); + expect(delegate.kv[valueKeyFor(userKey('1'))]).toBeUndefined(); + + await (q1.value as any).__refetch(); + await drain(store); + expect(delegate.kv[valueKeyFor(userKey('1'))]).toBeDefined(); + } finally { + consoleError.mockRestore(); + } + }); + + it('resizes a persisted LRU queue in both directions and defaults a missing maxCount', async () => { + const delegate = new AsyncDelegate(); + const store = writerStore(delegate); + const defId = 'upgrade-def'; + const def = (maxCount: number) => ({ statics: { id: defId, cache: { maxCount } } }) as never; + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {}); + try { + // A 4-slot queue from an older install, fully populated, with records behind each key. + const old = new Uint32Array([4, 3, 2, 1]); + delegate.kv[queueKeyFor(defId)] = old; + for (const key of old) { + delegate.kv[valueKeyFor(key)] = JSON.stringify({ __entityRef: key }); + delegate.kv[`sq:doc:updatedAt:${key}`] = 1; + } + delegate.kv[valueKeyFor(9)] = JSON.stringify({ __entityRef: 9 }); + + // Grow: the buffer is reallocated, the order kept, nothing evicted. + store.activateQuery(def(6), 9); + await drain(store); + let queue = delegate.kv[queueKeyFor(defId)] as Uint32Array; + expect(Array.from(queue)).toEqual([9, 4, 3, 2, 1, 0]); + expect(consoleError).not.toHaveBeenCalled(); + + // Shrink (after a restart, so the in-memory queue is not cached): the + // keys that no longer fit are evicted, not stranded. + const restarted = writerStore(delegate); + restarted.activateQuery(def(3), 9); + await drain(restarted); + queue = delegate.kv[queueKeyFor(defId)] as Uint32Array; + expect(Array.from(queue)).toEqual([9, 4, 3]); + expect(delegate.kv[valueKeyFor(2)]).toBeUndefined(); + expect(delegate.kv[valueKeyFor(1)]).toBeUndefined(); + expect(delegate.kv[valueKeyFor(4)]).toBeDefined(); + + // A message from a reader that predates `maxCount` uses the default size. + const fresh = new AsyncDelegate(); + const freshStore = writerStore(fresh); + fresh.kv[valueKeyFor(7)] = JSON.stringify({ __entityRef: 7 }); + (freshStore as unknown as { handleMessage(msg: unknown): void }).handleMessage({ + type: 2, + queryDefId: 'legacy-def', + queryKey: 7, + cacheTime: 1440, + }); + await drain(freshStore); + const legacyQueue = fresh.kv[queueKeyFor('legacy-def')] as Uint32Array; + expect(legacyQueue.length).toBe(DEFAULT_MAX_COUNT); + expect(legacyQueue[0]).toBe(7); + expect(consoleError).not.toHaveBeenCalled(); + } finally { + consoleError.mockRestore(); + } + }); +}); + +// ====================================================== +// Snapshot fast path +// ====================================================== + +describe('snapshot fast path follow-ups', () => { + const getClient = setupTestClient(); + + it('a NaN in a static field neither reports drift nor churns snapshot identity', async () => { + const { client, mockFetch } = getClient(); + // JSON cannot carry NaN; a lenient string-to-number format can. + registerFormat( + 'loose-number', + Mask.STRING, + (value: string) => Number(value), + (value: number) => String(value), + ); + class Stat extends Entity { + __typename = t.typename('Stat'); + id = t.id; + label = t.string; + ratio = t.format('loose-number' as never); + series = t.array(t.format('loose-number' as never)); + } + class GetStats extends RESTQuery { + path = '/stats'; + result = { stats: t.array(t.entity(Stat)) }; + } + mockFetch.get('/stats', { + stats: [ + { __typename: 'Stat', id: 's', label: 'a', ratio: 'N/A', series: ['1', 'N/A'] }, + { __typename: 'Stat', id: 'other', label: 'b', ratio: '1', series: [] }, + ], + }); + const error = vi.spyOn(console, 'error').mockImplementation(() => {}); + try { + const h = snapshotHarness(client, () => fetchQuery(GetStats)); + await h.query; + const first = h.read().stats[0]; + client.applyMutationEvent({ type: 'update', typename: 'Stat', data: { id: 'other', label: 'c' } }); + const second = h.read().stats[0]; + expect(Number.isNaN(second.ratio)).toBe(true); + expect(second).toBe(first); + expect(error).not.toHaveBeenCalled(); + } finally { + error.mockRestore(); + } + }); + + it('two classes sharing a typename: a child entity under a field the other class declares entity-free still reaches the consumer', async () => { + const { client, mockFetch } = getClient(); + class Org extends Entity { + __typename = t.typename('Org'); + id = t.id; + name = t.string; + } + class UserSummary extends Entity { + __typename = t.typename('User'); + id = t.id; + name = t.string; + meta = t.object({ x: t.number }); + } + class UserFull extends Entity { + __typename = t.typename('User'); + id = t.id; + name = t.string; + meta = t.object({ x: t.number, owner: t.entity(Org) }); + } + class GetSummary extends RESTQuery { + path = '/summary'; + result = { user: t.entity(UserSummary) }; + } + class GetFull extends RESTQuery { + path = '/full'; + result = { user: t.entity(UserFull) }; + } + mockFetch.get('/summary', { user: { __typename: 'User', id: 1, name: 'Alice', meta: { x: 1 } } }); + mockFetch.get('/full', { + user: { + __typename: 'User', + id: 1, + name: 'Alice', + meta: { x: 1, owner: { __typename: 'Org', id: 7, name: 'ACME' } }, + }, + }); + const error = vi.spyOn(console, 'error').mockImplementation(() => {}); + try { + const summary = snapshotHarness(client, () => fetchQuery(GetSummary)); + await summary.query; + const full = snapshotHarness(client, () => fetchQuery(GetFull)); + await full.query; + expect(summary.read().user.meta.owner.name).toBe('ACME'); + + client.applyMutationEvent({ type: 'update', typename: 'Org', data: { id: 7, name: 'NEW' } }); + expect(summary.read().user.meta.owner.name).toBe('NEW'); + client.applyMutationEvent({ type: 'update', typename: 'Org', data: { id: 7, name: 'NEWER' } }); + expect(summary.read().user.meta.owner.name).toBe('NEWER'); + // Served by the fast path's own bookkeeping, not by the dev guard's re-read. + expect(error).not.toHaveBeenCalled(); + } finally { + error.mockRestore(); + } + }); + + it('a reactive consumer of a narrowed entity array learns when a member becomes eligible', async () => { + const { client, mockFetch } = getClient(); + class UserPreview extends Entity { + __typename = t.typename('User'); + id = t.id; + name = t.string; + } + class UserFull extends Entity { + __typename = t.typename('User'); + id = t.id; + name = t.string; + email = t.string; + } + class Team extends Entity { + __typename = t.typename('Team'); + id = t.id; + owner = t.entity(UserPreview); + members = t.array(t.entity(UserFull)); + } + class GetTeam extends RESTQuery { + path = '/team'; + result = { team: t.entity(Team) }; + } + class GetUser extends RESTQuery { + params = { id: t.id }; + path = `/user/${this.params.id}`; + result = { user: t.entity(UserFull) }; + } + mockFetch.get('/team', { + team: { + __typename: 'Team', + id: 't-1', + owner: { __typename: 'User', id: 'u-1', name: 'Alice' }, + members: [ + { __typename: 'User', id: 'u-1', name: 'Alice', email: 'a@x' }, + { __typename: 'User', id: 'u-2', name: 'Bob', email: 'b@x' }, + ], + }, + }); + mockFetch.get('/user/u-1', { user: { __typename: 'User', id: 'u-1', name: 'Alice', email: 'a@x' } }); + + await testWithClient(client, async () => { + const team = fetchQuery(GetTeam); + await team; + const ids = watcher(() => (team.value as any).team.members.map((m: any) => m.id).join(',')); + ids.addListener(() => {}); + // u-1 was parsed with the preview def first, so it is narrowed out. + expect(ids.value).toBe('u-2'); + + await fetchQuery(GetUser, { id: 'u-1' }); + await sleep(5); + expect(ids.value).toBe('u-1,u-2'); + }); + }); + + it('query extras attached to a shared entity reach an existing consumer without another trigger', async () => { + const { client, mockFetch } = getClient(); + class Profile extends Entity { + __typename = t.typename('Profile'); + id = t.id; + name = t.string; + } + class GetWrapped extends RESTQuery { + path = '/wrapped'; + result = { profile: t.entity(Profile) }; + } + class GetRoot extends RESTQuery { + path = '/root'; + result = t.entity(Profile); + } + const profile = { __typename: 'Profile', id: 1, name: 'Alice' }; + mockFetch.get('/wrapped', { profile }); + mockFetch.get('/root', profile); + + const wrapped = snapshotHarness(client, () => fetchQuery(GetWrapped)); + await wrapped.query; + expect('__refetch' in wrapped.read().profile).toBe(false); + + const root = snapshotHarness(client, () => fetchQuery(GetRoot)); + await root.query; + expect(typeof wrapped.read().profile.__refetch).toBe('function'); + }); + + it('a length-only consumer of a narrowed entity array does not recompute when a member it holds changes', async () => { + const { client, mockFetch } = getClient(); + class UserPreview extends Entity { + __typename = t.typename('User'); + id = t.id; + name = t.string; + } + class UserFull extends Entity { + __typename = t.typename('User'); + id = t.id; + name = t.string; + email = t.string; + } + class Team extends Entity { + __typename = t.typename('Team'); + id = t.id; + owner = t.entity(UserPreview); + members = t.array(t.entity(UserFull)); + } + class GetTeam extends RESTQuery { + path = '/team'; + result = { team: t.entity(Team) }; + } + mockFetch.get('/team', { + team: { + __typename: 'Team', + id: 't-1', + owner: { __typename: 'User', id: 'u-1', name: 'Alice' }, + members: [ + { __typename: 'User', id: 'u-1', name: 'Alice', email: 'a@x' }, + { __typename: 'User', id: 'u-2', name: 'Bob', email: 'b@x' }, + ], + }, + }); + + const team = holdQuery(client, () => fetchQuery(GetTeam)); + await team; + let computes = 0; + const count = withContexts([[QueryClientContext, client]], () => { + const w = watcher(() => { + computes++; + return (team.value as any).team.members.length; + }); + w.addListener(() => {}); + return w; + }); + await sleep(5); + expect(count.value).toBe(1); + const base = computes; + + // A member the array holds changes a field the consumer never read. + client.applyMutationEvent({ type: 'update', typename: 'User', data: { id: 'u-2', name: 'Robert' } }); + await sleep(5); + expect((team.value as any).team.members[0].name).toBe('Robert'); + expect(count.value).toBe(1); + expect(computes).toBe(base); + + // A member the array left out becomes eligible: that is a change of the array. + client.applyMutationEvent({ type: 'update', typename: 'User', data: { id: 'u-1', email: 'alice@x' } }); + await sleep(5); + expect(count.value).toBe(2); + expect(computes).toBe(base + 1); + }); + + it('applies -0 over 0 and keeps NaN, at every level', async () => { + const { client, mockFetch } = getClient(); + class Quote extends Entity { + __typename = t.typename('Quote'); + id = t.id; + price = t.number; + history = t.array(t.number); + } + class GetQuote extends RESTQuery { + path = '/quote'; + result = { quote: t.entity(Quote) }; + } + mockFetch.get('/quote', { quote: { __typename: 'Quote', id: 'q', price: 0, history: [0, 1] } }); + const h = snapshotHarness(client, () => fetchQuery(GetQuote)); + await h.query; + const before = h.computes(); + + client.applyMutationEvent({ type: 'update', typename: 'Quote', data: { id: 'q', price: -0, history: [-0, 1] } }); + expect(Object.is((h.query.value as any).quote.price, -0)).toBe(true); + expect(Object.is(h.read().quote.price, -0)).toBe(true); + expect(Object.is(h.read().quote.history[0], -0)).toBe(true); + expect(h.computes()).toBe(before + 1); + + // The same data again is not a change. + client.applyMutationEvent({ type: 'update', typename: 'Quote', data: { id: 'q', price: -0, history: [-0, 1] } }); + expect(h.computes()).toBe(before + 1); + }); + + it('a snapshot handed back to the parser is read, not written to', async () => { + const { client, mockFetch } = getClient(); + class Doc extends Entity { + __typename = t.typename('Doc'); + id = t.id; + title = t.string; + meta = t.object({ views: t.number, tags: t.array(t.string) }); + extra = t.record(t.number); + } + class GetDoc extends RESTQuery { + path = '/doc'; + result = { doc: t.entity(Doc) }; + } + mockFetch.get('/doc', { + doc: { __typename: 'Doc', id: 'd', title: 'T', meta: { views: 1, tags: ['a'] }, extra: { k: 1 } }, + }); + const h = snapshotHarness(client, () => fetchQuery(GetDoc)); + await h.query; + const snap = h.read().doc; + expect(Object.isFrozen(snap.meta)).toBe(true); + const warn = vi.spyOn(client.getContext().log as { warn: (...args: unknown[]) => void }, 'warn'); + + // An app forwards part of a snapshot as an event payload (or a mutation's + // `effects.updates`); the parser must not write parsed values into it. + client.applyMutationEvent({ + type: 'update', + typename: 'Doc', + data: { id: 'd', title: 'T2', meta: { ...snap.meta, views: 2 }, extra: snap.extra }, + }); + const meta = { views: 3, tags: snap.meta.tags }; + client.applyMutationEvent({ type: 'update', typename: 'Doc', data: { id: 'd', meta, extra: snap.extra } }); + + expect(warn).not.toHaveBeenCalled(); + expect(h.read().doc.title).toBe('T2'); + expect(h.read().doc.meta.views).toBe(3); + expect(h.read().doc.extra.k).toBe(1); + expect(snap.meta.views).toBe(1); + expect(Object.isFrozen(h.read().doc.meta)).toBe(true); + }); + + it('two events sharing one nested object do not alias entity data', async () => { + const { client, mockFetch } = getClient(); + class Token extends Entity { + __typename = t.typename('Token'); + id = t.id; + walletId = t.string; + metadata = t.object({ image: t.string }); + } + class Wallet extends Entity { + __typename = t.typename('Wallet'); + id = t.id; + tokens = t.liveArray(Token, { constraints: { walletId: (this as any).id } }); + } + class GetWallet extends RESTQuery { + path = '/wallet'; + result = { wallet: t.entity(Wallet) }; + } + mockFetch.get('/wallet', { wallet: { __typename: 'Wallet', id: 'w', tokens: [] } }); + const h = snapshotHarness(client, () => fetchQuery(GetWallet)); + await h.query; + await sleep(5); + const error = vi.spyOn(console, 'error').mockImplementation(() => {}); + try { + const shared = { image: 'shared.png' }; + client.applyMutationEvent({ + type: 'create', + typename: 'Token', + data: { __typename: 'Token', id: 'tok-0', walletId: 'w', metadata: shared }, + }); + client.applyMutationEvent({ + type: 'create', + typename: 'Token', + data: { __typename: 'Token', id: 'tok-1', walletId: 'w', metadata: shared }, + }); + await sleep(5); + const tokens = () => h.read().wallet.tokens; + expect(tokens()).toHaveLength(2); + const tok1 = tokens()[1]; + client.applyMutationEvent({ + type: 'update', + typename: 'Token', + data: { id: 'tok-0', metadata: { image: 'new.png' } }, + }); + await sleep(5); + + expect(tokens()[0].metadata.image).toBe('new.png'); + expect(tokens()[1].metadata.image).toBe('shared.png'); + expect(tokens()[1]).toBe(tok1); + expect((h.query.value as any).wallet.tokens[1].metadata.image).toBe('shared.png'); + expect(shared.image).toBe('shared.png'); + expect(error).not.toHaveBeenCalled(); + } finally { + error.mockRestore(); + } + }); + + it('a Set-valued format with many object members is verified in linear time', async () => { + const { client, mockFetch } = getClient(); + registerFormat( + 'id-set', + Mask.STRING, + (value: string) => new Set(value.split(',').map(id => ({ id: Number(id) }))), + (value: Set<{ id: number }>) => [...value].map(item => item.id).join(','), + ); + class Roster extends Entity { + __typename = t.typename('Roster'); + id = t.id; + label = t.string; + members = t.format('id-set' as never); + } + class GetRoster extends RESTQuery { + path = '/roster'; + result = { roster: t.entity(Roster), other: t.entity(Roster) }; + } + const ids = Array.from({ length: 3000 }, (_, i) => i).join(','); + mockFetch.get('/roster', { + roster: { __typename: 'Roster', id: 'r', label: 'a', members: ids }, + other: { __typename: 'Roster', id: 'o', label: 'b', members: '0' }, + }); + const h = snapshotHarness(client, () => fetchQuery(GetRoster)); + await h.query; + expect(h.read().roster.members.size).toBe(3000); + + // Each event recomputes the snapshot at an unchanged version of `roster`, + // so the dev guard compares the rebuilt Set with the previous one. + const started = performance.now(); + for (let i = 0; i < 20; i++) { + client.applyMutationEvent({ type: 'update', typename: 'Roster', data: { id: 'o', label: `b${i}` } }); + expect(h.read().roster.members.size).toBe(3000); + } + expect(performance.now() - started).toBeLessThan(2000); + }); +}); diff --git a/packages/fetchium/src/applyEntities.ts b/packages/fetchium/src/applyEntities.ts index 06fdaa8..15d0214 100644 --- a/packages/fetchium/src/applyEntities.ts +++ b/packages/fetchium/src/applyEntities.ts @@ -27,6 +27,17 @@ export interface ApplyResult { entityRefs: Map; } +/** + * Whether an apply writes to the store. `true` writes every entity that needs + * it; `false` writes nothing (a cache hydration); `'existing'` writes entities + * that already existed in memory and not the ones this apply creates: those + * are written when a written record references them (`EntityInstance.save` + * writes a record's missing children first), or by the caller once it knows + * the root is retained (a streamed event whose root may end up routed + * nowhere). + */ +export type PersistMode = boolean | 'existing'; + /** * Single depth-first walk from the root that applies entities to the store, * replaces parsed data objects with entity proxies, and counts child refs. @@ -34,15 +45,27 @@ export interface ApplyResult { export function applyEntityRefs( ctx: ParseContext, rootData: unknown, - persist: boolean, + persist: PersistMode, appendMode: boolean = false, + created?: Set, ): ApplyResult { const queryClient = ctx.queryClient!; queryClient.currentParseId++; const seen = ctx.seen!; const entityRefs = new Map(); - const data = reifyAndApply(rootData, seen, queryClient, persist, entityRefs, appendMode); + let data: unknown; + try { + data = reifyAndApply(rootData, seen, queryClient, persist, entityRefs, appendMode, created); + } catch (e) { + queryClient.discardDeferredWrites(); + throw e; + } + + // Entities a written record referenced while they were still being reified + // (a payload that links back to an ancestor) are complete now, and so are + // the records that reference them. + queryClient.flushDeferredWrites(); return { data, entityRefs }; } @@ -55,22 +78,26 @@ function reifyAndApply( value: unknown, seen: Map, ParsedEntity>, queryClient: QueryClient, - persist: boolean, + persist: PersistMode, entityRefs: Map, appendMode: boolean, + created: Set | undefined, ): unknown { if (typeof value !== 'object' || value === null) return value; const entity = seen.get(value as Record); if (entity !== undefined) { - return applyEntity(entity, seen, queryClient, persist, entityRefs, appendMode); + return applyEntity(entity, seen, queryClient, persist, entityRefs, appendMode, created); } + // Only a slot whose value changes is written: values the parser did not + // declare (and so did not copy) may belong to the caller, and be frozen. if (Array.isArray(value)) { for (let i = 0; i < value.length; i++) { const item = value[i]; if (typeof item === 'object' && item !== null && !(item instanceof FormattedValue) && !PROXY_ID.has(item)) { - value[i] = reifyAndApply(item, seen, queryClient, persist, entityRefs, appendMode); + const reified = reifyAndApply(item, seen, queryClient, persist, entityRefs, appendMode, created); + if (reified !== item) value[i] = reified; } } return value; @@ -81,7 +108,8 @@ function reifyAndApply( for (const key of Object.keys(obj)) { const v = obj[key]; if (typeof v === 'object' && v !== null && !(v instanceof FormattedValue) && !PROXY_ID.has(v)) { - obj[key] = reifyAndApply(v, seen, queryClient, persist, entityRefs, appendMode); + const reified = reifyAndApply(v, seen, queryClient, persist, entityRefs, appendMode, created); + if (reified !== v) obj[key] = reified; } } } @@ -101,13 +129,25 @@ function applyEntity( entity: ParsedEntity, seen: Map, ParsedEntity>, queryClient: QueryClient, - persist: boolean, + persist: PersistMode, parentEntityRefs: Map, appendMode: boolean, + created: Set | undefined, ): Record { - const { key, data, shape: entityShape, rawKeys } = entity; + const { key, data, shape: entityShape, rawKeys, eventKeys, fillsPartial } = entity; const shapeFields = entityShape.shape; + // The same entity can appear more than once in one payload (an author under + // `author` and again under `mentions`). The parser hands every slot the one + // parsed object, so the walk arrives here once per slot; applying it again + // would find its children already reified, count none of them and release + // every child ref. Count this slot's reference and hand out the proxy. + const applied = queryClient.entityMap.getEntity(key); + if (applied !== undefined && applied.parseId === queryClient.currentParseId) { + parentEntityRefs.set(applied, (parentEntityRefs.get(applied) ?? 0) + 1); + return applied.getProxy(entityShape); + } + const entityInstance = queryClient.prepareEntity(key, data, entityShape); const existingData = entityInstance.data; const isUpdate = existingData !== data; @@ -122,24 +162,55 @@ function applyEntity( // A refetch or a poll that returns identical data is a no-op: nothing to // notify consumers about, and nothing new to write to the store. let changed = true; - if (isUpdate) { - changed = mergeFields( - shapeFields, - data, - existingData, - rawKeys, - entityInstance, - existingData, - seen, - queryClient, - persist, - childRefs, - appendMode, - ); - if (changed) entityInstance.notify(); - } else { - initFields(shapeFields, data, entityInstance, data, seen, queryClient, persist, childRefs, appendMode); + // Until the fields are reified the data is not a record yet: a write that + // reaches this instance through a reference from inside its own subtree is + // deferred to the end of the walk. + entityInstance._applying = true; + try { + if (isUpdate) { + changed = mergeFields( + shapeFields, + data, + existingData, + rawKeys, + entityInstance, + existingData, + seen, + queryClient, + persist, + childRefs, + appendMode, + created, + ); + if ((persist === true && rawKeys === undefined) || fillsPartial) { + // A full payload (a fetch), or the record's fields merged in, makes + // the data a complete record again. + entityInstance._partial = false; + entityInstance._partialKeys = undefined; + } else if (entityInstance._partial && rawKeys !== undefined) { + for (const k of rawKeys) entityInstance._partialKeys!.add(k); + } + } else { + initFields(shapeFields, data, entityInstance, data, seen, queryClient, persist, childRefs, appendMode, created); + if (eventKeys !== undefined) { + // Streamed events are partial: what this entity holds is what its + // events carried, not necessarily the whole record. + entityInstance._partial = true; + entityInstance._partialKeys = new Set(eventKeys); + // A record always carries its typename and id, whether or not the + // event did (the parser fills the literal in). + if (entityShape.typenameField !== undefined) entityInstance._partialKeys.add(entityShape.typenameField); + if (typeof entityShape.idField === 'string') entityInstance._partialKeys.add(entityShape.idField); + } else if (persist === false) { + // Hydrated from the store: its record exists. + entityInstance._recorded = true; + } + if (persist === 'existing') created?.add(entityInstance); + } + } finally { + entityInstance._applying = false; } + if (isUpdate && changed) entityInstance.notify(); if (appendMode && entityInstance.liveCollections.length > 0) { for (const binding of entityInstance.liveCollections) { @@ -159,9 +230,17 @@ function applyEntity( const newRefs = childRefs.size > 0 ? childRefs : undefined; const refsChanged = !sameRefs(entityInstance.entityRefs, newRefs); - // An entity hydrated from the store was never written, so there is no write to skip. - const needsPersist = changed || refsChanged || !entityInstance._persisted || !queryClient.storeReportsDeletes; - entityInstance.setChildRefs(newRefs, persist && needsPersist); + // An entity hydrated from the store was never written, so there is no write + // to skip. A skip is also only safe while the store has nothing queued: a + // deletion it has yet to process could drop the record this apply trusts. + const needsPersist = + changed || + refsChanged || + !entityInstance._persisted || + !queryClient.storeReportsDeletes || + !queryClient.storeIsSettled(); + const writes = persist === true || (persist === 'existing' && isUpdate); + entityInstance.setChildRefs(newRefs, writes && needsPersist); const proxy = entityInstance.getProxy(entityShape); @@ -183,16 +262,17 @@ function mergeFields( entityData: Record, seen: Map, ParsedEntity>, queryClient: QueryClient, - persist: boolean, + persist: PersistMode, childRefs: Map, appendMode: boolean, + created: Set | undefined, ): boolean { let changed = false; for (const [fieldKey, propShape] of entries(shape)) { if (rawKeys !== undefined && !rawKeys.has(fieldKey)) continue; if (shouldReify(data[fieldKey])) { - data[fieldKey] = reifyAndApply(data[fieldKey], seen, queryClient, persist, childRefs, appendMode); + data[fieldKey] = reifyAndApply(data[fieldKey], seen, queryClient, persist, childRefs, appendMode, created); } if (propShape instanceof ValidatorDef && propShape._liveConfig !== undefined) { @@ -208,13 +288,14 @@ function mergeFields( entityInstance, entityData, queryClient, + fieldKey, ); changed = true; } } else { const newVal = data[fieldKey]; const oldVal = existingData[fieldKey]; - if (newVal === oldVal) continue; + if (Object.is(newVal, oldVal)) continue; // Replace a union wholesale instead of merging by field, otherwise a // changed variant keeps the old variant's fields. Unlike entity unions, // there is no partial-update path to preserve here: a partial variant @@ -248,6 +329,7 @@ function mergeFields( persist, childRefs, appendMode, + created, ) ) { changed = true; @@ -295,7 +377,8 @@ export function sameKeys(a: Record, b: Record) * they were built from, arrays and plain objects element by element. */ export function sameValue(a: unknown, b: unknown): boolean { - if (a === b) return true; + // `Object.is`: re-applying NaN is not a change, and `-0` over `0` is one. + if (Object.is(a, b)) return true; if (typeof a !== 'object' || typeof b !== 'object' || a === null || b === null) return false; if (a instanceof FormattedValue || b instanceof FormattedValue) { return a instanceof FormattedValue && b instanceof FormattedValue && a._raw === b._raw; @@ -354,15 +437,16 @@ function initFields( entityData: Record, seen: Map, ParsedEntity>, queryClient: QueryClient, - persist: boolean, + persist: PersistMode, childRefs: Map, appendMode: boolean, + created: Set | undefined, ): void { for (const [fieldKey, propShape] of entries(shape)) { if (!(fieldKey in data)) continue; if (shouldReify(data[fieldKey])) { - data[fieldKey] = reifyAndApply(data[fieldKey], seen, queryClient, persist, childRefs, appendMode); + data[fieldKey] = reifyAndApply(data[fieldKey], seen, queryClient, persist, childRefs, appendMode, created); } if (propShape instanceof ValidatorDef && propShape._liveConfig !== undefined) { @@ -372,6 +456,7 @@ function initFields( entityInstance, entityData, queryClient, + fieldKey, ); } else { const val = data[fieldKey]; @@ -390,7 +475,18 @@ function initFields( ? (propShape.shape as Record) : undefined; if (nestedShape !== undefined) { - initFields(nestedShape, val, entityInstance, entityData, seen, queryClient, persist, childRefs, appendMode); + initFields( + nestedShape, + val, + entityInstance, + entityData, + seen, + queryClient, + persist, + childRefs, + appendMode, + created, + ); } } } diff --git a/packages/fetchium/src/errors.ts b/packages/fetchium/src/errors.ts index 270975c..65d6cf0 100644 --- a/packages/fetchium/src/errors.ts +++ b/packages/fetchium/src/errors.ts @@ -169,3 +169,16 @@ export function typeError(path: string, expectedType: InternalObjectFieldTypeDef } return new TypeError(`Validation error at ${path}`); } + +/** + * A cached query cannot be hydrated as-is: an entity it holds is already in + * memory and its data does not satisfy the query's shape. Unlike a bad array + * item, this drops the whole cached query (it is refetched) rather than the + * item, since a list missing an item would resolve without a word. + */ +export class CachedEntityMismatchError extends Error { + constructor(message: string) { + super(message); + this.name = 'CachedEntityMismatchError'; + } +} diff --git a/packages/fetchium/src/parseEntities.ts b/packages/fetchium/src/parseEntities.ts index 1563da6..4abaded 100644 --- a/packages/fetchium/src/parseEntities.ts +++ b/packages/fetchium/src/parseEntities.ts @@ -7,6 +7,7 @@ import { hashValue } from 'signalium/utils'; import type { QueryClient, PreloadedEntityMap } from './QueryClient.js'; +import type { EntityInstance } from './EntityInstance.js'; import { CaseInsensitiveSet, FormattedValue, @@ -15,7 +16,7 @@ import { VariantGroup, VariantSet, } from './typeDefs.js'; -import { typeError, typeToString, UnknownUnionVariantError } from './errors.js'; +import { typeError, typeToString, UnknownUnionVariantError, CachedEntityMismatchError } from './errors.js'; import { ARRAY_KEY, ArrayDef, @@ -49,8 +50,15 @@ export interface ParsedEntity { data: Record; /** Set for partial event updates — restricts mergeFields to only these keys. */ rawKeys: Set | undefined; + /** The keys a streamed event carried, whether or not the entity was in memory. */ + eventKeys: Set | undefined; + /** A cached record parsed to fill in the fields an entity built from events lacks. */ + fillsPartial: boolean; } +/** Trust-memory verdicts: `true`/`false` once settled, `'pending'` while a check is in progress (a cycle). */ +export type TrustMemo = Map, boolean | 'pending'>>; + // ====================================================== // Parse context — bundles threading parameters // ====================================================== @@ -64,6 +72,8 @@ export class ParseContext { isPartialEvent: boolean = false; seen: Map, ParsedEntity> | undefined = undefined; seenByKey: Map | undefined = undefined; + /** Verdicts of the trust-memory check, per instance and def, for one parse. */ + trusted: TrustMemo | undefined = undefined; reset( queryClient: QueryClient | undefined, @@ -75,6 +85,7 @@ export class ParseContext { this.preloadedEntities = preloadedEntities; this.warn = warn; this.isPartialEvent = isPartialEvent; + this.trusted = undefined; if (queryClient !== undefined) { if (this.seen === undefined) { this.seen = new Map(); @@ -381,6 +392,7 @@ function parseArrayData(array: unknown[], itemShape: ComplexTypeDef, ctx: ParseC try { result.push(parseData(array[i], itemShape as unknown as TypeDef, ctx, `${path}[${i}]`)); } catch (e) { + if (e instanceof CachedEntityMismatchError) throw e; ctx.warn('Failed to parse array item, filtering out', { index: i, value: array[i], @@ -392,17 +404,23 @@ function parseArrayData(array: unknown[], itemShape: ComplexTypeDef, ctx: ParseC return result; } +// Both walkers parse into a shallow copy of their input, like `parseArrayData` +// builds a new array. The input belongs to whoever handed it in (an adapter, +// an event payload, a consumer's snapshot — frozen in dev builds) and must not +// be written to; the copy keeps every key the input had, as before. + function parseRecordData( record: Record, valueShape: ComplexTypeDef, ctx: ParseContext, path: string, ): Record { + const result: Record = { ...record }; for (const [key, value] of entries(record)) { - record[key] = parseData(value, valueShape as unknown as TypeDef, ctx, `${path}["${key}"]`); + result[key] = parseData(value, valueShape as unknown as TypeDef, ctx, `${path}["${key}"]`); } - return record; + return result; } function parseObjectData( @@ -416,12 +434,13 @@ function parseObjectData( } const shape = objectShape.shape; + const result: Record = { ...obj }; for (const [key, propShape] of entries(shape)) { - obj[key] = parseData(obj[key], propShape as unknown as TypeDef, ctx, `${path}.${key}`); + result[key] = parseData(obj[key], propShape as unknown as TypeDef, ctx, `${path}.${key}`); } - return obj; + return result; } // ====================================================== @@ -459,33 +478,98 @@ function parseEntityData( return existingEntry.data; } - if (preloadedEntities !== undefined) { - const existing = queryClient.entityMap.getEntity(key); - const preloaded = existing?.data ?? preloadedEntities.get(key); - - if (preloaded === undefined) { - throw new Error(`Cached entity ${key} not found in preloaded map`); - } - - obj = preloaded; - } - const parsedData: Record = {}; // For symbol id fields (QUERY_ID), copy the id onto parsedData so // getOrCreateEntity can read it. entries(shape) skips symbol keys. if (typeof entityShape.idField === 'symbol') { parsedData[entityShape.idField] = id; } + + if (preloadedEntities !== undefined) { + const existing = queryClient.entityMap.getEntity(key); + + // The fields of an entity built from streamed events that the record on + // disk has and the events did not carry. + let fillKeys: Set | undefined; + const preloaded = preloadedEntities.get(key); + + if (existing !== undefined && existing._partial && existing._partialKeys !== undefined && preloaded !== undefined) { + // Built from events, the in-memory data lacks what the record has: + // parse the record for those fields only and merge them in. + fillKeys = existing._partialKeys; + obj = preloaded; + } else if (existing !== undefined) { + // The entity is already live in memory, so the cache has nothing newer + // to offer: the in-memory data is what every consumer already sees. + // Re-parsing that data would run already-parsed values (formatted + // values, parse results, child proxies) through the parser again, in + // place, and corrupt them. An empty partial entry merges nothing and + // hands the query the existing proxy. The data must still satisfy this + // query's shape, or the cached query is not usable as-is. + ctx.trusted ??= new Map(); + if (!dataSatisfiesDef(existing.data, entityShape as unknown as ValidatorDef, queryClient, ctx.trusted)) { + throw new CachedEntityMismatchError( + `Cached entity ${entityShape.typenameValue}:${String(existing.id)} in memory does not satisfy the query's shape`, + ); + } + const entry: ParsedEntity = { + key, + shape: entityShape, + data: parsedData, + rawKeys: new Set(), + eventKeys: undefined, + fillsPartial: false, + }; + ctx.seen!.set(parsedData, entry); + ctx.seenByKey!.set(key, entry); + return parsedData; + } else { + if (preloaded === undefined) { + throw new Error(`Cached entity ${key} not found in preloaded map`); + } + obj = preloaded; + } + + if (fillKeys !== undefined) { + const shapeKeys = Object.keys(entityShape.shape); + const rawKeys = new Set(); + for (const k of shapeKeys) if (!fillKeys.has(k)) rawKeys.add(k); + const entry: ParsedEntity = { + key, + shape: entityShape, + data: parsedData, + rawKeys, + eventKeys: undefined, + fillsPartial: true, + }; + ctx.seen!.set(parsedData, entry); + ctx.seenByKey!.set(key, entry); + const entityDesc = `[[${entityShape.typenameValue}:${id}]]`; + for (const [fieldKey, propShape] of entries(entityShape.shape)) { + if (!rawKeys.has(fieldKey)) continue; + parsedData[fieldKey] = parseData( + obj[fieldKey], + propShape as unknown as TypeDef, + ctx, + `${entityDesc}.${fieldKey}`, + ); + } + return parsedData; + } + } // For mutation events updating existing entities, track which keys are // present so mergeFields only touches those fields (true partial update). const existingInStore = queryClient.entityMap.getEntity(key); const isPartial = ctx.isPartialEvent && existingInStore !== undefined; + const eventKeys = ctx.isPartialEvent ? new Set(Object.keys(obj)) : undefined; const entry: ParsedEntity = { key, shape: entityShape, data: parsedData, - rawKeys: isPartial ? new Set(Object.keys(obj)) : undefined, + rawKeys: isPartial ? eventKeys : undefined, + eventKeys, + fillsPartial: false, }; ctx.seen!.set(parsedData, entry); ctx.seenByKey!.set(key, entry); @@ -508,6 +592,183 @@ function parseEntityData( // entitySatisfiesShape // ====================================================== +/** + * Whether already-parsed entity data can be handed to a query declaring `def` + * without being parsed again. Unlike `entitySatisfiesShape` (a top-level + * presence check used for narrowing) this descends into nested objects, arrays, + * records, unions and child entities, because a cached query that resolves with + * a required nested field missing is worse than one that refetches. Where a + * value cannot be judged it is accepted, matching the parser's leniency. + */ +export function dataSatisfiesDef( + data: Record, + def: ValidatorDef, + queryClient: QueryClient, + visiting: TrustMemo, +): boolean { + const shape = def.shape as Record | undefined; + if (shape === undefined || shape === null) return true; + for (const key of Object.keys(shape)) { + if (key === def.typenameField) continue; + if (!valueSatisfiesDef(data[key], shape[key], queryClient, visiting)) return false; + } + return true; +} + +function allowsMissing(mask: number): boolean { + return (mask & Mask.UNDEFINED) !== 0; +} + +function maskOf(value: unknown): number { + switch (typeof value) { + case 'number': + return Mask.NUMBER; + case 'string': + return Mask.STRING; + case 'boolean': + return Mask.BOOLEAN; + case 'undefined': + return Mask.UNDEFINED; + case 'object': + return value === null ? Mask.NULL : Array.isArray(value) ? Mask.ARRAY : Mask.OBJECT; + default: + return 0; + } +} + +function valueSatisfiesDef(value: unknown, fieldDef: unknown, queryClient: QueryClient, visiting: TrustMemo): boolean { + // Literals: the parser fills a missing literal in, and rejects any other value. + if (fieldDef instanceof VariantSet) return value === fieldDef.value; + if (typeof fieldDef === 'string') return value === fieldDef; + if (fieldDef instanceof Set) return fieldDef.has(value as never); + + if (typeof fieldDef === 'number') { + if (value === undefined) return allowsMissing(fieldDef); + if (value === null) return (fieldDef & Mask.NULL) !== 0; + if ((fieldDef & Mask.HAS_FORMAT) !== 0) return value instanceof FormattedValue; + return (fieldDef & maskOf(value)) !== 0; + } + + if (!(fieldDef instanceof ValidatorDef)) return true; + const mask = fieldDef.mask; + if (value === undefined) return allowsMissing(mask); + if (value === null) return (mask & Mask.NULL) !== 0; + + if (fieldDef._liveConfig !== undefined) return true; + if ((mask & Mask.PARSE_RESULT) !== 0) return typeof value === 'object' && 'success' in (value as object); + if (typeof value !== 'object') { + if ((mask & Mask.HAS_FORMAT) !== 0 && (mask & maskOf(value)) !== 0) return false; + return (mask & maskOf(value)) !== 0 || fieldDef.values?.has(value as never) === true; + } + if (value instanceof FormattedValue) return (mask & Mask.HAS_FORMAT) !== 0; + + if ((mask & Mask.ENTITY) !== 0 && (mask & Mask.UNION) === 0) { + return entitySatisfies(value as object, fieldDef, queryClient, visiting); + } + + if ((mask & Mask.UNION) !== 0) { + // Resolve the member the value belongs to; an unrecognised value is left + // to the union's own leniency. + const members = fieldDef.shape as Record | undefined; + if (members === undefined || members === null) return true; + if (Array.isArray(value)) { + const itemDef = members[ARRAY_KEY]; + if (itemDef === undefined || typeof itemDef === 'number') return true; + return arraySatisfies(value, itemDef, queryClient, visiting); + } + const entityKey = PROXY_ID.get(value as object); + const typename = + entityKey !== undefined + ? queryClient.entityMap.getEntity(entityKey)?.typename + : ((value as Record)[fieldDef.typenameField ?? '__typename'] as string | undefined); + const member = typename !== undefined ? members[typename] : undefined; + if (member instanceof VariantGroup) { + const source = + entityKey !== undefined ? queryClient.entityMap.getEntity(entityKey)?.data : (value as Record); + const variant = source?.[member.variantField] as string | undefined; + const variantDef = variant !== undefined ? member.defs[variant] : undefined; + return variantDef === undefined ? true : valueSatisfiesDef(value, variantDef, queryClient, visiting); + } + if (member === undefined || typeof member === 'number') { + const recordDef = members[RECORD_KEY]; + if (typename !== undefined || recordDef === undefined || entityKey !== undefined) return true; + for (const item of Object.values(value as Record)) { + if (!valueSatisfiesDef(item, recordDef, queryClient, visiting)) return false; + } + return true; + } + return valueSatisfiesDef(value, member, queryClient, visiting); + } + + if ((mask & Mask.ARRAY) !== 0) { + if (!Array.isArray(value)) return false; + return arraySatisfies(value, fieldDef.shape, queryClient, visiting); + } + + if ((mask & Mask.RECORD) !== 0) { + if (typeof value !== 'object' || Array.isArray(value)) return false; + for (const item of Object.values(value as Record)) { + if (!valueSatisfiesDef(item, fieldDef.shape, queryClient, visiting)) return false; + } + return true; + } + + if ((mask & Mask.OBJECT) !== 0) { + if (typeof value !== 'object' || Array.isArray(value)) return false; + if (PROXY_ID.has(value as object)) return true; + return dataSatisfiesDef(value as Record, fieldDef, queryClient, visiting); + } + + return (mask & maskOf(value)) !== 0; +} + +/** + * An entity field holds a proxy. One whose instance has left memory is served + * as it is (the data it still holds), as a re-parse would have handed it out + * too; a live one must satisfy the def, with verdicts shared across the parse + * and a cycle counted as satisfied. + */ +function entitySatisfies( + value: object, + def: ValidatorDef, + queryClient: QueryClient, + visiting: TrustMemo, +): boolean { + const entityKey = PROXY_ID.get(value); + if (entityKey === undefined) return false; + const child = queryClient.entityMap.getEntity(entityKey); + if (child === undefined) return true; + let checked = visiting.get(child); + if (checked === undefined) visiting.set(child, (checked = new Map())); + const known = checked.get(def); + if (known !== undefined) return known === 'pending' || known; + checked.set(def, 'pending'); + const verdict = dataSatisfiesDef(child.data, def, queryClient, visiting); + checked.set(def, verdict); + return verdict; +} + +/** + * Members of an entity array whose typename has more than one registered + * class are narrowed on read to the ones satisfying the field's def, so a + * member that does not satisfy it is not a mismatch: it is left out either + * way. + */ +function arraySatisfies(value: unknown[], itemDef: unknown, queryClient: QueryClient, visiting: TrustMemo): boolean { + let narrowed = false; + if (itemDef instanceof ValidatorDef && (itemDef.mask & Mask.ENTITY) !== 0 && (itemDef.mask & Mask.UNION) === 0) { + const typename = itemDef.typenameValue; + const defs = typename !== undefined ? queryClient.getEntityDefsForTypename(typename) : undefined; + narrowed = defs !== undefined && defs.length > 1; + } + for (const item of value) { + if (valueSatisfiesDef(item, itemDef, queryClient, visiting)) continue; + if (narrowed && typeof item === 'object' && item !== null && PROXY_ID.has(item)) continue; + return false; + } + return true; +} + export function entitySatisfiesShape(data: Record, def: ValidatorDef): boolean { return objectSatisfiesShape(data, def.shape as Record, def.typenameField); } From cb4a21112da6c53a41f9b15c1c12459abcb16cc3 Mon Sep 17 00:00:00 2001 From: Adam Azbel-Bedell Date: Mon, 5 Oct 2026 21:30:17 -0400 Subject: [PATCH 03/15] perf(fetchium): skip the per-snapshot key-cache lookup and an event's unused Set Moving the static/dynamic key split to a per-client cache made every entity snapshot re-resolve its key lists through a WeakMap lookup, so a list of N entities paid N lookups per re-render. A cache's entries are never overwritten, and the client replaces the whole cache when a typename gains a class, so comparing the cache's identity is enough to know whether the lists can have changed. applyMutationEvent allocated the set of created entities for every event; only an event whose root is new uses it. Co-Authored-By: Claude Opus 5.5 (1M context) --- packages/fetchium/src/EntityInstance.ts | 24 ++++++++++++++++-------- packages/fetchium/src/QueryClient.ts | 11 ++++++----- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/packages/fetchium/src/EntityInstance.ts b/packages/fetchium/src/EntityInstance.ts index ee948e2..63412d9 100644 --- a/packages/fetchium/src/EntityInstance.ts +++ b/packages/fetchium/src/EntityInstance.ts @@ -1070,21 +1070,29 @@ function createProxy( } // Shared per shape; only a query's root entity allocates its own lists. - let cachedBase: EntityKeys | undefined; + let cachedBaseCache: QueryClient['shapeKeyCache'] | undefined; let cachedMethods: Record | undefined; let cachedGetters: Record | undefined; let keys: EntityKeys | undefined; function entityKeys(): EntityKeys { - // The base is re-resolved on every call (a WeakMap lookup): the client - // resets its cache when another class registers for this typename, and - // the new split must reach snapshots already in flight. Both extras - // slots matter too: these lists decide what a snapshot walks and re-reads. - const base = shapeKeys(validatorDef, shapeFields, methods, queryClient); + // The client replaces its key cache when another class registers for + // this typename, and the new split must reach snapshots already in + // flight. A cache's entries are never overwritten, so the base can only + // have changed when the cache object did: an identity check, rather than + // a WeakMap lookup on every snapshot of every entity. Both extras slots + // matter too: these lists decide what a snapshot walks and re-reads. + const baseCache = queryClient.shapeKeyCache; const methodsNow = instance._extraMethods; const gettersNow = instance._extraGetters; - if (keys === undefined || base !== cachedBase || methodsNow !== cachedMethods || gettersNow !== cachedGetters) { - cachedBase = base; + if ( + keys === undefined || + baseCache !== cachedBaseCache || + methodsNow !== cachedMethods || + gettersNow !== cachedGetters + ) { + const base = shapeKeys(validatorDef, shapeFields, methods, queryClient); + cachedBaseCache = baseCache; cachedMethods = methodsNow; cachedGetters = gettersNow; keys = methodsNow === undefined && gettersNow === undefined ? base : withExtraKeys(base, methodsNow, gettersNow); diff --git a/packages/fetchium/src/QueryClient.ts b/packages/fetchium/src/QueryClient.ts index f7a7626..8162d14 100644 --- a/packages/fetchium/src/QueryClient.ts +++ b/packages/fetchium/src/QueryClient.ts @@ -804,8 +804,9 @@ export class QueryClient { // Entities the event creates are not written by the apply; entities it // merely updates are. A created entity is written once a written record - // references it, or below, once the root is known to be retained. - const created = new Set(); + // references it, or below, once the root is known to be retained. Only an + // event whose root is new can create entities that need collecting here. + const created = existing === undefined ? new Set() : undefined; try { const warn = this.context.log?.warn ?? (() => {}); const parseCtx = this.mutationParseContext; @@ -823,7 +824,7 @@ export class QueryClient { if (existing === undefined) { // Half applied: nothing built from it can stay. const createdRoot = this.entityMap.getEntity(key); - if (createdRoot !== undefined) this.evictCreated(createdRoot, created); + if (createdRoot !== undefined) this.evictCreated(createdRoot, created!); } return; } @@ -870,14 +871,14 @@ export class QueryClient { entity.save(); } catch (e) { this.context.log?.warn?.('Failed to apply mutation event', e); - this.evictUnlessAdopted(entity, created); + this.evictUnlessAdopted(entity, created!); return; } } this.routeEvent(typename, entity.data, key, type, eventSource); - if (!matched) this.evictUnlessAdopted(entity, created); + if (!matched) this.evictUnlessAdopted(entity, created!); } /** From f67c51607fcc040a4a8d4c91e4d2322e9421963b Mon Sep 17 00:00:00 2001 From: Adam Azbel-Bedell Date: Mon, 5 Oct 2026 23:22:43 -0400 Subject: [PATCH 04/15] perf(fetchium): take the extra store work back out of streamed events - An entity built from events stops being partial once its fields are known to be the whole record: the store held no record of it when it is first written (hasEntity), or its events carried every field of its shape. Until now every later write of such an entity was a merge that read the stored record back and parsed it, roughly halving the apply rate of updates to stream-born rows. An entity over a record the store holds keeps merging. - With an AsyncQueryStore writer, one entity's queued write no longer turns write skipping off for every entity. The store reports only queued work that can delete a record (new optional `hasQueuedDeletes()`), which excludes the client's own entity writes; an entity whose own write is still queued counts as persisted; and an entity a queued write stops referencing is written again, rather than trusted, when it is referenced again before that write lands. A record that turns out stale (a failed or discarded write) is no longer marked persisted by an older write's acknowledgement. - A create event that no live collection routes is routed once (the dry run's), not twice, and the duplicate-slot check's entity lookup is reused instead of repeated. - Dev builds skip the snapshot key-list check for frozen snapshots, which every snapshot handed out in dev is. Co-Authored-By: Claude Opus 5.5 (1M context) --- .changeset/skip-write-and-snapshot-fixes.md | 4 +- packages/fetchium/src/EntityInstance.ts | 44 +++- packages/fetchium/src/QueryClient.ts | 27 ++- .../src/__tests__/event-persistence.test.ts | 213 ++++++++++++++++++ packages/fetchium/src/applyEntities.ts | 27 ++- packages/fetchium/src/query-types.ts | 10 + packages/fetchium/src/stores/async.ts | 14 +- 7 files changed, 318 insertions(+), 21 deletions(-) diff --git a/.changeset/skip-write-and-snapshot-fixes.md b/.changeset/skip-write-and-snapshot-fixes.md index fac9ebc..bb5385e 100644 --- a/.changeset/skip-write-and-snapshot-fixes.md +++ b/.changeset/skip-write-and-snapshot-fixes.md @@ -7,9 +7,9 @@ Fixes for the entity snapshot fast path, unchanged-apply skipping and single-wri - Development builds hand out frozen snapshots, so a consumer that mutates one (a `sort()` in render, an assignment) fails at its own call site with a TypeError. Production snapshots stay mutable; treat them as read-only. The parser no longer writes into the objects it is given, so a snapshot (or part of one) handed back through a mutation event or a mutation's `effects` is read, not mutated, and two events sharing one nested object no longer make two entities share it. - The dev-only stale-snapshot check no longer throws from inside a snapshot. A throw there escaped Signalium's watcher flush and stopped every consumer on the page from updating. It now serves the re-read value and raises the error out of band, once per field: through the client's `log.error` and then as an uncaught error (a red box in React Native, an uncaught error in the browser, a failed run under vitest). It compares with `Object.is` (so a `NaN` field no longer trips it, and `-0` over `0` is applied at every level), compares a rebuilt `Set` in linear time, and restores the key set of a snapshot a consumer altered. - Hydrating a cached query no longer re-parses an entity that is already in memory. The re-parse ran already-parsed values through the parser again in place, wiping nested optional formats and parse results, and since the apply then saw no change it never notified; consumers of the first query kept stale snapshots. The in-memory data is used as-is once it structurally satisfies the query's shape, nested objects, arrays, records, unions and child entities included (members an array narrows out on read, and children that have left memory, are not held against it); otherwise the whole cached query is dropped and fetched, never served with an item missing. An entity built from streamed events is completed from its record instead. -- A streamed `create` (or an `update` for an entity not in memory) now writes the existing entities its payload changed, and the entities it created only once a written record references them: the root a live array retains is written before the event is routed (a write that fails is logged and nothing is routed, as a failed event always behaved), the rest follow from the records that reference them, and what nothing references is evicted at once, `gcTime` or not. A record's write puts its missing children first; a payload that links back to an ancestor defers both writes to the end of the apply. An entity built from events remembers the fields its events carried: its writes go through the store's new `mergeEntity`, which merges those fields over the record on disk, so the fields the events did not carry survive (a store without `mergeEntity` gets the whole in-memory data through `saveEntity`, as before 0.6). An event for an entity the store holds a record of (`hasEntity`) but that is not in memory refreshes that record the same way; a store that cannot say what it holds is not written, so no record nothing references is left behind. +- A streamed `create` (or an `update` for an entity not in memory) now writes the existing entities its payload changed, and the entities it created only once a written record references them: the root a live array retains is written before the event is routed (a write that fails is logged and nothing is routed, as a failed event always behaved), the rest follow from the records that reference them, and what nothing references is evicted at once, `gcTime` or not. A record's write puts its missing children first; a payload that links back to an ancestor defers both writes to the end of the apply. An entity built from events remembers the fields its events carried: its writes go through the store's new `mergeEntity`, which merges those fields over the record on disk, so the fields the events did not carry survive (a store without `mergeEntity` gets the whole in-memory data through `saveEntity`, as before 0.6). Once its fields are known to be the whole record (the store held no record of it when it was first written, or its events carried every field of its shape), it is written whole like any other entity, without reading the record back. An event for an entity the store holds a record of (`hasEntity`) but that is not in memory refreshes that record the same way; a store that cannot say what it holds is not written, so no record nothing references is left behind. - An entity that appears more than once in one payload (an author listed under `author` and again under `mentions`) is applied once: it keeps its child references and is written once. -- `_persisted` is only set once the store write succeeded, so a write that threw is retried by the next apply. `AsyncQueryStore` writers acknowledge processed entity writes (`onPersisted`), report a failed one, answer `hasEntity` from the keys they read from the delegate at start (alongside the queue, so writes do not wait for it; a queued write counts as held) and keep current, run `purgeStaleQueries` through the same serial queue as the writes (a reader with its own delegate still purges directly), ignore anything on the channel that is not one of their messages, and expose `isSettled()`; the client counts its own outstanding writes, so an acknowledgement of a write another instance queued does not mark a hydrated instance current, and it does not skip a write while the writer has operations queued, since one of them may delete the record the skip relies on. +- `_persisted` is only set once the store write succeeded, so a write that threw is retried by the next apply. `AsyncQueryStore` writers acknowledge processed entity writes (`onPersisted`), report a failed one, answer `hasEntity` from the keys they read from the delegate at start (alongside the queue, so writes do not wait for it; a queued write counts as held) and keep current, run `purgeStaleQueries` through the same serial queue as the writes (a reader with its own delegate still purges directly), ignore anything on the channel that is not one of their messages, and expose `isSettled()` and `hasQueuedDeletes()`; the client counts its own outstanding writes, so an acknowledgement of a write another instance queued does not mark a hydrated instance current. It does not skip a write while the writer has an operation queued that may delete the record the skip relies on (anything but the client's own entity writes); an entity whose own write is still queued counts as written, and an entity that one of the client's queued writes stops referencing is written again if it is referenced again before that write lands. - `AsyncQueryStore` resizes a persisted LRU queue by copying it (growing no longer throws `RangeError`, shrinking evicts the dropped keys instead of stranding them, never the key being activated), and keeps a queue's persisted size for a message without `maxCount` from an older reader. `SyncQueryStore` gets the same resize. - `onDelete` returns an unsubscribe and `QueryClient.destroy()` calls it, so a store no longer retains every client created against it. `SyncQueryStore` shares its delete listeners between every instance over the same backing kv. - Reading a shared-typename entity array consumes the notifiers of the members it currently leaves out, so a reactive consumer learns when a member becomes eligible without a data change, and a consumer that only reads the array is not recomputed when a member it holds changes. diff --git a/packages/fetchium/src/EntityInstance.ts b/packages/fetchium/src/EntityInstance.ts index 63412d9..c7b694f 100644 --- a/packages/fetchium/src/EntityInstance.ts +++ b/packages/fetchium/src/EntityInstance.ts @@ -393,7 +393,9 @@ const snapshotEntity = (current: object, prev: unknown, snap: SnapshotFn): unkno } // A consumer that added or deleted keys on the snapshot it was handed: // rebuild with the shape's keys, in the shape's order, as a walk would. - if (!sameKeyList(Object.keys(before), keys.enumerable)) { + // A frozen snapshot (every one this module hands out in dev) cannot + // have been altered, so only an unfrozen one pays for the key list. + if (!Object.isFrozen(before) && !sameKeyList(Object.keys(before), keys.enumerable)) { const source_ = result ?? before; const rebuilt: Record = {}; for (const key of keys.enumerable) rebuilt[key] = source_[key]; @@ -617,13 +619,27 @@ export class EntityInstance { } if (oldRefs !== undefined && oldRefs.size > 0) { for (const child of oldRefs.keys()) { - if (newRefs === undefined || !newRefs.has(child)) child.release(); + if (newRefs === undefined || !newRefs.has(child)) { + child.release(); + if (persist) this.writeDropsRef(child); + } } } this.entityRefs = newRefs; if (persist) this.save(); } + /** + * This entity's next write drops its reference to `child`, which can delete + * the child's record. A store that processes writes later (`onPersisted`) + * does that after anything an apply decides meanwhile, so the child is + * written again rather than trusted. A synchronous store reports the + * deletion (`onDelete`) as it happens. + */ + private writeDropsRef(child: EntityInstance): void { + if (this._queryClient.storeAcksWrites) child.recordDropped(); + } + addChildRef(child: EntityInstance, persist: boolean = true): void { if (this.entityRefs === undefined) this.entityRefs = new Map(); const count = this.entityRefs.get(child) ?? 0; @@ -639,6 +655,7 @@ export class EntityInstance { if (count <= 1) { this.entityRefs.delete(child); child.release(); + if (persist) this.writeDropsRef(child); } else { this.entityRefs.set(child, count - 1); } @@ -693,7 +710,7 @@ export class EntityInstance { } catch (e) { // This entity's record (if any) still lacks the reference: the // next apply must write it again. - this._persisted = false; + this.markUnwritten(); throw e; } if (child._deferredWrite) deferred = true; @@ -712,12 +729,18 @@ export class EntityInstance { // once the write has been processed. An entity built from events hands // the store only the fields they carried, to merge over its record. // Counted before the call: a store may acknowledge synchronously. + // With no record to merge over, the fields this entity holds are the + // whole record, so it stops being partial and later writes skip the + // merge's read of the stored record. + if (this._partial && !this._recorded && client.store.hasEntity?.(this.key) === false) { + this._partial = false; + this._partialKeys = undefined; + } if (client.storeAcksWrites) this._pendingWrites++; try { client.entityMap.save(this, this._partial ? this._partialKeys : undefined); } catch (e) { - if (client.storeAcksWrites && this._pendingWrites > 0) this._pendingWrites--; - this._persisted = false; + this.markUnwritten(); throw e; } if (!client.storeAcksWrites) this._persisted = this._recorded = true; @@ -746,8 +769,17 @@ export class EntityInstance { /** The store dropped (or failed to write) this entity's record. */ recordDropped(): void { - this._persisted = false; this._recorded = false; + this.markUnwritten(); + } + + /** + * Its record is stale: the next apply writes it even if it finds nothing + * changed. Writes still queued hold older data, so their acknowledgements + * no longer mark it persisted. + */ + markUnwritten(): void { + this._persisted = false; this._pendingWrites = 0; } diff --git a/packages/fetchium/src/QueryClient.ts b/packages/fetchium/src/QueryClient.ts index 8162d14..dad9585 100644 --- a/packages/fetchium/src/QueryClient.ts +++ b/packages/fetchium/src/QueryClient.ts @@ -347,7 +347,9 @@ export class QueryClient { * A synchronous store is always settled. */ storeIsSettled(): boolean { - return this.store.isSettled?.() ?? true; + const store = this.store; + if (store.hasQueuedDeletes !== undefined) return !store.hasQueuedDeletes(); + return store.isSettled?.() ?? true; } private registerEntityDef(def: ValidatorDef): void { @@ -762,8 +764,18 @@ export class QueryClient { return this.entityMap.getEntity(proxyKey!)!; } - prepareEntity(key: number, obj: Record, shape: EntityDef): EntityInstance { + /** `existing`: the instance the caller already looked up under `key`, if any. */ + prepareEntity( + key: number, + obj: Record, + shape: EntityDef, + existing?: EntityInstance, + ): EntityInstance { this.registerEntityDef(shape as unknown as ValidatorDef); + if (existing !== undefined) { + existing.parseId = this.currentParseId; + return existing; + } return this.entityMap.getOrCreateEntity(key, obj, shape, this); } @@ -876,9 +888,10 @@ export class QueryClient { } } - this.routeEvent(typename, entity.data, key, type, eventSource); - - if (!matched) this.evictUnlessAdopted(entity, created!); + // The dry run found no collection to route into: routing again would + // only recompute the constraint hashes. + if (matched) this.routeEvent(typename, entity.data, key, type, eventSource); + else this.evictUnlessAdopted(entity, created!); } /** @@ -944,7 +957,7 @@ export class QueryClient { for (const entity of deferred) { if (entity._deferredWrite) { entity._deferredWrite = false; - entity._persisted = false; + entity.markUnwritten(); } } } @@ -963,7 +976,7 @@ export class QueryClient { for (const entity of deferred) { if (entity._deferredWrite) { entity._deferredWrite = false; - entity._persisted = false; + entity.markUnwritten(); } } } diff --git a/packages/fetchium/src/__tests__/event-persistence.test.ts b/packages/fetchium/src/__tests__/event-persistence.test.ts index bc18d56..ef429cc 100644 --- a/packages/fetchium/src/__tests__/event-persistence.test.ts +++ b/packages/fetchium/src/__tests__/event-persistence.test.ts @@ -1542,3 +1542,216 @@ describe('AsyncQueryStore writer', () => { expect(first._persisted).toBe(true); }); }); + +// ====================================================== +// Store work per streamed event +// ====================================================== + +describe('store work per streamed event', () => { + class Author extends Entity { + __typename = t.typename('Author'); + id = t.id; + name = t.string; + } + class Note extends Entity { + __typename = t.typename('Note'); + id = t.id; + boardId = t.string; + body = t.string; + pinned = t.optional(t.boolean); + author = t.optional(t.entity(Author)); + } + class Board extends Entity { + __typename = t.typename('Board'); + id = t.id; + notes = t.liveArray(Note, { constraints: { boardId: (this as any).id } }); + } + class GetBoard extends RESTQuery { + params = { id: t.id }; + path = `/board/${this.params.id}`; + result = { board: t.entity(Board) }; + } + + const getClient = setupTestClient(); + + it('an entity an event built, of which the store held no record, is written whole afterwards', async () => { + const { client, mockFetch, kv, store } = getClient(); + mockFetch.get('/board/b1', { board: { __typename: 'Board', id: 'b1', notes: [] } }); + const boardQ = holdQuery(client, () => fetchQuery(GetBoard, { id: 'b1' })); + await boardQ; + const merges = vi.spyOn(store, 'mergeEntity'); + const reads = vi.spyOn(kv, 'getString'); + + client.applyMutationEvent({ + type: 'create', + typename: 'Note', + data: { __typename: 'Note', id: 'n1', boardId: 'b1', body: 'first' }, + }); + for (let i = 0; i < 3; i++) { + client.applyMutationEvent({ type: 'update', typename: 'Note', data: { id: 'n1', body: `edit ${i}` } }); + } + await sleep(5); + + expect(merges).not.toHaveBeenCalled(); + expect(reads.mock.calls.filter(([key]) => key === valueKeyFor(K('Note', 'n1')))).toEqual([]); + expect(getDoc(kv, K('Note', 'n1'))).toEqual({ __typename: 'Note', id: 'n1', boardId: 'b1', body: 'edit 2' }); + expect((boardQ.value as any).board.notes.map((n: any) => n.body)).toEqual(['edit 2']); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + }); + + it('an entity an event built over a record the store holds keeps merging into it', async () => { + const { client, mockFetch, kv, store } = getClient(); + mockFetch.get('/board/b1', { + board: { + __typename: 'Board', + id: 'b1', + notes: [{ __typename: 'Note', id: 'n1', boardId: 'b1', body: 'a', pinned: true }], + }, + }); + const boardQ = holdQuery(client, () => fetchQuery(GetBoard, { id: 'b1' })); + await boardQ; + // The note leaves memory but its record stays (the board's cache holds it). + client.applyMutationEvent({ type: 'delete', typename: 'Note', data: 'n1' }); + await sleep(5); + kv.setString( + valueKeyFor(K('Note', 'n1')), + JSON.stringify({ __typename: 'Note', id: 'n1', boardId: 'b1', body: 'a', pinned: true }), + ); + const merges = vi.spyOn(store, 'mergeEntity'); + + client.applyMutationEvent({ + type: 'create', + typename: 'Note', + data: { __typename: 'Note', id: 'n1', boardId: 'b1', body: 'b' }, + }); + await sleep(5); + + expect(merges.mock.calls.map(c => c[0])).toEqual([K('Note', 'n1')]); + // The field the event did not carry survives. + expect(getDoc(kv, K('Note', 'n1'))).toMatchObject({ body: 'b', pinned: true }); + }); +}); + +describe('AsyncQueryStore writer: write skipping while writes are queued', () => { + const clients: QueryClient[] = []; + afterEach(() => { + for (const client of clients.splice(0)) client.destroy(); + }); + + class Badge extends Entity { + // Lingers in memory once released, so a later apply finds the same instance. + static cache = { gcTime: 60 }; + __typename = t.typename('Badge'); + id = t.id; + label = t.string; + } + class User extends Entity { + __typename = t.typename('User'); + id = t.id; + name = t.string; + karma = t.number; + badge = t.optional(t.entity(Badge)); + } + class GetUser extends RESTQuery { + params = { id: t.id }; + path = `/user/${this.params.id}`; + result = { user: t.entity(User) }; + } + + async function setup(users: Record[]) { + const delegate = new AsyncDelegate(3); + const store = writerStore(delegate); + const mockFetch = createMockFetch(); + for (const user of users) mockFetch.get(`/user/${user.id as string}`, { user }); + const client = new QueryClient({ + store, + adapters: [new RESTQueryAdapter({ fetch: mockFetch as never, baseUrl: 'http://localhost' })], + log: { warn: () => {}, error: () => {} }, + } as never); + clients.push(client); + for (const user of users) await holdQuery(client, () => fetchQuery(GetUser, { id: user.id as string })); + await drain(store); + await scanned(store); + const saves = vi.spyOn(store, 'saveEntity'); + const merges = vi.spyOn(store, 'mergeEntity'); + const writes = () => [...saves.mock.calls.map(c => c[0]), ...merges.mock.calls.map(c => c[0])]; + return { delegate, store, client, writes }; + } + + const user = (id: string, karma: number, badge?: Record) => ({ + __typename: 'User', + id, + name: id, + karma, + ...(badge !== undefined ? { badge } : {}), + }); + + it("one entity's queued write leaves skipping on for identical events of other entities", async () => { + const { store, client, writes } = await setup([user('u1', 1), user('u2', 2), user('u3', 3)]); + + client.applyMutationEvent({ type: 'update', typename: 'User', data: { id: 'u1', karma: 10 } }); + client.applyMutationEvent({ type: 'update', typename: 'User', data: { id: 'u2', karma: 2 } }); + client.applyMutationEvent({ type: 'update', typename: 'User', data: { id: 'u3', karma: 3 } }); + // Identical to the write still queued for u1. + client.applyMutationEvent({ type: 'update', typename: 'User', data: { id: 'u1', karma: 10 } }); + expect(store.isSettled()).toBe(false); + await drain(store); + + expect(writes()).toEqual([K('User', 'u1')]); + }); + + it('a queued deletion still turns skipping off', async () => { + const { store, client, writes } = await setup([user('u1', 1)]); + + store.deleteQuery(12345); + client.applyMutationEvent({ type: 'update', typename: 'User', data: { id: 'u1', karma: 1 } }); + await drain(store); + + expect(writes()).toEqual([K('User', 'u1')]); + }); + + it('an entity a queued write stops referencing is written again when it is referenced again', async () => { + class Card extends Entity { + // Lingers in memory once released, so the next event finds the same instance. + static cache = { gcTime: 60 }; + __typename = t.typename('Card'); + id = t.id; + deckId = t.string; + title = t.string; + } + class Deck extends Entity { + __typename = t.typename('Deck'); + id = t.id; + cards = t.liveArray(Card, { constraints: { deckId: (this as any).id } }); + } + class GetDeck extends RESTQuery { + path = '/deck'; + result = { deck: t.entity(Deck) }; + } + const card = { __typename: 'Card', id: 'c1', deckId: 'd1', title: 'Ace' }; + const delegate = new AsyncDelegate(3); + const store = writerStore(delegate); + const mockFetch = createMockFetch(); + mockFetch.get('/deck', { deck: { __typename: 'Deck', id: 'd1', cards: [card] } }); + const client = new QueryClient({ + store, + adapters: [new RESTQueryAdapter({ fetch: mockFetch as never, baseUrl: 'http://localhost' })], + log: { warn: () => {}, error: () => {} }, + } as never); + clients.push(client); + const deckQ = holdQuery(client, () => fetchQuery(GetDeck)); + await deckQ; + await drain(store); + await scanned(store); + + // The deck's write drops its reference to c1, the only one on disk; before + // it is processed, the same card is created again with identical data. + client.applyMutationEvent({ type: 'delete', typename: 'Card', data: 'c1' }); + client.applyMutationEvent({ type: 'create', typename: 'Card', data: card }); + await drain(store); + + expect((deckQ.value as any).deck.cards.map((c: any) => c.title)).toEqual(['Ace']); + expect(Array.from(delegate.kv[refIdsKeyFor(K('Deck', 'd1'))] as Uint32Array)).toEqual([K('Card', 'c1')]); + expect(asyncDoc(delegate, K('Card', 'c1'))).toMatchObject({ title: 'Ace' }); + }); +}); diff --git a/packages/fetchium/src/applyEntities.ts b/packages/fetchium/src/applyEntities.ts index 15d0214..19af1f7 100644 --- a/packages/fetchium/src/applyEntities.ts +++ b/packages/fetchium/src/applyEntities.ts @@ -148,7 +148,7 @@ function applyEntity( return applied.getProxy(entityShape); } - const entityInstance = queryClient.prepareEntity(key, data, entityShape); + const entityInstance = queryClient.prepareEntity(key, data, entityShape, applied); const existingData = entityInstance.data; const isUpdate = existingData !== data; @@ -188,7 +188,12 @@ function applyEntity( entityInstance._partial = false; entityInstance._partialKeys = undefined; } else if (entityInstance._partial && rawKeys !== undefined) { - for (const k of rawKeys) entityInstance._partialKeys!.add(k); + const partialKeys = entityInstance._partialKeys!; + for (const k of rawKeys) partialKeys.add(k); + if (coversShape(partialKeys, shapeFields)) { + entityInstance._partial = false; + entityInstance._partialKeys = undefined; + } } } else { initFields(shapeFields, data, entityInstance, data, seen, queryClient, persist, childRefs, appendMode, created); @@ -201,6 +206,11 @@ function applyEntity( // event did (the parser fills the literal in). if (entityShape.typenameField !== undefined) entityInstance._partialKeys.add(entityShape.typenameField); if (typeof entityShape.idField === 'string') entityInstance._partialKeys.add(entityShape.idField); + // Events that carried every field built a whole record. + if (coversShape(entityInstance._partialKeys, shapeFields)) { + entityInstance._partial = false; + entityInstance._partialKeys = undefined; + } } else if (persist === false) { // Hydrated from the store: its record exists. entityInstance._recorded = true; @@ -231,12 +241,13 @@ function applyEntity( const newRefs = childRefs.size > 0 ? childRefs : undefined; const refsChanged = !sameRefs(entityInstance.entityRefs, newRefs); // An entity hydrated from the store was never written, so there is no write - // to skip. A skip is also only safe while the store has nothing queued: a - // deletion it has yet to process could drop the record this apply trusts. + // to skip. One whose own write is still queued has its current data on the + // way. A skip is also only safe while the store has no deletion queued: it + // could drop the record this apply trusts. const needsPersist = changed || refsChanged || - !entityInstance._persisted || + (!entityInstance._persisted && entityInstance._pendingWrites === 0) || !queryClient.storeReportsDeletes || !queryClient.storeIsSettled(); const writes = persist === true || (persist === 'existing' && isUpdate); @@ -249,6 +260,12 @@ function applyEntity( return proxy; } +/** Whether `keys` names every field of `shape`. */ +function coversShape(keys: Set, shape: Record): boolean { + for (const k in shape) if (!keys.has(k)) return false; + return true; +} + // ====================================================== // Field merge (update path) — reify + merge in one loop // ====================================================== diff --git a/packages/fetchium/src/query-types.ts b/packages/fetchium/src/query-types.ts index 4578cc9..5e309d2 100644 --- a/packages/fetchium/src/query-types.ts +++ b/packages/fetchium/src/query-types.ts @@ -150,6 +150,16 @@ export interface QueryStore { */ isSettled?(): boolean; + /** + * For stores that process writes asynchronously, a narrower `isSettled`: + * whether an operation that could delete a record is still queued. This + * client's own entity writes do not count; for those that drop a + * reference, the client itself writes the referenced entity again. When + * present, the client uses it instead of `isSettled`, so a burst of entity + * writes does not turn write skipping off for every other entity. + */ + hasQueuedDeletes?(): boolean; + /** * Whether the store currently holds a record for this entity key, or * `undefined` while it cannot tell yet (a store that has not finished diff --git a/packages/fetchium/src/stores/async.ts b/packages/fetchium/src/stores/async.ts index 3946c5d..9cf6023 100644 --- a/packages/fetchium/src/stores/async.ts +++ b/packages/fetchium/src/stores/async.ts @@ -124,6 +124,11 @@ export class AsyncQueryStore implements QueryStore { private deleteListeners: Array<(key: number) => void> = []; private persistedListeners: Array<(key: number) => void> = []; private processing = false; + // Queued work that can delete a record: everything except the entity + // writes of this writer's own client, which tracks the references those + // drop (see `hasQueuedDeletes`). + private queuedDeletes = 0; + private readonly ownEntityWrites = new WeakSet(); // The ids whose value the delegate holds (or is about to: a queued write // counts), mirrored by the writer so that `hasEntity` can answer // synchronously. Read from the delegate once at start, without holding up @@ -137,6 +142,7 @@ export class AsyncQueryStore implements QueryStore { onDelete?: (listener: (key: number) => void) => () => void; onPersisted?: (listener: (key: number) => void) => () => void; hasEntity?: (key: number) => boolean | undefined; + hasQueuedDeletes?: () => boolean; constructor(config: AsyncQueryStoreConfig) { this.isWriter = config.isWriter; @@ -144,6 +150,7 @@ export class AsyncQueryStore implements QueryStore { if (this.isWriter) { this.onDelete = listener => subscribe(this.deleteListeners, listener); this.onPersisted = listener => subscribe(this.persistedListeners, listener); + this.hasQueuedDeletes = () => this.queuedDeletes > 0; this.hasEntity = key => { if (this.heldKeys !== undefined) return this.heldKeys.has(key); // Not read yet: only what this writer itself wrote is known. @@ -183,6 +190,7 @@ export class AsyncQueryStore implements QueryStore { } private enqueueMessage(msg: QueuedWork): void { + if (!this.ownEntityWrites.has(msg)) this.queuedDeletes++; this.messageQueue.push(msg); // Wake up the queue processor if it's waiting if (this.resolveQueueWait) { @@ -193,7 +201,10 @@ export class AsyncQueryStore implements QueryStore { private dispatch(msg: StoreMessage): void { if (this.isWriter) { - if (msg.type === StoreMessageType.SaveEntity) this.noteHeld(msg.entityKey); + if (msg.type === StoreMessageType.SaveEntity) { + this.noteHeld(msg.entityKey); + this.ownEntityWrites.add(msg); + } this.enqueueMessage(msg); } else { this.sendMessage(msg); @@ -255,6 +266,7 @@ export class AsyncQueryStore implements QueryStore { } } finally { this.processing = false; + if (!this.ownEntityWrites.has(msg)) this.queuedDeletes--; } } } From e68c8758a653cf77dbb88a39bc0c4fcb96a1960b Mon Sep 17 00:00:00 2001 From: Adam Azbel-Bedell Date: Tue, 6 Oct 2026 00:27:49 -0400 Subject: [PATCH 05/15] perf(fetchium): stop re-running a shared entity's consumers, copying unchanged fetch data and asking the store twice A perf review of the fix found three costs it had added to the happy path. - A query whose result is an entity notified that entity when it attached its extras (__refetch, __hasNext, __isFetchingNext). Every existing consumer of the entity re-ran, and its list's snapshot got a new identity, whenever a detail query for it first applied, even with identical data. The notify is gone. The query's own consumers get the proxy through the relay and snapshot it afresh, so they still see the extras; other consumers see them on their next recompute, as before the fix. - The parser copied every nested t.object and t.record of every fetch (15-21% on a refetch of 1000 entities with nested values). A fetch result or a cached record is now copied only where a parsed value differs from the input, so unchanged nested data costs no allocation. Arrays work the same way (an array of primitives is no longer rebuilt on every fetch); an empty one is always a new array, since a live array grows the array it holds. Event payloads and mutation effects, which the caller may still hold, are still parsed into copies. The parser still never writes into its input: a reified value is written back only when it changed, and a merge into a nested value the adapter froze goes into a copy. - A streamed create asked the store hasEntity even when a live array was about to retain the root, and asked again in save() for an entity built from a partial event. The store is now asked only for a root nothing retains, and save() reuses that answer: routed creates make no kv.has call (a partial one makes one instead of two). An unrouted create keeps its one call; that is what refreshes the record of an entity on disk but not in memory. Co-Authored-By: Claude Opus 5.5 (1M context) --- .changeset/skip-write-and-snapshot-fixes.md | 6 +- packages/fetchium/src/EntityInstance.ts | 5 +- packages/fetchium/src/QueryClient.ts | 34 +++++-- packages/fetchium/src/QueryResult.ts | 11 ++- .../skip-write-and-snapshot-fixes.test.ts | 97 ++++++++++++++++++- packages/fetchium/src/applyEntities.ts | 33 +++++-- packages/fetchium/src/parseEntities.ts | 51 +++++++--- 7 files changed, 193 insertions(+), 44 deletions(-) diff --git a/.changeset/skip-write-and-snapshot-fixes.md b/.changeset/skip-write-and-snapshot-fixes.md index bb5385e..8d3fca3 100644 --- a/.changeset/skip-write-and-snapshot-fixes.md +++ b/.changeset/skip-write-and-snapshot-fixes.md @@ -4,14 +4,14 @@ Fixes for the entity snapshot fast path, unchanged-apply skipping and single-write events introduced in 0.6.0. -- Development builds hand out frozen snapshots, so a consumer that mutates one (a `sort()` in render, an assignment) fails at its own call site with a TypeError. Production snapshots stay mutable; treat them as read-only. The parser no longer writes into the objects it is given, so a snapshot (or part of one) handed back through a mutation event or a mutation's `effects` is read, not mutated, and two events sharing one nested object no longer make two entities share it. +- Development builds hand out frozen snapshots, so a consumer that mutates one (a `sort()` in render, an assignment) fails at its own call site with a TypeError. Production snapshots stay mutable; treat them as read-only. The parser no longer writes into the objects it is given, so a snapshot (or part of one) handed back through a mutation event or a mutation's `effects` is read, not mutated, and two events sharing one nested object no longer make two entities share it. Event payloads are parsed into copies; a fetch result or a cached record is copied only where a parsed value differs from it, so an unchanged nested object, record or array costs no allocation, and a later merge into one the adapter froze goes into a copy. - The dev-only stale-snapshot check no longer throws from inside a snapshot. A throw there escaped Signalium's watcher flush and stopped every consumer on the page from updating. It now serves the re-read value and raises the error out of band, once per field: through the client's `log.error` and then as an uncaught error (a red box in React Native, an uncaught error in the browser, a failed run under vitest). It compares with `Object.is` (so a `NaN` field no longer trips it, and `-0` over `0` is applied at every level), compares a rebuilt `Set` in linear time, and restores the key set of a snapshot a consumer altered. - Hydrating a cached query no longer re-parses an entity that is already in memory. The re-parse ran already-parsed values through the parser again in place, wiping nested optional formats and parse results, and since the apply then saw no change it never notified; consumers of the first query kept stale snapshots. The in-memory data is used as-is once it structurally satisfies the query's shape, nested objects, arrays, records, unions and child entities included (members an array narrows out on read, and children that have left memory, are not held against it); otherwise the whole cached query is dropped and fetched, never served with an item missing. An entity built from streamed events is completed from its record instead. -- A streamed `create` (or an `update` for an entity not in memory) now writes the existing entities its payload changed, and the entities it created only once a written record references them: the root a live array retains is written before the event is routed (a write that fails is logged and nothing is routed, as a failed event always behaved), the rest follow from the records that reference them, and what nothing references is evicted at once, `gcTime` or not. A record's write puts its missing children first; a payload that links back to an ancestor defers both writes to the end of the apply. An entity built from events remembers the fields its events carried: its writes go through the store's new `mergeEntity`, which merges those fields over the record on disk, so the fields the events did not carry survive (a store without `mergeEntity` gets the whole in-memory data through `saveEntity`, as before 0.6). Once its fields are known to be the whole record (the store held no record of it when it was first written, or its events carried every field of its shape), it is written whole like any other entity, without reading the record back. An event for an entity the store holds a record of (`hasEntity`) but that is not in memory refreshes that record the same way; a store that cannot say what it holds is not written, so no record nothing references is left behind. +- A streamed `create` (or an `update` for an entity not in memory) now writes the existing entities its payload changed, and the entities it created only once a written record references them: the root a live array retains is written before the event is routed (a write that fails is logged and nothing is routed, as a failed event always behaved), the rest follow from the records that reference them, and what nothing references is evicted at once, `gcTime` or not. A record's write puts its missing children first; a payload that links back to an ancestor defers both writes to the end of the apply. An entity built from events remembers the fields its events carried: its writes go through the store's new `mergeEntity`, which merges those fields over the record on disk, so the fields the events did not carry survive (a store without `mergeEntity` gets the whole in-memory data through `saveEntity`, as before 0.6). Once its fields are known to be the whole record (the store held no record of it when it was first written, or its events carried every field of its shape), it is written whole like any other entity, without reading the record back. An event for an entity the store holds a record of (`hasEntity`) but that is not in memory refreshes that record the same way; a store that cannot say what it holds is not written, so no record nothing references is left behind. The store is asked only for a created root no live array retains, once per event: each such event costs one `hasEntity` call (a `kv.has` with `SyncQueryStore`). - An entity that appears more than once in one payload (an author listed under `author` and again under `mentions`) is applied once: it keeps its child references and is written once. - `_persisted` is only set once the store write succeeded, so a write that threw is retried by the next apply. `AsyncQueryStore` writers acknowledge processed entity writes (`onPersisted`), report a failed one, answer `hasEntity` from the keys they read from the delegate at start (alongside the queue, so writes do not wait for it; a queued write counts as held) and keep current, run `purgeStaleQueries` through the same serial queue as the writes (a reader with its own delegate still purges directly), ignore anything on the channel that is not one of their messages, and expose `isSettled()` and `hasQueuedDeletes()`; the client counts its own outstanding writes, so an acknowledgement of a write another instance queued does not mark a hydrated instance current. It does not skip a write while the writer has an operation queued that may delete the record the skip relies on (anything but the client's own entity writes); an entity whose own write is still queued counts as written, and an entity that one of the client's queued writes stops referencing is written again if it is referenced again before that write lands. - `AsyncQueryStore` resizes a persisted LRU queue by copying it (growing no longer throws `RangeError`, shrinking evicts the dropped keys instead of stranding them, never the key being activated), and keeps a queue's persisted size for a message without `maxCount` from an older reader. `SyncQueryStore` gets the same resize. - `onDelete` returns an unsubscribe and `QueryClient.destroy()` calls it, so a store no longer retains every client created against it. `SyncQueryStore` shares its delete listeners between every instance over the same backing kv. - Reading a shared-typename entity array consumes the notifiers of the members it currently leaves out, so a reactive consumer learns when a member becomes eligible without a data change, and a consumer that only reads the array is not recomputed when a member it holds changes. - Two classes sharing a typename: fields that are entity-free in one class but hold an entity through the other are no longer classified static for the snapshot fast path. -- Query extras (`__refetch`, `__hasNext`, ...) attached to a shared entity notify existing consumers; a query whose cached value was rejected fetches even within `staleTime`; snapshots of proxies created by a previous (hot-reloaded) module instance are walked instead of returned as-is. +- A query whose cached value was rejected fetches even within `staleTime`; a query whose cached value was rejected fetches even within `staleTime`; snapshots of proxies created by a previous (hot-reloaded) module instance are walked instead of returned as-is. diff --git a/packages/fetchium/src/EntityInstance.ts b/packages/fetchium/src/EntityInstance.ts index c7b694f..c2c8e93 100644 --- a/packages/fetchium/src/EntityInstance.ts +++ b/packages/fetchium/src/EntityInstance.ts @@ -685,7 +685,8 @@ export class EntityInstance { return false; } - save(): void { + /** `storeHolds`: what the store's `hasEntity` just said about this entity, if the caller asked. */ + save(storeHolds?: boolean): void { const client = this._queryClient; if (this._saving) return; this._saving = true; @@ -732,7 +733,7 @@ export class EntityInstance { // With no record to merge over, the fields this entity holds are the // whole record, so it stops being partial and later writes skip the // merge's read of the stored record. - if (this._partial && !this._recorded && client.store.hasEntity?.(this.key) === false) { + if (this._partial && !this._recorded && (storeHolds ?? client.store.hasEntity?.(this.key)) === false) { this._partial = false; this._partialKeys = undefined; } diff --git a/packages/fetchium/src/QueryClient.ts b/packages/fetchium/src/QueryClient.ts index dad9585..c699fa9 100644 --- a/packages/fetchium/src/QueryClient.ts +++ b/packages/fetchium/src/QueryClient.ts @@ -712,11 +712,20 @@ export class QueryClient { /** * Parse data: validates, formats, produces parsed entity data objects. * Does NOT touch the entity store. Call applyRefs() after to commit entities. + * + * `copyInput: false` (a fetch result or a cached record, handed over to the + * client) copies a nested object only where a parsed value differs from it. */ - parseData(obj: unknown, shape: InternalTypeDef, preloadedEntities?: PreloadedEntityMap): ParseResult { + parseData( + obj: unknown, + shape: InternalTypeDef, + preloadedEntities?: PreloadedEntityMap, + copyInput: boolean = true, + ): ParseResult { const warn = this.context.log?.warn ?? (() => {}); const ctx = new ParseContext(); ctx.reset(this, preloadedEntities, warn); + ctx.copyInput = copyInput; const data = parseEntities(obj, shape as unknown as ComplexTypeDef, ctx); return { data, ctx }; } @@ -756,7 +765,7 @@ export class QueryClient { obj = { ...(obj as Record), [QUERY_ID]: queryId }; } - const parseResult = this.parseData(obj, rootEntityShape as unknown as InternalTypeDef, preloadedEntities); + const parseResult = this.parseData(obj, rootEntityShape as unknown as InternalTypeDef, preloadedEntities, false); const result = applyEntityRefs(parseResult.ctx, parseResult.data, persist, appendMode); // Discover the root entity from the returned proxy @@ -877,14 +886,19 @@ export class QueryClient { undefined, /* dryRun */ true, ); - const held = this.store.hasEntity?.(key) === true; - if ((retains || held) && !entity._persisted && entity._pendingWrites === 0) { - try { - entity.save(); - } catch (e) { - this.context.log?.warn?.('Failed to apply mutation event', e); - this.evictUnlessAdopted(entity, created!); - return; + // The store is asked only when no live array retains the root, and only + // when there is something to write; its answer is handed to save(). + let held: boolean | undefined; + if (!entity._persisted && entity._pendingWrites === 0) { + if (!retains) held = this.store.hasEntity?.(key); + if (retains || held === true) { + try { + entity.save(held); + } catch (e) { + this.context.log?.warn?.('Failed to apply mutation event', e); + this.evictUnlessAdopted(entity, created!); + return; + } } } diff --git a/packages/fetchium/src/QueryResult.ts b/packages/fetchium/src/QueryResult.ts index 3c2ea52..be7c832 100644 --- a/packages/fetchium/src/QueryResult.ts +++ b/packages/fetchium/src/QueryResult.ts @@ -271,10 +271,13 @@ export class QueryInstance { __hasNext: () => this.hasNext, __isFetchingNext: () => this._fetchNextPromise !== undefined, }; - // The entity's key set just changed. A shared entity (a query whose - // result is an entity) may already have consumers, and an apply that - // found its data unchanged will not have notified them. - this.rootEntity.notify(); + // The entity's key set just changed, but its consumers are not + // notified: this query's own consumers get the proxy through the relay + // and snapshot it afresh, and an existing consumer of a shared entity + // (a list showing the entity this query returns) has no use for another + // query's extras. Notifying would re-run every one of them, and give + // their snapshots a new identity, for data that did not change. They + // see the extras on their next recompute, as before. } return this.rootEntity.getProxy(def.statics.shape as unknown as EntityDef) as QueryResult; diff --git a/packages/fetchium/src/__tests__/skip-write-and-snapshot-fixes.test.ts b/packages/fetchium/src/__tests__/skip-write-and-snapshot-fixes.test.ts index c9664f3..81b098f 100644 --- a/packages/fetchium/src/__tests__/skip-write-and-snapshot-fixes.test.ts +++ b/packages/fetchium/src/__tests__/skip-write-and-snapshot-fixes.test.ts @@ -438,6 +438,85 @@ describe('hydration of a cached query whose entity is already in memory', () => expect(h.read().doc.meta.views).toBe(2); expect(snap.meta.views).toBe(1); }); + + it('a frozen fetch result is applied without being written to, and later merges reach its nested values', async () => { + class DocNote extends Entity { + __typename = t.typename('DocNote'); + id = t.id; + docId = t.string; + } + class Doc extends Entity { + __typename = t.typename('Doc'); + id = t.id; + title = t.string; + meta = t.object({ views: t.number, stats: t.object({ likes: t.number }) }); + counts = t.record(t.number); + tags = t.array(t.string); + notes = t.liveArray(DocNote, { constraints: { docId: (this as any).id } }); + } + class GetDoc extends RESTQuery { + path = '/doc'; + result = { doc: t.entity(Doc), note: t.object({ text: t.string }) }; + } + const deepFreeze = (value: T): T => { + if (typeof value === 'object' && value !== null) { + for (const v of Object.values(value)) deepFreeze(v); + Object.freeze(value); + } + return value; + }; + const body = deepFreeze({ + doc: { + __typename: 'Doc', + id: 'd', + title: 'T', + meta: { views: 1, stats: { likes: 3 } }, + counts: { a: 1 }, + tags: ['x'], + notes: [], + }, + note: { text: 'n' }, + }); + const before = JSON.stringify(body); + // Hands out the same frozen object on every call, as an adapter that + // keeps its last response may. + const fetch = async () => ({ ok: true, status: 200, headers: new Headers(), json: async () => body }); + const { client, warn } = makeClient( + new SyncQueryStore(new MemoryPersistentStore()), + fetch as unknown as ReturnType, + ); + const h = snapshotHarness(client, () => fetchQuery(GetDoc)); + await h.query; + expect(h.read().doc.meta.stats.likes).toBe(3); + expect(h.read().note.text).toBe('n'); + + client.applyMutationEvent({ + type: 'update', + typename: 'Doc', + data: { id: 'd', meta: { views: 2, stats: { likes: 4 } }, counts: { a: 2 } }, + }); + expect(warn).not.toHaveBeenCalled(); + expect(h.read().doc.meta.views).toBe(2); + expect(h.read().doc.meta.stats.likes).toBe(4); + expect(h.read().doc.counts.a).toBe(2); + // The live array grows its own array, not the response's (empty, frozen) one. + client.applyMutationEvent({ + type: 'create', + typename: 'DocNote', + data: { __typename: 'DocNote', id: 'n1', docId: 'd' }, + }); + expect(h.read().doc.notes.map((n: { id: string }) => n.id)).toEqual(['n1']); + expect(h.read().doc.tags).toEqual(['x']); + + // A refetch of the same (frozen) response puts the old values back. + await (h.query as any).value.__refetch(); + await sleep(5); + expect(h.read().doc.meta.stats.likes).toBe(3); + expect(h.read().doc.counts.a).toBe(1); + expect(warn).not.toHaveBeenCalled(); + expect(JSON.stringify(body)).toBe(before); + client.destroy(); + }); }); // ====================================================== @@ -992,7 +1071,7 @@ describe('snapshot fast path follow-ups', () => { }); }); - it('query extras attached to a shared entity reach an existing consumer without another trigger', async () => { + it('query extras attached to a shared entity reach the query itself without re-running its other consumers', async () => { const { client, mockFetch } = getClient(); class Profile extends Entity { __typename = t.typename('Profile'); @@ -1010,13 +1089,27 @@ describe('snapshot fast path follow-ups', () => { const profile = { __typename: 'Profile', id: 1, name: 'Alice' }; mockFetch.get('/wrapped', { profile }); mockFetch.get('/root', profile); + mockFetch.get('/wrapped', { profile: { ...profile, name: 'Alicia' } }); const wrapped = snapshotHarness(client, () => fetchQuery(GetWrapped)); await wrapped.query; - expect('__refetch' in wrapped.read().profile).toBe(false); + const before = wrapped.read(); + const computesBefore = wrapped.computes(); + expect('__refetch' in before.profile).toBe(false); + // The entity-rooted query applies identical data: its own consumer sees + // the extras, the wrapped query's consumer is not re-run for them. const root = snapshotHarness(client, () => fetchQuery(GetRoot)); await root.query; + await sleep(5); + expect(typeof root.read().__refetch).toBe('function'); + expect(wrapped.read()).toBe(before); + expect(wrapped.computes()).toBe(computesBefore); + + // Its next recompute (here, a change to the entity) picks them up. + await (wrapped.query as any).value.__refetch(); + await sleep(5); + expect(wrapped.read().profile.name).toBe('Alicia'); expect(typeof wrapped.read().profile.__refetch).toBe('function'); }); diff --git a/packages/fetchium/src/applyEntities.ts b/packages/fetchium/src/applyEntities.ts index 19af1f7..695babf 100644 --- a/packages/fetchium/src/applyEntities.ts +++ b/packages/fetchium/src/applyEntities.ts @@ -288,8 +288,12 @@ function mergeFields( for (const [fieldKey, propShape] of entries(shape)) { if (rawKeys !== undefined && !rawKeys.has(fieldKey)) continue; - if (shouldReify(data[fieldKey])) { - data[fieldKey] = reifyAndApply(data[fieldKey], seen, queryClient, persist, childRefs, appendMode, created); + const raw = data[fieldKey]; + if (shouldReify(raw)) { + // Written only when it changes: an object the parser kept as is (a + // fetch result's) may belong to the adapter, and be frozen. + const reified = reifyAndApply(raw, seen, queryClient, persist, childRefs, appendMode, created); + if (reified !== raw) data[fieldKey] = reified; } if (propShape instanceof ValidatorDef && propShape._liveConfig !== undefined) { @@ -332,12 +336,15 @@ function mergeFields( !(propShape.shape instanceof Set) ? (propShape.shape as Record) : undefined; + // The parser keeps a fetch result's unchanged objects as they are, and + // an adapter may hand out frozen ones: merge into a copy of those. + const target = Object.isFrozen(oldVal) ? { ...oldVal } : oldVal; if (nestedShape !== undefined) { if ( mergeFields( nestedShape, newVal, - oldVal, + target, undefined, entityInstance, entityData, @@ -350,18 +357,22 @@ function mergeFields( ) ) { changed = true; + existingData[fieldKey] = target; + } else { + existingData[fieldKey] = oldVal; } - existingData[fieldKey] = oldVal; } else if (sameKeys(oldVal, newVal)) { // Shapeless object (e.g. a record) with the same keys: copy field by field, keeping the // existing value — and its identity — wherever it already matches. + let merged = false; for (const k of Object.keys(newVal)) { if (!sameValue(oldVal[k], newVal[k])) { - oldVal[k] = newVal[k]; - changed = true; + target[k] = newVal[k]; + merged = true; } } - existingData[fieldKey] = oldVal; + if (merged) changed = true; + existingData[fieldKey] = merged ? target : oldVal; } else { // A key was added or removed. Copying field by field would never // apply a removal, leaving the old key in `data` indefinitely. @@ -462,8 +473,12 @@ function initFields( for (const [fieldKey, propShape] of entries(shape)) { if (!(fieldKey in data)) continue; - if (shouldReify(data[fieldKey])) { - data[fieldKey] = reifyAndApply(data[fieldKey], seen, queryClient, persist, childRefs, appendMode, created); + const raw = data[fieldKey]; + if (shouldReify(raw)) { + // Written only when it changes: an object the parser kept as is (a + // fetch result's) may belong to the adapter, and be frozen. + const reified = reifyAndApply(raw, seen, queryClient, persist, childRefs, appendMode, created); + if (reified !== raw) data[fieldKey] = reified; } if (propShape instanceof ValidatorDef && propShape._liveConfig !== undefined) { diff --git a/packages/fetchium/src/parseEntities.ts b/packages/fetchium/src/parseEntities.ts index 4abaded..f71e08e 100644 --- a/packages/fetchium/src/parseEntities.ts +++ b/packages/fetchium/src/parseEntities.ts @@ -74,6 +74,14 @@ export class ParseContext { seenByKey: Map | undefined = undefined; /** Verdicts of the trust-memory check, per instance and def, for one parse. */ trusted: TrustMemo | undefined = undefined; + /** + * Whether a nested object or record is always parsed into a copy. On for + * input the client does not own and may see again (an event payload, a + * mutation's effects, a consumer's snapshot fed back). Off for a fetch + * result or a cached record, which is copied only where a parsed value + * differs from what it holds: unchanged input costs no allocation. + */ + copyInput: boolean = true; reset( queryClient: QueryClient | undefined, @@ -86,6 +94,7 @@ export class ParseContext { this.warn = warn; this.isPartialEvent = isPartialEvent; this.trusted = undefined; + this.copyInput = true; if (queryClient !== undefined) { if (this.seen === undefined) { this.seen = new Map(); @@ -386,28 +395,36 @@ function parseUnionData( // ====================================================== function parseArrayData(array: unknown[], itemShape: ComplexTypeDef, ctx: ParseContext, path: string): unknown[] { - const result: unknown[] = []; + // Without `ctx.copyInput`, an array whose items all parse to themselves is + // returned as is; the copy starts at the first item that differs or fails. + // An empty one is always a new array: a live array grows the one it holds. + let result: unknown[] | undefined = ctx.copyInput || array.length === 0 ? [] : undefined; for (let i = 0; i < array.length; i++) { + const item = array[i]; try { - result.push(parseData(array[i], itemShape as unknown as TypeDef, ctx, `${path}[${i}]`)); + const parsed = parseData(item, itemShape as unknown as TypeDef, ctx, `${path}[${i}]`); + if (result !== undefined) result.push(parsed); + else if (parsed !== item) (result = array.slice(0, i)).push(parsed); } catch (e) { if (e instanceof CachedEntityMismatchError) throw e; + result ??= array.slice(0, i); ctx.warn('Failed to parse array item, filtering out', { index: i, - value: array[i], + value: item, error: e instanceof Error ? e.message : String(e), }); } } - return result; + return result ?? array; } -// Both walkers parse into a shallow copy of their input, like `parseArrayData` -// builds a new array. The input belongs to whoever handed it in (an adapter, -// an event payload, a consumer's snapshot — frozen in dev builds) and must not -// be written to; the copy keeps every key the input had, as before. +// Neither walker writes into its input. With `ctx.copyInput` they parse into +// a shallow copy, as `parseArrayData` builds a new array; without it the copy +// is made only once a parsed value differs from the input's, and input whose +// values all parse to themselves is returned as is. Either way the result +// keeps every key the input had, as before. function parseRecordData( record: Record, @@ -415,12 +432,14 @@ function parseRecordData( ctx: ParseContext, path: string, ): Record { - const result: Record = { ...record }; + let result: Record | undefined = ctx.copyInput ? { ...record } : undefined; for (const [key, value] of entries(record)) { - result[key] = parseData(value, valueShape as unknown as TypeDef, ctx, `${path}["${key}"]`); + const parsed = parseData(value, valueShape as unknown as TypeDef, ctx, `${path}["${key}"]`); + if (result !== undefined) result[key] = parsed; + else if (parsed !== value) (result = { ...record })[key] = parsed; } - return result; + return result ?? record; } function parseObjectData( @@ -434,13 +453,17 @@ function parseObjectData( } const shape = objectShape.shape; - const result: Record = { ...obj }; + let result: Record | undefined = ctx.copyInput ? { ...obj } : undefined; for (const [key, propShape] of entries(shape)) { - result[key] = parseData(obj[key], propShape as unknown as TypeDef, ctx, `${path}.${key}`); + const value = obj[key]; + const parsed = parseData(value, propShape as unknown as TypeDef, ctx, `${path}.${key}`); + if (result !== undefined) result[key] = parsed; + // A shape key the input lacks is still set (to undefined), as a copy does. + else if (parsed !== value || (value === undefined && !(key in obj))) (result = { ...obj })[key] = parsed; } - return result; + return result ?? obj; } // ====================================================== From e583e75fd2e6d9e8f5a78dbcf2d4e0d5757594ce Mon Sep 17 00:00:00 2001 From: Adam Azbel-Bedell Date: Tue, 6 Oct 2026 11:27:11 -0400 Subject: [PATCH 06/15] chore(fetchium): major changeset listing the behavior changes an upgrade brings Co-Authored-By: Claude Opus 5.5 (1M context) --- .changeset/upgrade-behavior-changes.md | 32 ++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .changeset/upgrade-behavior-changes.md diff --git a/.changeset/upgrade-behavior-changes.md b/.changeset/upgrade-behavior-changes.md new file mode 100644 index 0000000..0ca94b7 --- /dev/null +++ b/.changeset/upgrade-behavior-changes.md @@ -0,0 +1,32 @@ +--- +'fetchium': major +--- + +Upgrade notes: behavior an app can observe without using any new API. Each change has its own entry. + +Rendering + +- With `SyncQueryStore`, a cached query renders its data on the first render, with no loading frame. Tests that wait for a loading state or count renders may need updating. +- A refetch, poll or streamed update with identical data does not re-render. `isPending` and `isFetching` still change. +- `t.record` and `t.result` fields apply updates; a record field used to keep its old value. +- List snapshots keep each row's identity when the list is inserted into or re-sorted. +- A nested `t.object` or `t.record` value, or a live collection's value, is a new object after an update changes it, and keeps its identity otherwise. +- Development builds freeze query results, so mutating one throws a `TypeError`. Treat production results as read-only too. + +Fetching + +- The first fetch, and a refetch without `debounce`, start on a microtask instead of after `setTimeout(0)`. +- On resume, a subscription is restored before the request is sent, so a socket that replays a cached value no longer overwrites the fresh response. +- Changing a Signal param mid-fetch aborts that fetch and fetches the new params. A late response for the old params is dropped. +- A query unmounted mid-fetch refetches as soon as it mounts again, without showing the `AbortError`. +- `retry: {}` and `retry: { retryDelay }` retry like `retry: true`. A `shouldRetry` hook can stop retrying permanent errors. +- `shouldRetry`, `reactivationGraceMs`, `reactivationStaggerMs`, `activity` and `pollResumeJitterMs` are reserved `QueryClientConfig` names. Other keys still reach queries as `this.context`. +- `QueryClient.destroy()` aborts every fetch in flight (whoever awaits one gets an `AbortError`), and nothing is written to the store after it. + +Cache and store + +- A cached entry that can't be applied is treated as a miss and fetched, even within `staleTime`. +- A query whose result is not an entity caches each params key separately. Caches from earlier versions are corrected as each key is fetched again. +- Streamed events persist the entities they change. +- `QueryStore` gains optional `mergeEntity`, `onDelete`, `onPersisted`, `isSettled`, `hasQueuedDeletes` and `hasEntity`. A custom store without them writes as before. A store write that throws is retried. +- `AsyncQueryStore` honors `cache.maxCount`, and both built-in stores resize their LRU queue when it changes. From 48971039af5413373b70ef5fc98e5b20f9bf5f56 Mon Sep 17 00:00:00 2001 From: Adam Azbel-Bedell Date: Tue, 6 Oct 2026 11:27:11 -0400 Subject: [PATCH 07/15] fix(fetchium): keep merging an event-built entity whose events carried every field of its class Two entity classes can share a typename: a list class, and a detail class that declares more fields. When only the list class was registered in a session, a streamed update for an entity not in memory was parsed with the list class alone. The event carried every field of that class, so the entity was marked whole and written with saveEntity, which replaced its record and deleted the detail class's fields. At the next launch the detail's cached query could not hydrate and waited for the network. The same happened to an entity an event created under a root in memory. Covering every field of a class does not show that the event covers the record: another class, possibly one not registered this session, may have written more fields to it. An event-built entity now stays partial and is merged into its record. It still becomes whole when the store holds no record of it at its first write. Co-Authored-By: Claude Opus 5.5 (1M context) --- .changeset/skip-write-and-snapshot-fixes.md | 2 +- .../src/__tests__/event-persistence.test.ts | 136 ++++++++++++++++++ packages/fetchium/src/applyEntities.ts | 27 ++-- 3 files changed, 147 insertions(+), 18 deletions(-) diff --git a/.changeset/skip-write-and-snapshot-fixes.md b/.changeset/skip-write-and-snapshot-fixes.md index 8d3fca3..a45376b 100644 --- a/.changeset/skip-write-and-snapshot-fixes.md +++ b/.changeset/skip-write-and-snapshot-fixes.md @@ -7,7 +7,7 @@ Fixes for the entity snapshot fast path, unchanged-apply skipping and single-wri - Development builds hand out frozen snapshots, so a consumer that mutates one (a `sort()` in render, an assignment) fails at its own call site with a TypeError. Production snapshots stay mutable; treat them as read-only. The parser no longer writes into the objects it is given, so a snapshot (or part of one) handed back through a mutation event or a mutation's `effects` is read, not mutated, and two events sharing one nested object no longer make two entities share it. Event payloads are parsed into copies; a fetch result or a cached record is copied only where a parsed value differs from it, so an unchanged nested object, record or array costs no allocation, and a later merge into one the adapter froze goes into a copy. - The dev-only stale-snapshot check no longer throws from inside a snapshot. A throw there escaped Signalium's watcher flush and stopped every consumer on the page from updating. It now serves the re-read value and raises the error out of band, once per field: through the client's `log.error` and then as an uncaught error (a red box in React Native, an uncaught error in the browser, a failed run under vitest). It compares with `Object.is` (so a `NaN` field no longer trips it, and `-0` over `0` is applied at every level), compares a rebuilt `Set` in linear time, and restores the key set of a snapshot a consumer altered. - Hydrating a cached query no longer re-parses an entity that is already in memory. The re-parse ran already-parsed values through the parser again in place, wiping nested optional formats and parse results, and since the apply then saw no change it never notified; consumers of the first query kept stale snapshots. The in-memory data is used as-is once it structurally satisfies the query's shape, nested objects, arrays, records, unions and child entities included (members an array narrows out on read, and children that have left memory, are not held against it); otherwise the whole cached query is dropped and fetched, never served with an item missing. An entity built from streamed events is completed from its record instead. -- A streamed `create` (or an `update` for an entity not in memory) now writes the existing entities its payload changed, and the entities it created only once a written record references them: the root a live array retains is written before the event is routed (a write that fails is logged and nothing is routed, as a failed event always behaved), the rest follow from the records that reference them, and what nothing references is evicted at once, `gcTime` or not. A record's write puts its missing children first; a payload that links back to an ancestor defers both writes to the end of the apply. An entity built from events remembers the fields its events carried: its writes go through the store's new `mergeEntity`, which merges those fields over the record on disk, so the fields the events did not carry survive (a store without `mergeEntity` gets the whole in-memory data through `saveEntity`, as before 0.6). Once its fields are known to be the whole record (the store held no record of it when it was first written, or its events carried every field of its shape), it is written whole like any other entity, without reading the record back. An event for an entity the store holds a record of (`hasEntity`) but that is not in memory refreshes that record the same way; a store that cannot say what it holds is not written, so no record nothing references is left behind. The store is asked only for a created root no live array retains, once per event: each such event costs one `hasEntity` call (a `kv.has` with `SyncQueryStore`). +- A streamed `create` (or an `update` for an entity not in memory) now writes the existing entities its payload changed, and the entities it created only once a written record references them: the root a live array retains is written before the event is routed (a write that fails is logged and nothing is routed, as a failed event always behaved), the rest follow from the records that reference them, and what nothing references is evicted at once, `gcTime` or not. A record's write puts its missing children first; a payload that links back to an ancestor defers both writes to the end of the apply. An entity built from events remembers the fields its events carried: its writes go through the store's new `mergeEntity`, which merges those fields over the record on disk, so the fields the events did not carry survive (a store without `mergeEntity` gets the whole in-memory data through `saveEntity`, as before 0.6). Once its fields are known to be the whole record (the store held no record of it when it was first written), it is written whole like any other entity, without reading the record back. An event that carries every field of the entity's class does not make it whole: another class sharing the typename, possibly not registered this session, may have written fields to the record that this class does not declare, and a merge keeps them. An event for an entity the store holds a record of (`hasEntity`) but that is not in memory refreshes that record the same way; a store that cannot say what it holds is not written, so no record nothing references is left behind. The store is asked only for a created root no live array retains, once per event: each such event costs one `hasEntity` call (a `kv.has` with `SyncQueryStore`). - An entity that appears more than once in one payload (an author listed under `author` and again under `mentions`) is applied once: it keeps its child references and is written once. - `_persisted` is only set once the store write succeeded, so a write that threw is retried by the next apply. `AsyncQueryStore` writers acknowledge processed entity writes (`onPersisted`), report a failed one, answer `hasEntity` from the keys they read from the delegate at start (alongside the queue, so writes do not wait for it; a queued write counts as held) and keep current, run `purgeStaleQueries` through the same serial queue as the writes (a reader with its own delegate still purges directly), ignore anything on the channel that is not one of their messages, and expose `isSettled()` and `hasQueuedDeletes()`; the client counts its own outstanding writes, so an acknowledgement of a write another instance queued does not mark a hydrated instance current. It does not skip a write while the writer has an operation queued that may delete the record the skip relies on (anything but the client's own entity writes); an entity whose own write is still queued counts as written, and an entity that one of the client's queued writes stops referencing is written again if it is referenced again before that write lands. - `AsyncQueryStore` resizes a persisted LRU queue by copying it (growing no longer throws `RangeError`, shrinking evicts the dropped keys instead of stranding them, never the key being activated), and keeps a queue's persisted size for a message without `maxCount` from an older reader. `SyncQueryStore` gets the same resize. diff --git a/packages/fetchium/src/__tests__/event-persistence.test.ts b/packages/fetchium/src/__tests__/event-persistence.test.ts index ef429cc..cc771de 100644 --- a/packages/fetchium/src/__tests__/event-persistence.test.ts +++ b/packages/fetchium/src/__tests__/event-persistence.test.ts @@ -1755,3 +1755,139 @@ describe('AsyncQueryStore writer: write skipping while writes are queued', () => expect(asyncDoc(delegate, K('Card', 'c1'))).toMatchObject({ title: 'Ace' }); }); }); + +// ====================================================== +// A typename two classes share, across sessions +// ====================================================== + +describe('an event for a typename whose other class is not registered this session', () => { + class Tag extends Entity { + __typename = t.typename('Tag'); + id = t.id; + label = t.string; + } + // A list's class and a detail class of one typename: a list-shaped event + // carries every field of the list's class, but not the detail's. + class CoinRow extends Entity { + __typename = t.typename('Coin'); + id = t.id; + symbol = t.string; + price = t.number; + } + class CoinDetail extends Entity { + __typename = t.typename('Coin'); + id = t.id; + symbol = t.string; + price = t.number; + about = t.string; + tag = t.entity(Tag); + } + class Market extends Entity { + __typename = t.typename('Market'); + id = t.id; + top = t.entity(CoinRow); + } + class GetCoinDetail extends RESTQuery { + path = '/coin'; + result = { coin: t.entity(CoinDetail) }; + } + class GetCoinRows extends RESTQuery { + path = '/coins'; + result = { coins: t.array(t.entity(CoinRow)) }; + } + class GetMarket extends RESTQuery { + path = '/market'; + result = { market: t.entity(Market) }; + } + + const clients: QueryClient[] = []; + afterEach(() => { + for (const client of clients.splice(0)) client.destroy(); + }); + + const detailCoin = { + __typename: 'Coin', + id: 'c1', + symbol: 'C', + price: 1, + about: 'cached', + tag: { __typename: 'Tag', id: 'g1', label: 'gold' }, + }; + const row = (id: string, price: number) => ({ __typename: 'Coin', id, symbol: 'C', price }); + const detailRecord = (price: number) => ({ + __typename: 'Coin', + id: 'c1', + symbol: 'C', + price, + about: 'cached', + tag: { __entityRef: K('Tag', 'g1') }, + }); + + function session(kv: MemoryPersistentStore, mockFetch: ReturnType): QueryClient { + const client = new QueryClient({ + store: new SyncQueryStore(kv), + adapters: [new RESTQueryAdapter({ fetch: mockFetch as never, baseUrl: 'http://localhost' })], + } as never); + clients.push(client); + return client; + } + + /** A first session caches the detail of c1, then ends. */ + async function cacheDetail(kv: MemoryPersistentStore): Promise> { + const mockFetch = createMockFetch(); + mockFetch.get('/coin', { coin: detailCoin }); + const first = session(kv, mockFetch); + await holdQuery(first, () => fetchQuery(GetCoinDetail)); + first.destroy(); + return mockFetch; + } + + /** The next session's detail query, served from the cache before its slow fetch lands. */ + async function detailAtNextStart( + kv: MemoryPersistentStore, + mockFetch: ReturnType, + ): Promise { + mockFetch.get('/coin', { coin: { ...detailCoin, about: 'network' } }, { delay: 200 }); + const next = session(kv, mockFetch); + const detailQ = holdQuery(next, () => fetchQuery(GetCoinDetail)); + await sleep(20); + return detailQ.isReady ? (detailQ.value as any).coin.about : 'not ready'; + } + + it('an update for an entity not in memory merges into its record instead of replacing it', async () => { + const kv = new MemoryPersistentStore(); + const mockFetch = await cacheDetail(kv); + + // Only the list's class is registered (a list of other coins); c1 is not in memory. + mockFetch.get('/coins', { coins: [row('c2', 2)] }); + const client = session(kv, mockFetch); + await holdQuery(client, () => fetchQuery(GetCoinRows)); + client.applyMutationEvent({ type: 'update', typename: 'Coin', data: row('c1', 3) }); + + expect(getDoc(kv, K('Coin', 'c1'))).toEqual(detailRecord(3)); + expect(refIds(kv, K('Coin', 'c1'))).toEqual([K('Tag', 'g1')]); + client.destroy(); + expect(await detailAtNextStart(kv, mockFetch)).toBe('cached'); + }); + + it('an entity an event builds under a root in memory merges into its record too', async () => { + const kv = new MemoryPersistentStore(); + const mockFetch = await cacheDetail(kv); + + mockFetch.get('/market', { market: { __typename: 'Market', id: 'm1', top: row('c2', 2) } }); + const client = session(kv, mockFetch); + await holdQuery(client, () => fetchQuery(GetMarket)); + client.applyMutationEvent({ + type: 'update', + typename: 'Market', + data: { __typename: 'Market', id: 'm1', top: row('c1', 3) }, + }); + // A later event for the in-memory entity still merges. + client.applyMutationEvent({ type: 'update', typename: 'Coin', data: row('c1', 4) }); + + expect(getDoc(kv, K('Coin', 'c1'))).toEqual(detailRecord(4)); + expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); + client.destroy(); + expect(await detailAtNextStart(kv, mockFetch)).toBe('cached'); + }); +}); diff --git a/packages/fetchium/src/applyEntities.ts b/packages/fetchium/src/applyEntities.ts index 695babf..f9722a5 100644 --- a/packages/fetchium/src/applyEntities.ts +++ b/packages/fetchium/src/applyEntities.ts @@ -188,12 +188,11 @@ function applyEntity( entityInstance._partial = false; entityInstance._partialKeys = undefined; } else if (entityInstance._partial && rawKeys !== undefined) { - const partialKeys = entityInstance._partialKeys!; - for (const k of rawKeys) partialKeys.add(k); - if (coversShape(partialKeys, shapeFields)) { - entityInstance._partial = false; - entityInstance._partialKeys = undefined; - } + // Stays partial even once the events have carried every field of + // this class: another class sharing the typename may have written + // fields to the record that this one does not declare, and only a + // merge keeps them. + for (const k of rawKeys) entityInstance._partialKeys!.add(k); } } else { initFields(shapeFields, data, entityInstance, data, seen, queryClient, persist, childRefs, appendMode, created); @@ -206,11 +205,11 @@ function applyEntity( // event did (the parser fills the literal in). if (entityShape.typenameField !== undefined) entityInstance._partialKeys.add(entityShape.typenameField); if (typeof entityShape.idField === 'string') entityInstance._partialKeys.add(entityShape.idField); - // Events that carried every field built a whole record. - if (coversShape(entityInstance._partialKeys, shapeFields)) { - entityInstance._partial = false; - entityInstance._partialKeys = undefined; - } + // An event that carried every field of this class is still partial: + // the record may hold fields another class sharing the typename + // declares (one not registered this session, too), so it is merged + // into, not replaced. With no record to merge into, the first write + // makes it whole (see `EntityInstance.save()`). } else if (persist === false) { // Hydrated from the store: its record exists. entityInstance._recorded = true; @@ -260,12 +259,6 @@ function applyEntity( return proxy; } -/** Whether `keys` names every field of `shape`. */ -function coversShape(keys: Set, shape: Record): boolean { - for (const k in shape) if (!keys.has(k)) return false; - return true; -} - // ====================================================== // Field merge (update path) — reify + merge in one loop // ====================================================== From 94a754ef33a8c08a099f0699d396cc825dd8b20c Mon Sep 17 00:00:00 2001 From: Adam Azbel-Bedell Date: Tue, 6 Oct 2026 11:27:11 -0400 Subject: [PATCH 08/15] chore(fetchium): shorten this change's changesets for app developers Co-Authored-By: Claude Opus 5.5 (1M context) --- .changeset/skip-write-and-snapshot-fixes.md | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/.changeset/skip-write-and-snapshot-fixes.md b/.changeset/skip-write-and-snapshot-fixes.md index a45376b..7a268ec 100644 --- a/.changeset/skip-write-and-snapshot-fixes.md +++ b/.changeset/skip-write-and-snapshot-fixes.md @@ -2,16 +2,13 @@ 'fetchium': patch --- -Fixes for the entity snapshot fast path, unchanged-apply skipping and single-write events introduced in 0.6.0. +Fixes for snapshots, unchanged-data skipping and streamed-event writes. -- Development builds hand out frozen snapshots, so a consumer that mutates one (a `sort()` in render, an assignment) fails at its own call site with a TypeError. Production snapshots stay mutable; treat them as read-only. The parser no longer writes into the objects it is given, so a snapshot (or part of one) handed back through a mutation event or a mutation's `effects` is read, not mutated, and two events sharing one nested object no longer make two entities share it. Event payloads are parsed into copies; a fetch result or a cached record is copied only where a parsed value differs from it, so an unchanged nested object, record or array costs no allocation, and a later merge into one the adapter froze goes into a copy. -- The dev-only stale-snapshot check no longer throws from inside a snapshot. A throw there escaped Signalium's watcher flush and stopped every consumer on the page from updating. It now serves the re-read value and raises the error out of band, once per field: through the client's `log.error` and then as an uncaught error (a red box in React Native, an uncaught error in the browser, a failed run under vitest). It compares with `Object.is` (so a `NaN` field no longer trips it, and `-0` over `0` is applied at every level), compares a rebuilt `Set` in linear time, and restores the key set of a snapshot a consumer altered. -- Hydrating a cached query no longer re-parses an entity that is already in memory. The re-parse ran already-parsed values through the parser again in place, wiping nested optional formats and parse results, and since the apply then saw no change it never notified; consumers of the first query kept stale snapshots. The in-memory data is used as-is once it structurally satisfies the query's shape, nested objects, arrays, records, unions and child entities included (members an array narrows out on read, and children that have left memory, are not held against it); otherwise the whole cached query is dropped and fetched, never served with an item missing. An entity built from streamed events is completed from its record instead. -- A streamed `create` (or an `update` for an entity not in memory) now writes the existing entities its payload changed, and the entities it created only once a written record references them: the root a live array retains is written before the event is routed (a write that fails is logged and nothing is routed, as a failed event always behaved), the rest follow from the records that reference them, and what nothing references is evicted at once, `gcTime` or not. A record's write puts its missing children first; a payload that links back to an ancestor defers both writes to the end of the apply. An entity built from events remembers the fields its events carried: its writes go through the store's new `mergeEntity`, which merges those fields over the record on disk, so the fields the events did not carry survive (a store without `mergeEntity` gets the whole in-memory data through `saveEntity`, as before 0.6). Once its fields are known to be the whole record (the store held no record of it when it was first written), it is written whole like any other entity, without reading the record back. An event that carries every field of the entity's class does not make it whole: another class sharing the typename, possibly not registered this session, may have written fields to the record that this class does not declare, and a merge keeps them. An event for an entity the store holds a record of (`hasEntity`) but that is not in memory refreshes that record the same way; a store that cannot say what it holds is not written, so no record nothing references is left behind. The store is asked only for a created root no live array retains, once per event: each such event costs one `hasEntity` call (a `kv.has` with `SyncQueryStore`). -- An entity that appears more than once in one payload (an author listed under `author` and again under `mentions`) is applied once: it keeps its child references and is written once. -- `_persisted` is only set once the store write succeeded, so a write that threw is retried by the next apply. `AsyncQueryStore` writers acknowledge processed entity writes (`onPersisted`), report a failed one, answer `hasEntity` from the keys they read from the delegate at start (alongside the queue, so writes do not wait for it; a queued write counts as held) and keep current, run `purgeStaleQueries` through the same serial queue as the writes (a reader with its own delegate still purges directly), ignore anything on the channel that is not one of their messages, and expose `isSettled()` and `hasQueuedDeletes()`; the client counts its own outstanding writes, so an acknowledgement of a write another instance queued does not mark a hydrated instance current. It does not skip a write while the writer has an operation queued that may delete the record the skip relies on (anything but the client's own entity writes); an entity whose own write is still queued counts as written, and an entity that one of the client's queued writes stops referencing is written again if it is referenced again before that write lands. -- `AsyncQueryStore` resizes a persisted LRU queue by copying it (growing no longer throws `RangeError`, shrinking evicts the dropped keys instead of stranding them, never the key being activated), and keeps a queue's persisted size for a message without `maxCount` from an older reader. `SyncQueryStore` gets the same resize. -- `onDelete` returns an unsubscribe and `QueryClient.destroy()` calls it, so a store no longer retains every client created against it. `SyncQueryStore` shares its delete listeners between every instance over the same backing kv. -- Reading a shared-typename entity array consumes the notifiers of the members it currently leaves out, so a reactive consumer learns when a member becomes eligible without a data change, and a consumer that only reads the array is not recomputed when a member it holds changes. -- Two classes sharing a typename: fields that are entity-free in one class but hold an entity through the other are no longer classified static for the snapshot fast path. -- A query whose cached value was rejected fetches even within `staleTime`; a query whose cached value was rejected fetches even within `staleTime`; snapshots of proxies created by a previous (hot-reloaded) module instance are walked instead of returned as-is. +- Development builds freeze snapshots, so mutating a query result (a `sort()` in render, an assignment) throws a `TypeError` where it happens. Production snapshots are not frozen; treat them as read-only. Fetchium no longer mutates objects you pass it, such as a snapshot handed back in a mutation's `effects`. +- In development, the stale-snapshot check reports through the client's `log.error` and an uncaught error (a red box on React Native) instead of throwing during a snapshot, which stopped every other consumer from updating. +- Mounting a second query whose entities are already in memory no longer leaves the first query's consumers with stale data. The in-memory data is used when it fits the new query's shape; otherwise the cached query is dropped and fetched instead of rendering with an item missing. +- Streamed events persist every entity they change, including existing entities inside a `create` payload and stored entities that are not in memory. An entity built from events is merged into its stored record (new optional `QueryStore.mergeEntity`), so the fields its events did not carry survive a restart. +- An entity that appears more than once in one response is applied and written once. +- A store write that throws is retried by the next apply. `AsyncQueryStore` reports finished writes back to the client, and both built-in stores resize their LRU queue when `cache.maxCount` changes. +- `QueryClient.destroy()` unsubscribes from the store, so a store no longer keeps every client created against it alive. +- A query whose cached value was rejected fetches even within `staleTime`. Snapshots work for entities created before a hot reload. From 8be1a63b9673cc2ea12b1bd29886713dced00e1a Mon Sep 17 00:00:00 2001 From: Adam Azbel-Bedell Date: Tue, 6 Oct 2026 13:00:49 -0400 Subject: [PATCH 09/15] chore(fetchium): rewrite this change's changesets for React developers Lead with what changes for an app using useQuery or fetchQuery, keep migration notes to a line or two, and drop internal detail. No source change. Co-Authored-By: Claude Opus 5.5 (1M context) --- .changeset/skip-write-and-snapshot-fixes.md | 13 ++----- .changeset/upgrade-behavior-changes.md | 41 +++++++-------------- 2 files changed, 18 insertions(+), 36 deletions(-) diff --git a/.changeset/skip-write-and-snapshot-fixes.md b/.changeset/skip-write-and-snapshot-fixes.md index 7a268ec..6eb00ef 100644 --- a/.changeset/skip-write-and-snapshot-fixes.md +++ b/.changeset/skip-write-and-snapshot-fixes.md @@ -2,13 +2,8 @@ 'fetchium': patch --- -Fixes for snapshots, unchanged-data skipping and streamed-event writes. +Fixed stale data in components when a second query loads entities that are already in memory, and streamed updates that were lost on restart. -- Development builds freeze snapshots, so mutating a query result (a `sort()` in render, an assignment) throws a `TypeError` where it happens. Production snapshots are not frozen; treat them as read-only. Fetchium no longer mutates objects you pass it, such as a snapshot handed back in a mutation's `effects`. -- In development, the stale-snapshot check reports through the client's `log.error` and an uncaught error (a red box on React Native) instead of throwing during a snapshot, which stopped every other consumer from updating. -- Mounting a second query whose entities are already in memory no longer leaves the first query's consumers with stale data. The in-memory data is used when it fits the new query's shape; otherwise the cached query is dropped and fetched instead of rendering with an item missing. -- Streamed events persist every entity they change, including existing entities inside a `create` payload and stored entities that are not in memory. An entity built from events is merged into its stored record (new optional `QueryStore.mergeEntity`), so the fields its events did not carry survive a restart. -- An entity that appears more than once in one response is applied and written once. -- A store write that throws is retried by the next apply. `AsyncQueryStore` reports finished writes back to the client, and both built-in stores resize their LRU queue when `cache.maxCount` changes. -- `QueryClient.destroy()` unsubscribes from the store, so a store no longer keeps every client created against it alive. -- A query whose cached value was rejected fetches even within `staleTime`. Snapshots work for entities created before a hot reload. +- Development builds freeze query results, so mutating one (such as calling `sort()` on a list in render) throws a `TypeError`. Copy the data first, and treat production results as read-only too. +- In development, the stale-data check logs an error instead of throwing and blocking other components' updates. +- Custom stores can add the new optional `QueryStore.mergeEntity`, so an entity built from streamed updates doesn't overwrite stored fields those updates didn't include. Without it, writes work as before. diff --git a/.changeset/upgrade-behavior-changes.md b/.changeset/upgrade-behavior-changes.md index 0ca94b7..626d5b9 100644 --- a/.changeset/upgrade-behavior-changes.md +++ b/.changeset/upgrade-behavior-changes.md @@ -2,31 +2,18 @@ 'fetchium': major --- -Upgrade notes: behavior an app can observe without using any new API. Each change has its own entry. +Breaking changes. Most apps need no code changes, but tests that depend on loading frames, render counts or timers may. -Rendering - -- With `SyncQueryStore`, a cached query renders its data on the first render, with no loading frame. Tests that wait for a loading state or count renders may need updating. -- A refetch, poll or streamed update with identical data does not re-render. `isPending` and `isFetching` still change. -- `t.record` and `t.result` fields apply updates; a record field used to keep its old value. -- List snapshots keep each row's identity when the list is inserted into or re-sorted. -- A nested `t.object` or `t.record` value, or a live collection's value, is a new object after an update changes it, and keeps its identity otherwise. -- Development builds freeze query results, so mutating one throws a `TypeError`. Treat production results as read-only too. - -Fetching - -- The first fetch, and a refetch without `debounce`, start on a microtask instead of after `setTimeout(0)`. -- On resume, a subscription is restored before the request is sent, so a socket that replays a cached value no longer overwrites the fresh response. -- Changing a Signal param mid-fetch aborts that fetch and fetches the new params. A late response for the old params is dropped. -- A query unmounted mid-fetch refetches as soon as it mounts again, without showing the `AbortError`. -- `retry: {}` and `retry: { retryDelay }` retry like `retry: true`. A `shouldRetry` hook can stop retrying permanent errors. -- `shouldRetry`, `reactivationGraceMs`, `reactivationStaggerMs`, `activity` and `pollResumeJitterMs` are reserved `QueryClientConfig` names. Other keys still reach queries as `this.context`. -- `QueryClient.destroy()` aborts every fetch in flight (whoever awaits one gets an `AbortError`), and nothing is written to the store after it. - -Cache and store - -- A cached entry that can't be applied is treated as a miss and fetched, even within `staleTime`. -- A query whose result is not an entity caches each params key separately. Caches from earlier versions are corrected as each key is fetched again. -- Streamed events persist the entities they change. -- `QueryStore` gains optional `mergeEntity`, `onDelete`, `onPersisted`, `isSettled`, `hasQueuedDeletes` and `hasEntity`. A custom store without them writes as before. A store write that throws is retried. -- `AsyncQueryStore` honors `cache.maxCount`, and both built-in stores resize their LRU queue when it changes. +- **Results are frozen in development.** Mutating query data throws a `TypeError`; copy it first (`[...data.items].sort()`). +- **Cached data shows on the first render** with `SyncQueryStore`. Update tests that wait for a loading state or count renders. +- **Fetches start sooner.** The first fetch and refetches without `debounce` start on a microtask, not after `setTimeout(0)`; update tests that advance timers to see them. +- **Identical data doesn't re-render.** A refetch, poll or streamed update with the same data skips the render; read `isFetching` instead of counting renders. +- **Nested values change identity only when they change.** `t.object`, `t.record` and live-collection values are new objects after a change, so memoized children re-render; drop workarounds that forced it. +- **`t.record` and `t.result` fields apply updates** instead of keeping their old value. +- **Changing a Signal param mid-fetch aborts that fetch** and fetches the new params; the old response is dropped. +- **`QueryClient.destroy()` aborts fetches in flight.** Code that awaits a query during teardown gets an `AbortError`; catch it. +- **Awaiting a query can wait for a remount.** If the last component using a query unmounts before its first data arrives, `await fetchQuery(...)` waits for the next mount, `gcTime` collection or `destroy()` instead of rejecting with an `AbortError`. +- **New reserved `QueryClientConfig` names:** `shouldRetry`, `reactivationGraceMs`, `reactivationStaggerMs`, `activity` and `pollResumeJitterMs` are read as options; rename custom context values that use them. +- **`retry: {}` and `retry: { retryDelay }` now retry** like `retry: true`; use `retry: false` for no retries. +- **`AsyncQueryStore` honors `cache.maxCount`.** Check the value if you set one. +- **Custom `QueryStore`s** get new optional methods (`mergeEntity`, `onDelete`, `onPersisted`, `isSettled`, `hasQueuedDeletes`, `hasEntity`); without them nothing changes. From 281e4281b204ca74577688af840240f491aebe02 Mon Sep 17 00:00:00 2001 From: Adam Azbel-Bedell Date: Tue, 6 Oct 2026 19:49:35 -0400 Subject: [PATCH 10/15] test(fetchium): cover the async write order of a created child and its parent With an AsyncQueryStore writer, the parent's record must not reference a child created by an event until the child's record has been written. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/__tests__/event-persistence.test.ts | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/packages/fetchium/src/__tests__/event-persistence.test.ts b/packages/fetchium/src/__tests__/event-persistence.test.ts index cc771de..1cd529b 100644 --- a/packages/fetchium/src/__tests__/event-persistence.test.ts +++ b/packages/fetchium/src/__tests__/event-persistence.test.ts @@ -1541,6 +1541,65 @@ describe('AsyncQueryStore writer', () => { expect(first._pendingWrites).toBe(0); expect(first._persisted).toBe(true); }); + + it('does not write a parent referencing a created child before the child is written', async () => { + class Item extends Entity { + __typename = t.typename('Item'); + id = t.id; + listId = t.string; + title = t.string; + } + class List extends Entity { + __typename = t.typename('List'); + id = t.id; + items = t.liveArray(Item, { constraints: { listId: (this as any).id } }); + } + class GetList extends RESTQuery { + path = '/list'; + result = { list: t.entity(List) }; + } + + // Holds the write of one record until released. + class GatedDelegate extends AsyncDelegate { + gatedKey: string | undefined; + gate: Promise | undefined; + override async setString(key: string, value: string) { + if (key === this.gatedKey) await this.gate; + return super.setString(key, value); + } + } + + const delegate = new GatedDelegate(); + const store = writerStore(delegate); + const mockFetch = createMockFetch(); + mockFetch.get('/list', { list: { __typename: 'List', id: 'l1', items: [] } }); + const client = makeClient(store, mockFetch); + await holdQuery(client, () => fetchQuery(GetList)); + await drain(store); + + const listKey = K('List', 'l1'); + const itemKey = K('Item', 'i1'); + let release = () => {}; + delegate.gate = new Promise(resolve => (release = resolve)); + delegate.gatedKey = valueKeyFor(itemKey); + + client.applyMutationEvent({ + type: 'create', + typename: 'Item', + data: { __typename: 'Item', id: 'i1', listId: 'l1', title: 'new' }, + }); + await sleep(50); + + // The child's write is held, so a cache read now must not find the parent pointing at it. + expect(asyncDoc(delegate, itemKey)).toBeUndefined(); + expect(asyncDoc(delegate, listKey)).toMatchObject({ items: [] }); + expect(Array.from((delegate.kv[refIdsKeyFor(listKey)] as Uint32Array | undefined) ?? [])).toEqual([]); + + release(); + await drain(store); + expect(asyncDoc(delegate, itemKey)).toMatchObject({ title: 'new' }); + expect(asyncDoc(delegate, listKey)).toMatchObject({ items: [{ __entityRef: itemKey }] }); + }); }); // ====================================================== From 3213594bc0340ad988f742ffb63a48c39921ce00 Mon Sep 17 00:00:00 2001 From: Adam Azbel-Bedell Date: Tue, 6 Oct 2026 23:48:55 -0400 Subject: [PATCH 11/15] chore(fetchium): tighten comments Co-Authored-By: Claude Opus 5.5 (1M context) --- packages/fetchium/src/EntityInstance.ts | 135 ++++++++---------- packages/fetchium/src/QueryClient.ts | 51 +++---- packages/fetchium/src/QueryResult.ts | 12 +- .../src/__tests__/entity-snapshot.test.ts | 6 +- .../src/__tests__/event-persistence.test.ts | 16 +-- .../skip-write-and-snapshot-fixes.test.ts | 10 +- packages/fetchium/src/applyEntities.ts | 56 ++++---- packages/fetchium/src/errors.ts | 8 +- packages/fetchium/src/parseEntities.ts | 61 ++++---- packages/fetchium/src/query-types.ts | 41 +++--- packages/fetchium/src/stores/async.ts | 53 +++---- packages/fetchium/src/stores/shared.ts | 4 +- packages/fetchium/src/stores/sync.ts | 7 +- 13 files changed, 200 insertions(+), 260 deletions(-) diff --git a/packages/fetchium/src/EntityInstance.ts b/packages/fetchium/src/EntityInstance.ts index 84b012c..4206853 100644 --- a/packages/fetchium/src/EntityInstance.ts +++ b/packages/fetchium/src/EntityInstance.ts @@ -160,11 +160,9 @@ function snapshotCameFrom(snapshotObj: unknown, sourceId: number): boolean { } /** - * Dev builds hand out frozen snapshots. A consumer that mutates one (a - * `sort()` in render, an assignment) then fails at its own call site with a - * TypeError and a stack trace, instead of the fast path quietly carrying the - * mutation forward. Production keeps the objects mutable: freezing costs a - * call per new container and the contract is documented instead. + * Dev builds freeze snapshots, so a consumer that mutates one (a `sort()` in + * render) throws at its own call site instead of the fast path silently + * carrying the mutation forward. Production skips the per-container cost. */ function freezeInDev(obj: T): T { return IS_DEV ? Object.freeze(obj) : obj; @@ -235,10 +233,9 @@ function snapshotRawValue(value: unknown, prev: unknown, snap: SnapshotFn): unkn } /** - * Structural equality of two snapshot values (plain objects, arrays, and - * primitives compared with `Object.is`). Used by the dev guard to tell a real - * drift from a value that merely lost reference identity, e.g. a `Set` a - * format parser returned, which Signalium's `snapshotSet` rebuilds each walk. + * Structural equality of two snapshot values. The dev guard uses it to tell + * real drift from a value that only lost identity, such as a format parser's + * `Set`, which Signalium's `snapshotSet` rebuilds each walk. */ function sameSnapshotValue(a: unknown, b: unknown): boolean { if (Object.is(a, b)) return true; @@ -250,9 +247,8 @@ function sameSnapshotValue(a: unknown, b: unknown): boolean { } if (a instanceof Set || b instanceof Set) { if (!(a instanceof Set) || !(b instanceof Set) || a.size !== b.size) return false; - // A rebuilt Set (Signalium's `snapshotSet` copies one every walk) keeps - // the insertion order, so pair members by position first and only search - // for a member that fails to line up. This keeps the common case linear. + // A rebuilt Set keeps insertion order, so pair members by position and + // only search for one that fails to line up. The common case stays linear. const bItems = [...b]; // Indices of `b` already paired, tracked from the first misalignment on. let matched: Set | undefined; @@ -313,10 +309,10 @@ function walkFields( } /** - * The pre-fast-path walk: `Object.keys` on the proxy and a read per key. Kept - * for a proxy this module instance did not create — after a hot reload - * re-evaluates this file, proxies made by the previous instance are still - * live but absent from the new `snapshotSources`. + * Full walk: `Object.keys` on the proxy and a read per key. For proxies this + * module instance did not create: after a hot reload re-evaluates this file, + * the previous instance's proxies are still live but absent from + * `snapshotSources`. */ function snapshotProxyByWalking(current: object, prev: unknown, snap: SnapshotFn): unknown { const obj = current as Record; @@ -337,11 +333,9 @@ function snapshotProxyByWalking(current: object, prev: unknown, snap: SnapshotFn const snapshotEntity = (current: object, prev: unknown, snap: SnapshotFn): unknown => { const source = snapshotSources.get(current); if (source === undefined) { - // A proxy from another instance of this module (hot reload re-evaluated - // this file and its registries, but the proxies are still live): walk it - // the old way rather than handing the proxy back, which would stop React - // from ever re-rendering on its changes. Every entity proxy answers - // `toJSON`; a bare `Entity` instance does not. + // A proxy from a previous instance of this module (hot reload): walk it. + // Handing the proxy back would stop React re-rendering on its changes. + // Every entity proxy answers `toJSON`; a bare `Entity` instance does not. if (PROXY_ID.has(current) || typeof (current as { toJSON?: unknown }).toJSON === 'function') { return snapshotProxyByWalking(current, prev, snap); } @@ -382,19 +376,17 @@ const snapshotEntity = (current: object, prev: unknown, snap: SnapshotFn): unkno } if (IS_DEV) { - // Dev builds re-read what the fast path skipped. A field that drifted is - // served from the re-read (the pre-fast-path behaviour) and reported, - // rather than thrown: a throw from inside a snapshot lands in Signalium's - // watcher flush, which has no catch, and stalls every consumer on the page. + // Re-read what the fast path skipped. A drifted field is served from the + // re-read and reported, not thrown: a throw inside a snapshot lands in + // Signalium's watcher flush, which has no catch, and stalls every consumer. const drifted = verifyStaticFields(source, keys, before, snap); if (drifted !== undefined) { if (result === undefined) result = { ...before }; for (const key of Object.keys(drifted)) result[key] = drifted[key]; } - // A consumer that added or deleted keys on the snapshot it was handed: - // rebuild with the shape's keys, in the shape's order, as a walk would. - // A frozen snapshot (every one this module hands out in dev) cannot - // have been altered, so only an unfrozen one pays for the key list. + // A consumer added or deleted keys on its snapshot: rebuild with the + // shape's keys in shape order. Dev snapshots are frozen and cannot have + // been altered, so only an unfrozen one pays for the key list. if (!Object.isFrozen(before) && !sameKeyList(Object.keys(before), keys.enumerable)) { const source_ = result ?? before; const rebuilt: Record = {}; @@ -435,11 +427,9 @@ const reportedDrift = new Set(); export type SnapshotDriftHandler = (error: Error, queryClient: QueryClient) => void; /** - * Dev-only default: log through the client's logger, then raise the error - * outside the snapshot so it surfaces as an uncaught error (a red box in - * React Native, "Uncaught Error" in a browser, a failed run under vitest) - * without throwing from inside Signalium's watcher flush, which has no catch - * and would stop every consumer on the page from updating. + * Dev-only default: log, then rethrow in a microtask so the error surfaces as + * uncaught (a red box in React Native, a failed vitest run) without throwing + * inside Signalium's watcher flush. */ const defaultDriftHandler: SnapshotDriftHandler = (error, queryClient) => { queryClient.getContext().log?.error?.(error.message, error); @@ -476,10 +466,9 @@ function reportDrift(source: EntitySnapshotSource, key: string, what: string): v /** * Checks the fast path's assumptions: `data` only changes through a `notify()` * that bumps `version`, and every field outside `keys.dynamic` is a pure - * function of `data`. Re-reads the skipped fields and returns the ones whose - * value differs structurally from the cached snapshot (identity alone is not - * enough: a `Set` from a format parser is rebuilt on every walk), or - * `undefined` when nothing drifted. Skipped fields only — re-reading a dynamic + * function of `data`. Re-reads the skipped fields and returns those that + * differ structurally from the cached snapshot, or `undefined` if none do. + * Skipped fields only — re-reading a dynamic * field is not identity-stable, since snapshotting a child updates state it * pairs on. * @@ -536,9 +525,9 @@ export class EntityInstance { /** Set while an apply is still reifying this instance's fields; its data is not yet a record. */ _applying: boolean = false; /** - * The data came from streamed events only (the entity was created by one), - * so it may lack fields a record of it on disk has: a write merges over - * that record. Cleared by the first full payload applied to the entity. + * The data came only from streamed events, so it may lack fields the stored + * record has: a write merges over that record. Cleared by the first full + * payload applied to the entity. */ _partial: boolean = false; /** With `_partial`: the fields the events carried, i.e. the ones a write merges. */ @@ -631,10 +620,9 @@ export class EntityInstance { /** * This entity's next write drops its reference to `child`, which can delete - * the child's record. A store that processes writes later (`onPersisted`) - * does that after anything an apply decides meanwhile, so the child is - * written again rather than trusted. A synchronous store reports the - * deletion (`onDelete`) as it happens. + * the child's record. An acknowledging store does that after anything an + * apply decides meanwhile, so the child is written again rather than + * trusted. A synchronous store reports the deletion through `onDelete`. */ private writeDropsRef(child: EntityInstance): void { if (this._queryClient.storeAcksWrites) child.recordDropped(); @@ -691,11 +679,10 @@ export class EntityInstance { if (this._saving) return; this._saving = true; try { - // A record's references must point at records that exist: a child the - // store holds no record of (never written, dropped, or a failed write) - // is written first. While an apply is still reifying a child (a payload - // that links back to an ancestor), neither it nor this entity can be - // written yet: both are written, in that order, once the apply is done. + // A record's references must point at existing records, so a child the + // store has no record of is written first. A child an apply is still + // reifying (a payload that links back to an ancestor) can't be written + // yet: both are written, child first, once the apply is done. const refs = this.entityRefs; if (refs !== undefined) { let deferred = false; @@ -709,8 +696,7 @@ export class EntityInstance { try { child.save(); } catch (e) { - // This entity's record (if any) still lacks the reference: the - // next apply must write it again. + // This entity's record still lacks the reference. this.markUnwritten(); throw e; } @@ -724,26 +710,24 @@ export class EntityInstance { } } this._deferredWrite = false; - // Mark after the store call: a store that throws leaves the record stale - // or missing, so the flag is cleared and the next apply writes again. A - // store that acknowledges writes later (`onPersisted`) marks it itself - // once the write has been processed. An entity built from events hands - // the store only the fields they carried, to merge over its record. - // Counted before the call: a store may acknowledge synchronously. - // With no record to merge over, the fields this entity holds are the - // whole record, so it stops being partial and later writes skip the - // merge's read of the stored record. + // With no record to merge over, the fields held are the whole record: + // stop being partial so later writes skip the merge's read. if (this._partial && !this._recorded && (storeHolds ?? client.store.hasEntity?.(this.key)) === false) { this._partial = false; this._partialKeys = undefined; } + // Counted before the call: a store may acknowledge synchronously. if (client.storeAcksWrites) this._pendingWrites++; + // A partial entity hands the store only the fields its events carried. try { client.entityMap.save(this, this._partial ? this._partialKeys : undefined); } catch (e) { this.markUnwritten(); throw e; } + // Marked after the call, so a store that throws leaves it unset and the + // next apply writes again. An acknowledging store marks it in + // `acknowledgeWrite`. if (!client.storeAcksWrites) this._persisted = this._recorded = true; } finally { this._saving = false; @@ -756,12 +740,10 @@ export class EntityInstance { } /** - * The store processed a write of this entity's record. Only a write this - * instance dispatched counts: an acknowledgement can also belong to a write - * queued by a previous instance of the same entity (one that was collected - * and re-hydrated from the store while the write was in flight), and that - * write carried a value this instance never had. The record is current - * once every dispatched write has landed. + * The store processed a write of this entity's record. Only writes this + * instance dispatched count: an acknowledgement can belong to a write from a + * previous instance (collected and re-hydrated while it was in flight), + * carrying data this instance never had. */ acknowledgeWrite(): void { if (this._pendingWrites === 0) return; @@ -1013,12 +995,10 @@ function bindMethod( } /** - * The members a narrowed array leaves out are dependencies of the read: one - * of them gaining the def's fields notifies itself, and that is what must - * recompute the consumer. Members already in the array are read through the - * array by any consumer that cares about their fields, and a member never - * loses eligibility, so consuming them here would only add recomputes (and - * cost a lookup per member on every read of the field). + * Members a narrowed array leaves out are dependencies of the read: one that + * gains the def's fields notifies itself, which must recompute the consumer. + * Included members are read through the array and never lose eligibility, so + * consuming them would only add recomputes. */ function consumeExcluded(excluded: EntityInstance[]): void { for (let i = 0; i < excluded.length; i++) excluded[i].consume(); @@ -1114,11 +1094,10 @@ function createProxy( function entityKeys(): EntityKeys { // The client replaces its key cache when another class registers for - // this typename, and the new split must reach snapshots already in - // flight. A cache's entries are never overwritten, so the base can only - // have changed when the cache object did: an identity check, rather than - // a WeakMap lookup on every snapshot of every entity. Both extras slots - // matter too: these lists decide what a snapshot walks and re-reads. + // this typename, and live proxies must pick up the new split. Entries are + // never overwritten, so checking the cache's identity is enough and avoids + // a WeakMap lookup per snapshot. Both extras slots matter too: these lists + // decide what a snapshot walks and re-reads. const baseCache = queryClient.shapeKeyCache; const methodsNow = instance._extraMethods; const gettersNow = instance._extraGetters; diff --git a/packages/fetchium/src/QueryClient.ts b/packages/fetchium/src/QueryClient.ts index c699fa9..a00a98a 100644 --- a/packages/fetchium/src/QueryClient.ts +++ b/packages/fetchium/src/QueryClient.ts @@ -823,10 +823,10 @@ export class QueryClient { return; } - // Entities the event creates are not written by the apply; entities it - // merely updates are. A created entity is written once a written record - // references it, or below, once the root is known to be retained. Only an - // event whose root is new can create entities that need collecting here. + // The apply writes entities the event updates, not ones it creates. A + // created entity is written once a written record references it, or below + // once the root is known to be retained. Only an event with a new root can + // create entities that need collecting here. const created = existing === undefined ? new Set() : undefined; try { const warn = this.context.log?.warn ?? (() => {}); @@ -858,19 +858,13 @@ export class QueryClient { return; } - // The root the event created is written, with the entities it created - // under it, when a live array is about to retain it (the array's owner - // is written with the reference) or when the store already holds a record - // of it: a query that was collected from memory can still hold this - // entity in its cache, and the event is that record's only chance to stay - // fresh. That write merges the event's fields over the record, since an - // event is a partial update and the fields it leaves out are not known - // here. The write comes before the routing: if it fails, nothing is - // routed, the failure is logged, and memory stays as it was, as a failed - // event always did. A store that cannot say what it holds (no - // `hasEntity`, or not yet) is not written, so it never accumulates - // records nothing references. Anything else the event created was only - // ever referenced by an unwritten root and stays unwritten. + // The created root (and what it created) is written when a live array is + // about to retain it, or when the store already holds a record of it: a + // collected query's cache can still hold this entity, and the event is + // that record's only chance to stay fresh. An event is partial, so the + // write merges over the record. It comes before routing so that a failed + // write routes nothing. A store that can't say what it holds (no + // `hasEntity`) is not written, so it never accumulates unreferenced records. let matched = false; let retains = false; this.routeEvent( @@ -909,10 +903,9 @@ export class QueryClient { } /** - * Evicts the root an event created unless an entity that existed before - * the event now references it (its owner's data holds the proxy, so it is - * live). References from entities the same event created, the root itself - * included, do not count: they were only ever reachable through the root. + * Evicts the root an event created unless an entity that existed before the + * event now references it. References from entities the same event created + * (the root included) don't count: they were only reachable through the root. */ private evictUnlessAdopted(root: EntityInstance, created: Set): void { let createdHolders = 0; @@ -921,10 +914,9 @@ export class QueryClient { } /** - * Evicts the root an event created, and with it every entity the event - * created that nothing references any more: they were never written, so - * lingering until a `gcTime` runs out would let a later event write them - * as records nothing references. + * Evicts the root an event created, along with every entity the event + * created that is no longer referenced. They were never written, and + * lingering until `gcTime` would let a later event write them as orphans. */ private evictCreated(root: EntityInstance, created: Set): void { root.evict(); @@ -954,8 +946,8 @@ export class QueryClient { /** * Writes the deferred entities in the order they were deferred: children - * before their parents. A write that fails leaves the rest unwritten, and - * marked so that the next apply writes them. + * before their parents. If a write fails, the rest are marked so the next + * apply writes them. */ /** @internal */ flushDeferredWrites(): void { @@ -978,9 +970,8 @@ export class QueryClient { } /** - * An apply that failed leaves nothing to write: what it deferred may be half - * reified. The records that were waiting are stale, so the next apply - * writes them. + * After a failed apply, what it deferred may be half reified, so nothing is + * written. Those records are stale and the next apply writes them. */ /** @internal */ discardDeferredWrites(): void { diff --git a/packages/fetchium/src/QueryResult.ts b/packages/fetchium/src/QueryResult.ts index e6879c9..5b184e1 100644 --- a/packages/fetchium/src/QueryResult.ts +++ b/packages/fetchium/src/QueryResult.ts @@ -278,13 +278,11 @@ export class QueryInstance { __hasNext: () => this.hasNext, __isFetchingNext: () => this._fetchNextPromise !== undefined, }; - // The entity's key set just changed, but its consumers are not - // notified: this query's own consumers get the proxy through the relay - // and snapshot it afresh, and an existing consumer of a shared entity - // (a list showing the entity this query returns) has no use for another - // query's extras. Notifying would re-run every one of them, and give - // their snapshots a new identity, for data that did not change. They - // see the extras on their next recompute, as before. + // The entity's key set changed, but consumers are deliberately not + // notified. This query's consumers get the proxy through the relay and + // snapshot it fresh, and other consumers of a shared entity have no use + // for this query's extras. Notifying would re-run them all and change + // their snapshot identity for unchanged data. } return this.rootEntity.getProxy(def.statics.shape as unknown as EntityDef) as QueryResult; diff --git a/packages/fetchium/src/__tests__/entity-snapshot.test.ts b/packages/fetchium/src/__tests__/entity-snapshot.test.ts index e1cbf4f..fe00aa4 100644 --- a/packages/fetchium/src/__tests__/entity-snapshot.test.ts +++ b/packages/fetchium/src/__tests__/entity-snapshot.test.ts @@ -687,10 +687,8 @@ describe('Entity Snapshots', () => { try { client.applyMutationEvent({ type: 'update', typename: 'Token', data: { id: 'tok-1', price: 5 } }); - // The guard must not throw from inside the snapshot: that escapes - // Signalium's watcher flush and stalls every consumer on the page. It - // raises the error out of band (by default as an uncaught error, through - // the client's logger) and hands out what a full walk would have produced. + // The guard reports out of band instead of throwing inside the snapshot, + // and hands out what a full walk would have produced. const snap = read() as unknown as { tokens: { symbol: string }[] }; expect(snap.tokens[0].symbol).toBe('MUTATED'); expect(raised).toHaveLength(1); diff --git a/packages/fetchium/src/__tests__/event-persistence.test.ts b/packages/fetchium/src/__tests__/event-persistence.test.ts index 1cd529b..7e2cfb0 100644 --- a/packages/fetchium/src/__tests__/event-persistence.test.ts +++ b/packages/fetchium/src/__tests__/event-persistence.test.ts @@ -327,9 +327,8 @@ describe('records a streamed event writes', () => { expect(warn).toHaveBeenCalledWith('Failed to apply mutation event', expect.any(Error)); // Memory and the store agree: the comment is in neither, and no record - // references it. (The author written before the failing write stays - // behind as an unreferenced record, as any write that fails mid-walk has - // always left its predecessors.) + // references it. The author written before the failing write stays behind + // unreferenced, as with any write that fails mid-walk. expect((postQ.value as any).post.comments).toEqual([]); expect((postQ.value as any).post.commentCount).toBe(0); expect(client.entityMap.getEntity(K('Comment', 'c9'))).toBeUndefined(); @@ -1018,9 +1017,8 @@ describe('records a streamed event writes when released entities linger (gcTime) }); await sleep(5); - // Nothing references what the event created; had the author lingered - // until its gcTime, a later event could have written it as a record - // nothing references. + // Nothing references what the event created. Had the author lingered + // until its gcTime, a later event could have written it as an orphan. expect(client.entityMap.getEntity(K('Comment', 'c9'))).toBeUndefined(); expect(client.entityMap.getEntity(K('User', 'u7'))).toBeUndefined(); expect(client.entityMap.getEntity(K('Badge', 'b7'))).toBeUndefined(); @@ -1427,8 +1425,8 @@ describe('AsyncQueryStore writer', () => { data: { __typename: 'User', id: 'u1', name: 'Alicia', karma: 10 }, }); await sleep(5); - // Written whole (the badge the event did not carry is gone, as before - // 0.6), never as a record holding only the event's fields. + // Written whole, dropping the badge the event did not carry, never as a + // record holding only the event's fields. expect(getDoc(kv, K('User', 'u1'))).toMatchObject({ __typename: 'User', id: 'u1', name: 'Alicia', karma: 10 }); expect(getDoc(kv, K('User', 'u1'))!.badge).toBeUndefined(); }); @@ -1803,7 +1801,7 @@ describe('AsyncQueryStore writer: write skipping while writes are queued', () => await drain(store); await scanned(store); - // The deck's write drops its reference to c1, the only one on disk; before + // The deck's write drops its reference to c1, the only one on disk. Before // it is processed, the same card is created again with identical data. client.applyMutationEvent({ type: 'delete', typename: 'Card', data: 'c1' }); client.applyMutationEvent({ type: 'create', typename: 'Card', data: card }); diff --git a/packages/fetchium/src/__tests__/skip-write-and-snapshot-fixes.test.ts b/packages/fetchium/src/__tests__/skip-write-and-snapshot-fixes.test.ts index 81b098f..dc2219e 100644 --- a/packages/fetchium/src/__tests__/skip-write-and-snapshot-fixes.test.ts +++ b/packages/fetchium/src/__tests__/skip-write-and-snapshot-fixes.test.ts @@ -14,10 +14,8 @@ import { valueKeyFor, refIdsKeyFor, queueKeyFor, DEFAULT_MAX_COUNT } from '../st import { createMockFetch, setupTestClient, testWithClient, sleep } from './utils.js'; /** - * Regression coverage for the follow-ups to the snapshot fast path (#59), - * unchanged-apply skipping (#60), single-write events (#61) and store - * deletion reporting (#64). Each case failed on the tree with those PRs and - * passed on the tree before them. + * Invariants the snapshot fast path (#59), unchanged-apply skipping (#60), + * single-write events (#61) and store deletion reporting (#64) must keep. */ function getDoc(kv: MemoryPersistentStore, key: number): Record | undefined { @@ -1097,8 +1095,8 @@ describe('snapshot fast path follow-ups', () => { const computesBefore = wrapped.computes(); expect('__refetch' in before.profile).toBe(false); - // The entity-rooted query applies identical data: its own consumer sees - // the extras, the wrapped query's consumer is not re-run for them. + // The entity-rooted query applies identical data. Its own consumer sees + // the extras, but the wrapped query's consumer is not re-run for them. const root = snapshotHarness(client, () => fetchQuery(GetRoot)); await root.query; await sleep(5); diff --git a/packages/fetchium/src/applyEntities.ts b/packages/fetchium/src/applyEntities.ts index f9722a5..86c01d8 100644 --- a/packages/fetchium/src/applyEntities.ts +++ b/packages/fetchium/src/applyEntities.ts @@ -29,12 +29,10 @@ export interface ApplyResult { /** * Whether an apply writes to the store. `true` writes every entity that needs - * it; `false` writes nothing (a cache hydration); `'existing'` writes entities - * that already existed in memory and not the ones this apply creates: those - * are written when a written record references them (`EntityInstance.save` - * writes a record's missing children first), or by the caller once it knows - * the root is retained (a streamed event whose root may end up routed - * nowhere). + * it, and `false` writes nothing (cache hydration). `'existing'` writes only + * entities already in memory. Ones this apply creates are written when a + * written record references them (`EntityInstance.save` writes missing + * children first), or by the caller once it knows the root is retained. */ export type PersistMode = boolean | 'existing'; @@ -90,8 +88,8 @@ function reifyAndApply( return applyEntity(entity, seen, queryClient, persist, entityRefs, appendMode, created); } - // Only a slot whose value changes is written: values the parser did not - // declare (and so did not copy) may belong to the caller, and be frozen. + // Assign only slots whose value changes: values the parser did not declare + // (and so did not copy) may belong to the caller and be frozen. if (Array.isArray(value)) { for (let i = 0; i < value.length; i++) { const item = value[i]; @@ -137,11 +135,10 @@ function applyEntity( const { key, data, shape: entityShape, rawKeys, eventKeys, fillsPartial } = entity; const shapeFields = entityShape.shape; - // The same entity can appear more than once in one payload (an author under - // `author` and again under `mentions`). The parser hands every slot the one - // parsed object, so the walk arrives here once per slot; applying it again - // would find its children already reified, count none of them and release - // every child ref. Count this slot's reference and hand out the proxy. + // An entity can appear in several slots of one payload (under `author` and + // `mentions`), all sharing one parsed object. Applying it again would find + // its children already reified, count none and release every child ref, so + // only count this slot's reference. const applied = queryClient.entityMap.getEntity(key); if (applied !== undefined && applied.parseId === queryClient.currentParseId) { parentEntityRefs.set(applied, (parentEntityRefs.get(applied) ?? 0) + 1); @@ -188,10 +185,9 @@ function applyEntity( entityInstance._partial = false; entityInstance._partialKeys = undefined; } else if (entityInstance._partial && rawKeys !== undefined) { - // Stays partial even once the events have carried every field of - // this class: another class sharing the typename may have written - // fields to the record that this one does not declare, and only a - // merge keeps them. + // Stays partial even after events carry every field of this class: + // another class sharing the typename may have written fields this one + // doesn't declare, and only a merge keeps them. for (const k of rawKeys) entityInstance._partialKeys!.add(k); } } else { @@ -205,11 +201,9 @@ function applyEntity( // event did (the parser fills the literal in). if (entityShape.typenameField !== undefined) entityInstance._partialKeys.add(entityShape.typenameField); if (typeof entityShape.idField === 'string') entityInstance._partialKeys.add(entityShape.idField); - // An event that carried every field of this class is still partial: - // the record may hold fields another class sharing the typename - // declares (one not registered this session, too), so it is merged - // into, not replaced. With no record to merge into, the first write - // makes it whole (see `EntityInstance.save()`). + // Partial even if the event carried every field of this class, for the + // same reason (the other class may not be registered this session). + // With no record to merge into, the first write makes it whole. } else if (persist === false) { // Hydrated from the store: its record exists. entityInstance._recorded = true; @@ -240,9 +234,9 @@ function applyEntity( const newRefs = childRefs.size > 0 ? childRefs : undefined; const refsChanged = !sameRefs(entityInstance.entityRefs, newRefs); // An entity hydrated from the store was never written, so there is no write - // to skip. One whose own write is still queued has its current data on the - // way. A skip is also only safe while the store has no deletion queued: it - // could drop the record this apply trusts. + // to skip. One with a write still queued has its current data on the way. + // Skipping is only safe while the store has no deletion queued, which could + // drop the record this apply trusts. const needsPersist = changed || refsChanged || @@ -283,8 +277,8 @@ function mergeFields( const raw = data[fieldKey]; if (shouldReify(raw)) { - // Written only when it changes: an object the parser kept as is (a - // fetch result's) may belong to the adapter, and be frozen. + // Assign only on change: an object the parser kept as is (from a fetch + // result) may belong to the adapter and be frozen. const reified = reifyAndApply(raw, seen, queryClient, persist, childRefs, appendMode, created); if (reified !== raw) data[fieldKey] = reified; } @@ -329,8 +323,8 @@ function mergeFields( !(propShape.shape instanceof Set) ? (propShape.shape as Record) : undefined; - // The parser keeps a fetch result's unchanged objects as they are, and - // an adapter may hand out frozen ones: merge into a copy of those. + // An unchanged object kept from a fetch result may be frozen by the + // adapter: merge into a copy. const target = Object.isFrozen(oldVal) ? { ...oldVal } : oldVal; if (nestedShape !== undefined) { if ( @@ -468,8 +462,8 @@ function initFields( const raw = data[fieldKey]; if (shouldReify(raw)) { - // Written only when it changes: an object the parser kept as is (a - // fetch result's) may belong to the adapter, and be frozen. + // Assign only on change: an object the parser kept as is (from a fetch + // result) may belong to the adapter and be frozen. const reified = reifyAndApply(raw, seen, queryClient, persist, childRefs, appendMode, created); if (reified !== raw) data[fieldKey] = reified; } diff --git a/packages/fetchium/src/errors.ts b/packages/fetchium/src/errors.ts index 65d6cf0..b13059a 100644 --- a/packages/fetchium/src/errors.ts +++ b/packages/fetchium/src/errors.ts @@ -171,10 +171,10 @@ export function typeError(path: string, expectedType: InternalObjectFieldTypeDef } /** - * A cached query cannot be hydrated as-is: an entity it holds is already in - * memory and its data does not satisfy the query's shape. Unlike a bad array - * item, this drops the whole cached query (it is refetched) rather than the - * item, since a list missing an item would resolve without a word. + * A cached query can't be hydrated as-is: an entity it holds is already in + * memory and doesn't satisfy the query's shape. Unlike a bad array item, this + * drops the whole cached query (which is refetched), since a list silently + * missing an item is worse. */ export class CachedEntityMismatchError extends Error { constructor(message: string) { diff --git a/packages/fetchium/src/parseEntities.ts b/packages/fetchium/src/parseEntities.ts index f71e08e..15a6348 100644 --- a/packages/fetchium/src/parseEntities.ts +++ b/packages/fetchium/src/parseEntities.ts @@ -75,11 +75,10 @@ export class ParseContext { /** Verdicts of the trust-memory check, per instance and def, for one parse. */ trusted: TrustMemo | undefined = undefined; /** - * Whether a nested object or record is always parsed into a copy. On for - * input the client does not own and may see again (an event payload, a - * mutation's effects, a consumer's snapshot fed back). Off for a fetch - * result or a cached record, which is copied only where a parsed value - * differs from what it holds: unchanged input costs no allocation. + * Whether nested objects and records always parse into a copy. On for input + * the client doesn't own and may see again (an event payload, mutation + * effects, a snapshot fed back). Off for a fetch result or cached record, + * which is copied only where a parsed value differs from it. */ copyInput: boolean = true; @@ -396,8 +395,9 @@ function parseUnionData( function parseArrayData(array: unknown[], itemShape: ComplexTypeDef, ctx: ParseContext, path: string): unknown[] { // Without `ctx.copyInput`, an array whose items all parse to themselves is - // returned as is; the copy starts at the first item that differs or fails. - // An empty one is always a new array: a live array grows the one it holds. + // returned as is, and the copy starts at the first item that differs or + // fails. An empty array is always new, since a live array grows the one it + // holds. let result: unknown[] | undefined = ctx.copyInput || array.length === 0 ? [] : undefined; for (let i = 0; i < array.length; i++) { @@ -421,10 +421,9 @@ function parseArrayData(array: unknown[], itemShape: ComplexTypeDef, ctx: ParseC } // Neither walker writes into its input. With `ctx.copyInput` they parse into -// a shallow copy, as `parseArrayData` builds a new array; without it the copy -// is made only once a parsed value differs from the input's, and input whose -// values all parse to themselves is returned as is. Either way the result -// keeps every key the input had, as before. +// a shallow copy. Without it, the copy is made only once a parsed value +// differs, and input whose values all parse to themselves is returned as is. +// The result keeps every key the input had. function parseRecordData( record: Record, @@ -511,8 +510,8 @@ function parseEntityData( if (preloadedEntities !== undefined) { const existing = queryClient.entityMap.getEntity(key); - // The fields of an entity built from streamed events that the record on - // disk has and the events did not carry. + // For an entity built from events: the fields they carried. The record + // supplies the rest. let fillKeys: Set | undefined; const preloaded = preloadedEntities.get(key); @@ -522,13 +521,11 @@ function parseEntityData( fillKeys = existing._partialKeys; obj = preloaded; } else if (existing !== undefined) { - // The entity is already live in memory, so the cache has nothing newer - // to offer: the in-memory data is what every consumer already sees. - // Re-parsing that data would run already-parsed values (formatted - // values, parse results, child proxies) through the parser again, in - // place, and corrupt them. An empty partial entry merges nothing and - // hands the query the existing proxy. The data must still satisfy this - // query's shape, or the cached query is not usable as-is. + // A live entity is newer than the cache. Re-parsing its data would run + // already-parsed values (formatted values, parse results, child + // proxies) through the parser again and corrupt them. An empty partial + // entry merges nothing and hands the query the existing proxy. The data + // must still satisfy this query's shape, or the cached query is unusable. ctx.trusted ??= new Map(); if (!dataSatisfiesDef(existing.data, entityShape as unknown as ValidatorDef, queryClient, ctx.trusted)) { throw new CachedEntityMismatchError( @@ -617,11 +614,10 @@ function parseEntityData( /** * Whether already-parsed entity data can be handed to a query declaring `def` - * without being parsed again. Unlike `entitySatisfiesShape` (a top-level - * presence check used for narrowing) this descends into nested objects, arrays, - * records, unions and child entities, because a cached query that resolves with - * a required nested field missing is worse than one that refetches. Where a - * value cannot be judged it is accepted, matching the parser's leniency. + * without re-parsing. Unlike `entitySatisfiesShape` (a top-level presence + * check for narrowing), this descends into nested values and child entities: + * a cached query missing a required nested field is worse than a refetch. + * Values that can't be judged are accepted, matching the parser's leniency. */ export function dataSatisfiesDef( data: Record, @@ -746,10 +742,10 @@ function valueSatisfiesDef(value: unknown, fieldDef: unknown, queryClient: Query } /** - * An entity field holds a proxy. One whose instance has left memory is served - * as it is (the data it still holds), as a re-parse would have handed it out - * too; a live one must satisfy the def, with verdicts shared across the parse - * and a cycle counted as satisfied. + * An entity field holds a proxy. One whose instance has left memory is + * accepted, since a re-parse would hand out its data too. A live one must + * satisfy the def. Verdicts are shared across the parse, and a cycle counts as + * satisfied. */ function entitySatisfies( value: object, @@ -772,10 +768,9 @@ function entitySatisfies( } /** - * Members of an entity array whose typename has more than one registered - * class are narrowed on read to the ones satisfying the field's def, so a - * member that does not satisfy it is not a mismatch: it is left out either - * way. + * When the item typename has several registered classes, the array is + * narrowed on read to members that satisfy the def, so a member that doesn't + * is not a mismatch. */ function arraySatisfies(value: unknown[], itemDef: unknown, queryClient: QueryClient, visiting: TrustMemo): boolean { let narrowed = false; diff --git a/packages/fetchium/src/query-types.ts b/packages/fetchium/src/query-types.ts index 5e309d2..e6518e4 100644 --- a/packages/fetchium/src/query-types.ts +++ b/packages/fetchium/src/query-types.ts @@ -112,12 +112,11 @@ export interface QueryStore { saveEntity(entityKey: number, value: unknown, refIds?: Set): void; /** - * Writes the fields streamed events supplied for an entity built from them: - * a store that holds a record for the key merges `fields` over it (and - * derives the record's references from the merged value); one that does not - * writes `fields` as the record. A store without this method gets the - * entity's whole in-memory data through `saveEntity` instead, which drops - * from its record the fields the events did not carry. + * Writes the fields streamed events supplied for an entity built from them. + * If the store holds a record for the key, it merges `fields` over it and + * derives references from the merged value. Otherwise `fields` becomes the + * record. Without this method the client calls `saveEntity` with the whole + * in-memory data, which drops the fields the events did not carry. */ mergeEntity?(entityKey: number, fields: unknown, refIds?: Set): void; @@ -129,16 +128,16 @@ export interface QueryStore { /** * Called with the key of every record the store drops on its own (eviction, - * cascade, purge). May return an unsubscribe function; the client calls it - * from `destroy()`. A store without this hook is written on every apply. + * cascade, purge). May return an unsubscribe function, which the client + * calls from `destroy()`. A store without this hook is written on every + * apply. */ onDelete?(listener: (key: number) => void): void | (() => void); /** - * For stores that process writes asynchronously: called with the key of - * every entity record once its write has actually been applied. While a - * store offers this, the client treats an entity as persisted only after - * the acknowledgement, so a write the store dropped is retried. + * For stores that process writes asynchronously: called with each entity + * key once its write has been applied. The client then treats an entity as + * persisted only after this acknowledgement, so a dropped write is retried. */ onPersisted?(listener: (key: number) => void): void | (() => void); @@ -152,20 +151,18 @@ export interface QueryStore { /** * For stores that process writes asynchronously, a narrower `isSettled`: - * whether an operation that could delete a record is still queued. This - * client's own entity writes do not count; for those that drop a - * reference, the client itself writes the referenced entity again. When - * present, the client uses it instead of `isSettled`, so a burst of entity - * writes does not turn write skipping off for every other entity. + * whether an operation that could delete a record is still queued. The + * client's own entity writes don't count, since it rewrites any entity whose + * reference they drop. When present it replaces `isSettled`, so a burst of + * entity writes doesn't disable write skipping for every other entity. */ hasQueuedDeletes?(): boolean; /** - * Whether the store currently holds a record for this entity key, or - * `undefined` while it cannot tell yet (a store that has not finished - * reading what it holds). A streamed event for an entity that is not in - * memory refreshes the record only when this answers `true`; a store - * without this hook is not written for such events. + * Whether the store holds a record for this entity key, or `undefined` if + * it can't tell yet (it hasn't finished reading what it holds). A streamed + * event for an entity not in memory refreshes the record only when this + * returns `true`. Without this hook, such events are not written. */ hasEntity?(key: number): boolean | undefined; } diff --git a/packages/fetchium/src/stores/async.ts b/packages/fetchium/src/stores/async.ts index 9cf6023..ab257ea 100644 --- a/packages/fetchium/src/stores/async.ts +++ b/packages/fetchium/src/stores/async.ts @@ -75,9 +75,9 @@ export interface AsyncQueryStoreConfig { } /** - * Writer-internal work that runs through the same serial queue as the wire - * messages, so it is ordered with respect to the writes and deletions around - * it. Never sent over the wire, and distinguishable from anything that is. + * Writer-internal work run through the same serial queue as wire messages, so + * it is ordered with the writes and deletions around it. Never sent over the + * wire. */ class InternalWork { constructor( @@ -124,21 +124,19 @@ export class AsyncQueryStore implements QueryStore { private deleteListeners: Array<(key: number) => void> = []; private persistedListeners: Array<(key: number) => void> = []; private processing = false; - // Queued work that can delete a record: everything except the entity - // writes of this writer's own client, which tracks the references those - // drop (see `hasQueuedDeletes`). + // Queued work that can delete a record: everything except this writer's + // own client's entity writes, whose dropped references the client tracks. private queuedDeletes = 0; private readonly ownEntityWrites = new WeakSet(); - // The ids whose value the delegate holds (or is about to: a queued write - // counts), mirrored by the writer so that `hasEntity` can answer - // synchronously. Read from the delegate once at start, without holding up - // the queue; until that read lands, writes and deletions are noted aside - // and folded in. + // Ids whose value the delegate holds (queued writes included), mirrored so + // `hasEntity` can answer synchronously. Read from the delegate once at + // start without blocking the queue. Writes and deletions before that read + // lands are noted aside and folded in. private heldKeys: Set | undefined; private heldSinceScan: Set | undefined = new Set(); private droppedSinceScan: Set | undefined = new Set(); - // Only the writer sees deletions and completed writes; a reader offers no - // hook, so the client writes every apply. + // Only the writer sees deletions and completed writes. A reader offers no + // hooks, so the client writes every apply. onDelete?: (listener: (key: number) => void) => () => void; onPersisted?: (listener: (key: number) => void) => () => void; hasEntity?: (key: number) => boolean | undefined; @@ -257,9 +255,8 @@ export class AsyncQueryStore implements QueryStore { } } catch (error) { console.error('Error processing message:', error); - // A failed entity write leaves the record missing or stale, which to - // the client is the same as the store having dropped it: it must - // write the entity again on its next apply. + // A failed entity write leaves the record missing or stale. Report it + // as dropped so the client writes the entity again on its next apply. if (!(msg instanceof InternalWork) && msg.type === StoreMessageType.SaveEntity) { this.noteDropped(msg.entityKey); for (let i = 0; i < this.deleteListeners.length; i++) this.deleteListeners[i](msg.entityKey); @@ -274,8 +271,8 @@ export class AsyncQueryStore implements QueryStore { /** * Reads which records the delegate holds, alongside the queue rather than * ahead of it. Writes and deletions processed meanwhile are folded in. - * A read that fails is retried; if it keeps failing, nothing is assumed - * held (an event for an entity not in memory is then not written). + * A failed read is retried. If it keeps failing nothing is assumed held, so + * an event for an entity not in memory is not written. */ private async scanHeldKeys(): Promise { const held = new Set(); @@ -312,8 +309,6 @@ export class AsyncQueryStore implements QueryStore { private async processMessage(msg: StoreMessage): Promise { switch (msg.type) { - // `maxCount` is absent from messages sent by readers built before it - // was added; the writer then keeps whatever size the queue has. case StoreMessageType.SaveQuery: await this.writerSaveQuery( msg.queryDefId, @@ -515,12 +510,11 @@ export class AsyncQueryStore implements QueryStore { queue = new Uint32Array(maxCount ?? DEFAULT_MAX_COUNT); await this.delegate!.setBuffer(queueKey, queue); } else if (maxCount !== undefined && queue.length !== maxCount) { - // `maxCount` changed since the queue was written (until 0.6 the writer - // always used the default, so every older install has a 50-slot queue). - // A view over the old buffer cannot grow past it, and a shorter view - // would silently strand the keys it drops: copy what fits and evict - // the rest properly. The key being activated is about to move to the - // front, so it is kept whichever slot it sits in. + // `maxCount` changed since the queue was written (writers before 0.6 + // always used the default of 50). A view over the old buffer can't + // grow, and a shorter one would strand the keys it drops: copy what + // fits and evict the rest. The key being activated moves to the + // front, so it is kept wherever it sits. const resized = new Uint32Array(maxCount); resized.set(queue.subarray(0, Math.min(queue.length, maxCount))); for (let i = maxCount; i < queue.length; i++) { @@ -562,10 +556,9 @@ export class AsyncQueryStore implements QueryStore { } /** - * Drops every query definition's cached queries once the definition has not - * been used for its `cacheTime`. Runs through the writer's serial queue, so - * its deletions are ordered with the writes and acknowledgements around - * them: a record it drops cannot be reported as written afterwards. + * Drops every query definition's cached queries once the definition has + * gone unused for its `cacheTime`. Runs through the writer's serial queue, + * so a record it drops cannot be reported as written afterwards. */ purgeStaleQueries(): Promise { if (!this.delegate) return Promise.resolve(); diff --git a/packages/fetchium/src/stores/shared.ts b/packages/fetchium/src/stores/shared.ts index e2ed01c..3d2dcb2 100644 --- a/packages/fetchium/src/stores/shared.ts +++ b/packages/fetchium/src/stores/shared.ts @@ -32,8 +32,8 @@ export function mergeStoredRecord( } catch { return undefined; } - // A record that is not an object is not one to merge into; the caller - // writes the fields as the record, which heals it. + // A non-object record can't be merged into. The caller writes the fields as + // the record instead, which heals it. if (typeof record !== 'object' || record === null || Array.isArray(record)) return undefined; const value = { ...(record as Record), diff --git a/packages/fetchium/src/stores/sync.ts b/packages/fetchium/src/stores/sync.ts index 1f96303..7d3cf71 100644 --- a/packages/fetchium/src/stores/sync.ts +++ b/packages/fetchium/src/stores/sync.ts @@ -198,10 +198,9 @@ export class SyncQueryStore implements QueryStore { this.kv.setBuffer(queueKeyFor(queryDefId), queue); } else if (queue.length !== maxCount) { // `maxCount` changed since the queue was written. A view over the old - // buffer cannot grow past it, and a shorter view would silently strand - // the keys it drops: copy what fits and evict the rest properly. The - // key being activated is about to move to the front, so it is kept - // whichever slot it sits in. + // buffer can't grow, and a shorter one would strand the keys it drops: + // copy what fits and evict the rest. The key being activated moves to + // the front, so it is kept wherever it sits. const resized = new Uint32Array(maxCount); resized.set(queue.subarray(0, Math.min(queue.length, maxCount))); for (let i = maxCount; i < queue.length; i++) { From f412cb9eb5a13f2616bc9761aa4d840e1121c0b5 Mon Sep 17 00:00:00 2001 From: Adam Azbel-Bedell Date: Tue, 6 Oct 2026 23:51:24 -0400 Subject: [PATCH 12/15] chore(fetchium): drop PR numbers from a test header Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/__tests__/skip-write-and-snapshot-fixes.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/fetchium/src/__tests__/skip-write-and-snapshot-fixes.test.ts b/packages/fetchium/src/__tests__/skip-write-and-snapshot-fixes.test.ts index dc2219e..e8b2d7d 100644 --- a/packages/fetchium/src/__tests__/skip-write-and-snapshot-fixes.test.ts +++ b/packages/fetchium/src/__tests__/skip-write-and-snapshot-fixes.test.ts @@ -14,8 +14,8 @@ import { valueKeyFor, refIdsKeyFor, queueKeyFor, DEFAULT_MAX_COUNT } from '../st import { createMockFetch, setupTestClient, testWithClient, sleep } from './utils.js'; /** - * Invariants the snapshot fast path (#59), unchanged-apply skipping (#60), - * single-write events (#61) and store deletion reporting (#64) must keep. + * Invariants the snapshot fast path, unchanged-apply skipping, single-write + * events and store deletion reporting must keep. */ function getDoc(kv: MemoryPersistentStore, key: number): Record | undefined { From 99982a577506a4546abdeab7fd6ae0a7b62138e5 Mon Sep 17 00:00:00 2001 From: Adam Azbel-Bedell Date: Tue, 6 Oct 2026 23:59:28 -0400 Subject: [PATCH 13/15] fix(fetchium): keep store bookkeeping right after a failed write or an unsubscribe - A failed entity write asks the delegate whether the previous record is still there, instead of assuming it is gone, so hasEntity() keeps answering true and later events for the entity still reach it. - Store listeners are notified over a copy, so a listener that unsubscribes while being notified no longer makes the next one miss the event. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../skip-write-and-snapshot-fixes.test.ts | 35 +++++++++++++++++++ .../src/__tests__/store-on-delete.test.ts | 29 +++++++++++++++ packages/fetchium/src/stores/async.ts | 21 ++++++++--- packages/fetchium/src/stores/sync.ts | 3 +- 4 files changed, 83 insertions(+), 5 deletions(-) diff --git a/packages/fetchium/src/__tests__/skip-write-and-snapshot-fixes.test.ts b/packages/fetchium/src/__tests__/skip-write-and-snapshot-fixes.test.ts index e8b2d7d..1f17054 100644 --- a/packages/fetchium/src/__tests__/skip-write-and-snapshot-fixes.test.ts +++ b/packages/fetchium/src/__tests__/skip-write-and-snapshot-fixes.test.ts @@ -856,6 +856,41 @@ describe('AsyncQueryStore writer used in-process', () => { } }); + it('a rejected SaveEntity over an existing record still treats the record as held', async () => { + const delegate = new AsyncDelegate(); + const store = writerStore(delegate); + const mockFetch = createMockFetch(); + mockFetch.get('/user/profile/1', { user: { __typename: 'User', id: '1', name: 'Alice' } }); + const { client } = makeClient(store, mockFetch); + clients.push(client); + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {}); + const name = () => JSON.parse(delegate.kv[valueKeyFor(userKey('1'))] as string).name; + const update = (newName: string) => + client.applyMutationEvent({ + type: 'update', + typename: 'User', + data: { __typename: 'User', id: '1', name: newName }, + }); + try { + await holdQuery(client, () => fetchQuery(GetProfile, { id: '1' })); + await drain(store); + + delegate.failNextSetString = valueKeyFor(userKey('1')); + update('Bob'); + await drain(store); + expect(name()).toBe('Alice'); + expect(store.hasEntity!(userKey('1'))).toBe(true); + + // With the entity out of memory, the next update must still reach the record. + client.entityMap.getEntity(userKey('1'))!.evict(); + update('Carol'); + await drain(store); + expect(name()).toBe('Carol'); + } finally { + consoleError.mockRestore(); + } + }); + it('resizes a persisted LRU queue in both directions and defaults a missing maxCount', async () => { const delegate = new AsyncDelegate(); const store = writerStore(delegate); diff --git a/packages/fetchium/src/__tests__/store-on-delete.test.ts b/packages/fetchium/src/__tests__/store-on-delete.test.ts index b8e44ec..fad9cf7 100644 --- a/packages/fetchium/src/__tests__/store-on-delete.test.ts +++ b/packages/fetchium/src/__tests__/store-on-delete.test.ts @@ -112,6 +112,35 @@ describe('QueryStore.onDelete', () => { await sleep(20); expect(deleted).toEqual([q1]); }); + + it('SyncQueryStore still notifies the next listener when one unsubscribes during a deletion', () => { + const store = new SyncQueryStore(new MemoryPersistentStore()); + const deleted: number[] = []; + const unsubscribe = store.onDelete(() => unsubscribe()); + store.onDelete(key => deleted.push(key)); + + const q1 = hashValue(['GET:/u', { id: '1' }]); + store.saveQuery(queryDef('GET:/u', 10), q1, { __entityRef: 0 }, Date.now(), new Set()); + store.deleteQuery(q1); + expect(deleted).toEqual([q1]); + }); + + it('AsyncQueryStore writer still notifies the next listener when one unsubscribes during a deletion', async () => { + const writer = new AsyncQueryStore({ + isWriter: true, + delegate: new MockAsyncPersistentStore(), + connect: (_handle: (msg: StoreMessage) => void) => ({ sendMessage: () => {} }), + }); + const deleted: number[] = []; + const unsubscribe = writer.onDelete!(() => unsubscribe()); + writer.onDelete!(key => deleted.push(key)); + + const q1 = hashValue(['GET:/v', { id: '1' }]); + writer.saveQuery(queryDef('GET:/v', 10), q1, { __entityRef: 0 }, Date.now(), new Set()); + writer.deleteQuery(q1); + await sleep(20); + expect(deleted).toEqual([q1]); + }); }); describe('stores without onDelete', () => { diff --git a/packages/fetchium/src/stores/async.ts b/packages/fetchium/src/stores/async.ts index ab257ea..4db29d7 100644 --- a/packages/fetchium/src/stores/async.ts +++ b/packages/fetchium/src/stores/async.ts @@ -101,6 +101,7 @@ function isStoreMessage(msg: unknown): msg is StoreMessage { ); } +/** Callers notify over a copy, since a listener may unsubscribe while being notified. */ function subscribe(listeners: Array<(key: number) => void>, listener: (key: number) => void): () => void { listeners.push(listener); return () => { @@ -225,6 +226,18 @@ export class AsyncQueryStore implements QueryStore { } } + /** The previous record may have survived a failed write, so ask the delegate. */ + private async recheckHeld(id: number): Promise { + let held = false; + try { + held = await this.delegate!.has(valueKeyFor(id)); + } catch { + // Unknown: treat it as missing. + } + if (held) this.noteHeld(id); + else this.noteDropped(id); + } + private startQueueProcessor(): void { this.queueProcessorPromise = this.processQueue(); } @@ -258,8 +271,8 @@ export class AsyncQueryStore implements QueryStore { // A failed entity write leaves the record missing or stale. Report it // as dropped so the client writes the entity again on its next apply. if (!(msg instanceof InternalWork) && msg.type === StoreMessageType.SaveEntity) { - this.noteDropped(msg.entityKey); - for (let i = 0; i < this.deleteListeners.length; i++) this.deleteListeners[i](msg.entityKey); + await this.recheckHeld(msg.entityKey); + for (const listener of this.deleteListeners.slice()) listener(msg.entityKey); } } finally { this.processing = false; @@ -322,7 +335,7 @@ export class AsyncQueryStore implements QueryStore { break; case StoreMessageType.SaveEntity: await this.writerSaveEntity(msg.entityKey, msg.value, msg.refIds, msg.merge === true); - for (let i = 0; i < this.persistedListeners.length; i++) this.persistedListeners[i](msg.entityKey); + for (const listener of this.persistedListeners.slice()) listener(msg.entityKey); break; case StoreMessageType.ActivateQuery: await this.writerActivateQuery(msg.queryDefId, msg.queryKey, msg.cacheTime, msg.maxCount); @@ -656,7 +669,7 @@ export class AsyncQueryStore implements QueryStore { await delegate.delete(valueKeyFor(id)); this.noteDropped(id); await delegate.delete(refCountKeyFor(id)); - for (let i = 0; i < this.deleteListeners.length; i++) this.deleteListeners[i](id); + for (const listener of this.deleteListeners.slice()) listener(id); const refIds = await delegate.getBuffer(refIdsKey); await delegate.delete(refIdsKey); // Clean up the refIds key diff --git a/packages/fetchium/src/stores/sync.ts b/packages/fetchium/src/stores/sync.ts index 7d3cf71..b9edf49 100644 --- a/packages/fetchium/src/stores/sync.ts +++ b/packages/fetchium/src/stores/sync.ts @@ -326,7 +326,8 @@ export class SyncQueryStore implements QueryStore { kv.delete(valueKeyFor(id)); kv.delete(refCountKeyFor(id)); - for (let i = 0; i < this.deleteListeners.length; i++) this.deleteListeners[i](id); + // A copy: a listener may unsubscribe while being notified. + for (const listener of this.deleteListeners.slice()) listener(id); const refIds = kv.getBuffer(refIdsKeyFor(id)); kv.delete(refIdsKeyFor(id)); // Clean up the refIds key From feeacdcfb21e92cb135a7ebfdcab7bc7811e89e6 Mon Sep 17 00:00:00 2001 From: Adam Azbel-Bedell Date: Wed, 7 Oct 2026 00:16:22 -0400 Subject: [PATCH 14/15] chore(fetchium): cut and shorten comments Co-Authored-By: Claude Opus 5.5 (1M context) --- packages/fetchium/src/ConstraintMatcher.ts | 6 +- packages/fetchium/src/EntityInstance.ts | 144 +++++------------- packages/fetchium/src/EntityStore.ts | 7 +- packages/fetchium/src/LiveCollection.ts | 2 +- packages/fetchium/src/QueryClient.ts | 68 ++------- packages/fetchium/src/QueryResult.ts | 7 +- .../src/__tests__/entity-snapshot.test.ts | 5 +- .../src/__tests__/entity-write-counts.test.ts | 4 +- .../src/__tests__/event-persistence.test.ts | 95 +++--------- .../fetchium/src/__tests__/same-value.test.ts | 2 - .../skip-write-and-snapshot-fixes.test.ts | 70 ++------- packages/fetchium/src/applyEntities.ts | 55 ++----- packages/fetchium/src/errors.ts | 6 +- packages/fetchium/src/parseEntities.ts | 58 ++----- packages/fetchium/src/query-types.ts | 40 ++--- packages/fetchium/src/stores/async.ts | 57 ++----- packages/fetchium/src/stores/shared.ts | 8 +- packages/fetchium/src/stores/sync.ts | 17 +-- 18 files changed, 150 insertions(+), 501 deletions(-) diff --git a/packages/fetchium/src/ConstraintMatcher.ts b/packages/fetchium/src/ConstraintMatcher.ts index e5610aa..8058205 100644 --- a/packages/fetchium/src/ConstraintMatcher.ts +++ b/packages/fetchium/src/ConstraintMatcher.ts @@ -201,11 +201,7 @@ export class ConstraintMatcher { } } - /** - * With `dryRun`, reports through `onMatch` what the event would do (whether - * a binding matches, and whether one would retain the entity) without - * touching any collection. - */ + /** With `dryRun`, only reports matches through `onMatch`, touching no collection. */ routeEvent( typename: string, entityData: Record, diff --git a/packages/fetchium/src/EntityInstance.ts b/packages/fetchium/src/EntityInstance.ts index 76c598d..2339fde 100644 --- a/packages/fetchium/src/EntityInstance.ts +++ b/packages/fetchium/src/EntityInstance.ts @@ -159,11 +159,7 @@ function snapshotCameFrom(snapshotObj: unknown, sourceId: number): boolean { return snapshotOrigins.get(snapshotObj)?.sourceId === sourceId; } -/** - * Dev builds freeze snapshots, so a consumer that mutates one (a `sort()` in - * render) throws at its own call site instead of the fast path silently - * carrying the mutation forward. Production skips the per-container cost. - */ +/** Dev-only: a consumer that mutates a snapshot throws at its call site. */ function freezeInDev(obj: T): T { return IS_DEV ? Object.freeze(obj) : obj; } @@ -207,8 +203,7 @@ function snapshotRawValue(value: unknown, prev: unknown, snap: SnapshotFn): unkn } const next = snapshotRawValue(item, prevItem, snap); result[i] = next; - // `Object.is`, not `!==`: a NaN slot must not mark the array changed on - // every walk, or the snapshot never reaches a stable identity. + // `Object.is`, so a NaN slot doesn't change identity on every walk. if (!changed && !Object.is(next, prevArr![i])) changed = true; } return changed ? freezeInDev(result) : prevArr!; @@ -232,11 +227,7 @@ function snapshotRawValue(value: unknown, prev: unknown, snap: SnapshotFn): unkn return snap(value, prev); } -/** - * Structural equality of two snapshot values. The dev guard uses it to tell - * real drift from a value that only lost identity, such as a format parser's - * `Set`, which Signalium's `snapshotSet` rebuilds each walk. - */ +/** Dev guard: tells real drift from a value that only lost identity, like a rebuilt `Set`. */ function sameSnapshotValue(a: unknown, b: unknown): boolean { if (Object.is(a, b)) return true; if (typeof a !== 'object' || typeof b !== 'object' || a === null || b === null) return false; @@ -247,10 +238,8 @@ function sameSnapshotValue(a: unknown, b: unknown): boolean { } if (a instanceof Set || b instanceof Set) { if (!(a instanceof Set) || !(b instanceof Set) || a.size !== b.size) return false; - // A rebuilt Set keeps insertion order, so pair members by position and - // only search for one that fails to line up. The common case stays linear. + // Pair by position, searching only after the first misalignment. const bItems = [...b]; - // Indices of `b` already paired, tracked from the first misalignment on. let matched: Set | undefined; let i = 0; for (const item of a) { @@ -308,12 +297,7 @@ function walkFields( } } -/** - * Full walk: `Object.keys` on the proxy and a read per key. For proxies this - * module instance did not create: after a hot reload re-evaluates this file, - * the previous instance's proxies are still live but absent from - * `snapshotSources`. - */ +/** For proxies from a previous instance of this module (hot reload), absent from `snapshotSources`. */ function snapshotProxyByWalking(current: object, prev: unknown, snap: SnapshotFn): unknown { const obj = current as Record; const keys = Object.keys(obj); @@ -333,15 +317,11 @@ function snapshotProxyByWalking(current: object, prev: unknown, snap: SnapshotFn const snapshotEntity = (current: object, prev: unknown, snap: SnapshotFn): unknown => { const source = snapshotSources.get(current); if (source === undefined) { - // A proxy from a previous instance of this module (hot reload): walk it. - // Handing the proxy back would stop React re-rendering on its changes. - // Every entity proxy answers `toJSON`; a bare `Entity` instance does not. + // Hot-reloaded proxy. Returning it as-is would stop React re-rendering on its changes. if (PROXY_ID.has(current) || typeof (current as { toJSON?: unknown }).toJSON === 'function') { return snapshotProxyByWalking(current, prev, snap); } - // An `Entity` that never went through `createProxy` has no fields to read — - // its own properties are the shape's type defs. Hand it back untouched, the - // way Signalium treats any class it has no handler for. + // A bare `Entity` has no fields to read, only the shape's type defs. return current; } @@ -376,17 +356,13 @@ const snapshotEntity = (current: object, prev: unknown, snap: SnapshotFn): unkno } if (IS_DEV) { - // Re-read what the fast path skipped. A drifted field is served from the - // re-read and reported, not thrown: a throw inside a snapshot lands in - // Signalium's watcher flush, which has no catch, and stalls every consumer. + // Report drift, don't throw: Signalium's watcher flush has no catch. const drifted = verifyStaticFields(source, keys, before, snap); if (drifted !== undefined) { if (result === undefined) result = { ...before }; for (const key of Object.keys(drifted)) result[key] = drifted[key]; } - // A consumer added or deleted keys on its snapshot: rebuild with the - // shape's keys in shape order. Dev snapshots are frozen and cannot have - // been altered, so only an unfrozen one pays for the key list. + // A consumer altered the snapshot's keys. Frozen ones can't have been. if (!Object.isFrozen(before) && !sameKeyList(Object.keys(before), keys.enumerable)) { const source_ = result ?? before; const rebuilt: Record = {}; @@ -421,16 +397,12 @@ function sameKeyList(a: string[], b: string[]): boolean { return true; } -/** Dev-only: (typename, field) pairs already reported, so a drift is raised once. */ +/** Dev-only: drifts already raised, keyed `typename.field`. */ const reportedDrift = new Set(); export type SnapshotDriftHandler = (error: Error, queryClient: QueryClient) => void; -/** - * Dev-only default: log, then rethrow in a microtask so the error surfaces as - * uncaught (a red box in React Native, a failed vitest run) without throwing - * inside Signalium's watcher flush. - */ +/** Rethrows in a microtask so the error surfaces as uncaught, outside Signalium's watcher flush. */ const defaultDriftHandler: SnapshotDriftHandler = (error, queryClient) => { queryClient.getContext().log?.error?.(error.message, error); queueMicrotask(() => { @@ -440,10 +412,7 @@ const defaultDriftHandler: SnapshotDriftHandler = (error, queryClient) => { let snapshotDriftHandler: SnapshotDriftHandler = defaultDriftHandler; -/** - * Dev-only test hook: replace how a stale snapshot is raised, and forget which - * fields were already raised. `undefined` restores the default handler. - */ +/** Dev-only test hook. Also clears the raised set. `undefined` restores the default. */ export function __setSnapshotDriftHandler(handler: SnapshotDriftHandler | undefined): void { snapshotDriftHandler = handler ?? defaultDriftHandler; reportedDrift.clear(); @@ -466,11 +435,8 @@ function reportDrift(source: EntitySnapshotSource, key: string, what: string): v /** * Checks the fast path's assumptions: `data` only changes through a `notify()` * that bumps `version`, and every field outside `keys.dynamic` is a pure - * function of `data`. Re-reads the skipped fields and returns those that - * differ structurally from the cached snapshot, or `undefined` if none do. - * Skipped fields only — re-reading a dynamic - * field is not identity-stable, since snapshotting a child updates state it - * pairs on. + * function of `data`. Returns the static fields that drifted, if any. + * Dynamic fields are skipped, since re-reading them is not identity-stable. * * The re-read cancels the fast path's savings, so the speedup only shows in * production builds. @@ -520,21 +486,17 @@ export class EntityInstance { refCount: number = 0; /** Whether this instance has been written to the store. */ _persisted: boolean = false; - /** Writes handed to a store that acknowledges them, not yet acknowledged. */ + /** Writes sent to an acknowledging store, not yet acknowledged. */ _pendingWrites: number = 0; - /** Set while an apply is still reifying this instance's fields; its data is not yet a record. */ + /** An apply is still reifying this instance's fields. */ _applying: boolean = false; - /** - * The data came only from streamed events, so it may lack fields the stored - * record has: a write merges over that record. Cleared by the first full - * payload applied to the entity. - */ + /** Data came only from events, so a write merges over the stored record. */ _partial: boolean = false; - /** With `_partial`: the fields the events carried, i.e. the ones a write merges. */ + /** With `_partial`: the fields a write merges. */ _partialKeys: Set | undefined = undefined; - /** A write requested while a child was still being reified; performed when that apply is done. */ + /** A write waiting for a child's apply to finish. */ _deferredWrite: boolean = false; - /** The store is known to hold a record of this entity (hydrated from it, or written to it). */ + /** The store is known to hold a record of this entity. */ _recorded: boolean = false; private _saving: boolean = false; entityRefs: Map | undefined; @@ -619,10 +581,8 @@ export class EntityInstance { } /** - * This entity's next write drops its reference to `child`, which can delete - * the child's record. An acknowledging store does that after anything an - * apply decides meanwhile, so the child is written again rather than - * trusted. A synchronous store reports the deletion through `onDelete`. + * The write may delete the child's record after the current apply, so it is + * rewritten rather than trusted. Sync stores report deletes via `onDelete`. */ private writeDropsRef(child: EntityInstance): void { if (this._queryClient.storeAcksWrites) child.recordDropped(); @@ -673,16 +633,14 @@ export class EntityInstance { return false; } - /** `storeHolds`: what the store's `hasEntity` just said about this entity, if the caller asked. */ + /** `storeHolds`: the store's `hasEntity` answer, if the caller already has it. */ save(storeHolds?: boolean): void { const client = this._queryClient; if (this._saving) return; this._saving = true; try { - // A record's references must point at existing records, so a child the - // store has no record of is written first. A child an apply is still - // reifying (a payload that links back to an ancestor) can't be written - // yet: both are written, child first, once the apply is done. + // References must point at existing records, so unrecorded children are + // written first. A child still being reified defers both writes. const refs = this.entityRefs; if (refs !== undefined) { let deferred = false; @@ -696,7 +654,6 @@ export class EntityInstance { try { child.save(); } catch (e) { - // This entity's record still lacks the reference. this.markUnwritten(); throw e; } @@ -710,24 +667,20 @@ export class EntityInstance { } } this._deferredWrite = false; - // With no record to merge over, the fields held are the whole record: - // stop being partial so later writes skip the merge's read. + // No record to merge over, so the fields held are the whole record. if (this._partial && !this._recorded && (storeHolds ?? client.store.hasEntity?.(this.key)) === false) { this._partial = false; this._partialKeys = undefined; } // Counted before the call: a store may acknowledge synchronously. if (client.storeAcksWrites) this._pendingWrites++; - // A partial entity hands the store only the fields its events carried. try { client.entityMap.save(this, this._partial ? this._partialKeys : undefined); } catch (e) { this.markUnwritten(); throw e; } - // Marked after the call, so a store that throws leaves it unset and the - // next apply writes again. An acknowledging store marks it in - // `acknowledgeWrite`. + // After the call, so a throwing store leaves it unset. if (!client.storeAcksWrites) this._persisted = this._recorded = true; } finally { this._saving = false; @@ -739,12 +692,7 @@ export class EntityInstance { if (this._partial) this._partialKeys?.add(fieldKey); } - /** - * The store processed a write of this entity's record. Only writes this - * instance dispatched count: an acknowledgement can belong to a write from a - * previous instance (collected and re-hydrated while it was in flight), - * carrying data this instance never had. - */ + /** Ignores acks for writes a previous instance of this entity dispatched. */ acknowledgeWrite(): void { if (this._pendingWrites === 0) return; if (--this._pendingWrites === 0) this._persisted = this._recorded = true; @@ -756,11 +704,7 @@ export class EntityInstance { this.markUnwritten(); } - /** - * Its record is stale: the next apply writes it even if it finds nothing - * changed. Writes still queued hold older data, so their acknowledgements - * no longer mark it persisted. - */ + /** Forces the next apply to write. Queued writes hold older data, so their acks are dropped. */ markUnwritten(): void { this._persisted = false; this._pendingWrites = 0; @@ -785,7 +729,7 @@ function sameMembers(a: unknown[], b: unknown[]): boolean { interface NarrowedArray { source: unknown[]; filtered: unknown[]; - /** Members that do not (yet) satisfy the def; the only ones whose membership can change. */ + /** Members not (yet) satisfying the def, the only ones whose membership can change. */ excluded: EntityInstance[]; parseId: number; } @@ -890,21 +834,14 @@ export interface EntityKeys { enumerableSet: Set; } -/** - * A field whose def carries no shape: a primitive, a format, `t.typename`, or - * a set of literals. Its value can only be a primitive or a format's parsed - * value, so no other class sharing the typename can put an entity under it. - */ +/** No class sharing the typename can put an entity under a shapeless field. */ function isShapelessFieldDef(def: unknown): boolean { if (typeof def === 'number' || typeof def === 'string') return true; if (def instanceof Set) return true; return def instanceof ValidatorDef && (def.shape === undefined || def.shape === null); } -/** - * Cached per client (`queryClient.shapeKeyCache`): the split depends on which - * other classes share the typename, and that set grows as queries register. - */ +/** Cached per client: the split depends on which classes share the typename. */ function shapeKeys( validatorDef: ValidatorDef, shapeFields: Record, @@ -923,9 +860,7 @@ function shapeKeys( if (!own.includes(methodKey)) own.push(methodKey); } } - // The data object is shared by every class with this typename, so a nested - // object that is entity-free in this class may hold a child entity written - // through another class's shape. Only shapeless fields stay static then. + // Another class sharing the typename may write an entity into a nested object. const typename = validatorDef.typenameValue; const shared = typename !== undefined && (queryClient.getEntityDefsForTypename(typename)?.length ?? 0) > 1; const dynamic = enumerable.filter( @@ -993,12 +928,7 @@ function bindMethod( return bound; } -/** - * Members a narrowed array leaves out are dependencies of the read: one that - * gains the def's fields notifies itself, which must recompute the consumer. - * Included members are read through the array and never lose eligibility, so - * consuming them would only add recomputes. - */ +/** An excluded member that gains the def's fields must recompute the consumer. */ function consumeExcluded(excluded: EntityInstance[]): void { for (let i = 0; i < excluded.length; i++) excluded[i].consume(); } @@ -1092,11 +1022,7 @@ function createProxy( let keys: EntityKeys | undefined; function entityKeys(): EntityKeys { - // The client replaces its key cache when another class registers for - // this typename, and live proxies must pick up the new split. Entries are - // never overwritten, so checking the cache's identity is enough and avoids - // a WeakMap lookup per snapshot. Both extras slots matter too: these lists - // decide what a snapshot walks and re-reads. + // The client replaces its key cache when a class registers for this typename. const baseCache = queryClient.shapeKeyCache; const methodsNow = instance._extraMethods; const gettersNow = instance._extraGetters; diff --git a/packages/fetchium/src/EntityStore.ts b/packages/fetchium/src/EntityStore.ts index 12b8fb8..45410a7 100644 --- a/packages/fetchium/src/EntityStore.ts +++ b/packages/fetchium/src/EntityStore.ts @@ -6,7 +6,6 @@ import { ValidatorDef } from './typeDefs.js'; export class EntityStore { private instances = new Map(); private persistEntity: (key: number, data: Record, refKeys?: Set, merge?: boolean) => void; - /** Whether the store can merge fields over a record it holds. */ mergesEntities: boolean = false; constructor( @@ -62,11 +61,7 @@ export class EntityStore { this.instances.clear(); } - /** - * With `mergeKeys`, only those fields are handed to the store, to be merged - * over the record it holds (the rest of the instance's data is what streamed - * events left unset, not what the record says). - */ + /** With `mergeKeys`, only those fields are sent, merged over the stored record. */ save(instance: EntityInstance, mergeKeys?: Set): void { let refKeys: Set | undefined; if (instance.entityRefs) { diff --git a/packages/fetchium/src/LiveCollection.ts b/packages/fetchium/src/LiveCollection.ts index 88f0c6c..84f5164 100644 --- a/packages/fetchium/src/LiveCollection.ts +++ b/packages/fetchium/src/LiveCollection.ts @@ -66,7 +66,7 @@ export interface LiveCollectionParent { addChildRef(child: EntityInstance, persist?: boolean): void; removeChildRef(child: EntityInstance, persist?: boolean): void; save(): void; - /** A live field's membership changed outside an apply (an event was routed into it). */ + /** A live field's membership changed outside an apply. */ liveFieldChanged?(fieldKey: string): void; } diff --git a/packages/fetchium/src/QueryClient.ts b/packages/fetchium/src/QueryClient.ts index e3a2b21..1a7f539 100644 --- a/packages/fetchium/src/QueryClient.ts +++ b/packages/fetchium/src/QueryClient.ts @@ -165,13 +165,9 @@ export class QueryClient { currentParseId: number = 0; /** Without `store.onDelete`, `_persisted` cannot be trusted. */ storeReportsDeletes: boolean = false; - /** With `store.onPersisted`, `_persisted` is set by the store's acknowledgement, not by `save()`. */ + /** With `store.onPersisted`, `_persisted` is set on acknowledgement, not by `save()`. */ storeAcksWrites: boolean = false; - /** - * Per-client static/dynamic key split for each entity shape (see - * `shapeKeys` in EntityInstance.ts). Replaced wholesale when a typename - * gains a second class, since the split depends on the other classes. - */ + /** Replaced when a typename gains a class, since the static/dynamic split depends on it. */ shapeKeyCache = new WeakMap, EntityKeys>(); /** See `QueryClientConfig.reactivationGraceMs`. */ @@ -246,7 +242,6 @@ export class QueryClient { this.entityMap.getEntity(key)?.recordDropped(); }); if (typeof offDelete === 'function') this.storeUnsubscribes.push(offDelete); - // A store that processes writes later says when a record is really there. this.storeAcksWrites = typeof this.store.onPersisted === 'function'; const offPersisted = this.store.onPersisted?.(key => { this.entityMap.getEntity(key)?.acknowledgeWrite(); @@ -338,11 +333,7 @@ export class QueryClient { // Typename Registry (per-client) // ====================================================== - /** - * Whether every write handed to the store has been processed, so a skipped - * write cannot be undone by a deletion the store has queued but not yet run. - * A synchronous store is always settled. - */ + /** No queued store deletion can undo a skipped write. */ storeIsSettled(): boolean { const store = this.store; if (store.hasQueuedDeletes !== undefined) return !store.hasQueuedDeletes(); @@ -362,8 +353,6 @@ export class QueryClient { existing.push(def); this.mergedDefCache.delete(typename); this.getMergedDef(typename); - // The snapshot fast path's static/dynamic split for this typename's - // classes now has to account for the new class's fields. this.shapeKeyCache = new WeakMap(); } else { this.typenameRegistry.set(typename, [def]); @@ -713,9 +702,7 @@ export class QueryClient { /** * Parse data: validates, formats, produces parsed entity data objects. * Does NOT touch the entity store. Call applyRefs() after to commit entities. - * - * `copyInput: false` (a fetch result or a cached record, handed over to the - * client) copies a nested object only where a parsed value differs from it. + * `copyInput: false` copies a nested object only where a parsed value differs. */ parseData( obj: unknown, @@ -774,7 +761,6 @@ export class QueryClient { return this.entityMap.getEntity(proxyKey!)!; } - /** `existing`: the instance the caller already looked up under `key`, if any. */ prepareEntity( key: number, obj: Record, @@ -824,10 +810,7 @@ export class QueryClient { return; } - // The apply writes entities the event updates, not ones it creates. A - // created entity is written once a written record references it, or below - // once the root is known to be retained. Only an event with a new root can - // create entities that need collecting here. + // Entities the event creates are written only once something retains them. const created = existing === undefined ? new Set() : undefined; try { const warn = this.context.log?.warn ?? (() => {}); @@ -844,7 +827,6 @@ export class QueryClient { this.context.log?.warn?.('Failed to apply mutation event', e); } if (existing === undefined) { - // Half applied: nothing built from it can stay. const createdRoot = this.entityMap.getEntity(key); if (createdRoot !== undefined) this.evictCreated(createdRoot, created!); } @@ -859,13 +841,8 @@ export class QueryClient { return; } - // The created root (and what it created) is written when a live array is - // about to retain it, or when the store already holds a record of it: a - // collected query's cache can still hold this entity, and the event is - // that record's only chance to stay fresh. An event is partial, so the - // write merges over the record. It comes before routing so that a failed - // write routes nothing. A store that can't say what it holds (no - // `hasEntity`) is not written, so it never accumulates unreferenced records. + // Write a created root if a live array will retain it or the store already + // holds it. Before routing, so a failed write routes nothing. let matched = false; let retains = false; this.routeEvent( @@ -881,8 +858,6 @@ export class QueryClient { undefined, /* dryRun */ true, ); - // The store is asked only when no live array retains the root, and only - // when there is something to write; its answer is handed to save(). let held: boolean | undefined; if (!entity._persisted && entity._pendingWrites === 0) { if (!retains) held = this.store.hasEntity?.(key); @@ -897,31 +872,20 @@ export class QueryClient { } } - // The dry run found no collection to route into: routing again would - // only recompute the constraint hashes. if (matched) this.routeEvent(typename, entity.data, key, type, eventSource); else this.evictUnlessAdopted(entity, created!); } - /** - * Evicts the root an event created unless an entity that existed before the - * event now references it. References from entities the same event created - * (the root included) don't count: they were only reachable through the root. - */ + /** References from entities the same event created don't count as adoption. */ private evictUnlessAdopted(root: EntityInstance, created: Set): void { let createdHolders = 0; for (const c of created) if (c.entityRefs?.has(root)) createdHolders++; if (root.refCount <= createdHolders) this.evictCreated(root, created); } - /** - * Evicts the root an event created, along with every entity the event - * created that is no longer referenced. They were never written, and - * lingering until `gcTime` would let a later event write them as orphans. - */ + /** Also evicts unreferenced entities the event created, so a later event can't write them as orphans. */ private evictCreated(root: EntityInstance, created: Set): void { root.evict(); - // Evicting one releases what it held; repeat until nothing more is free. let evicted = true; while (evicted) { evicted = false; @@ -934,10 +898,6 @@ export class QueryClient { } } - /** - * Entities whose write was requested while an apply was still reifying - * them; written once that apply is done. - */ private deferredWrites: Set | undefined; /** @internal */ @@ -945,11 +905,7 @@ export class QueryClient { (this.deferredWrites ??= new Set()).add(entity); } - /** - * Writes the deferred entities in the order they were deferred: children - * before their parents. If a write fails, the rest are marked so the next - * apply writes them. - */ + /** Children were deferred before parents, so insertion order is write order. */ /** @internal */ flushDeferredWrites(): void { const deferred = this.deferredWrites; @@ -970,10 +926,6 @@ export class QueryClient { } } - /** - * After a failed apply, what it deferred may be half reified, so nothing is - * written. Those records are stale and the next apply writes them. - */ /** @internal */ discardDeferredWrites(): void { const deferred = this.deferredWrites; diff --git a/packages/fetchium/src/QueryResult.ts b/packages/fetchium/src/QueryResult.ts index 50a72c0..63ac80b 100644 --- a/packages/fetchium/src/QueryResult.ts +++ b/packages/fetchium/src/QueryResult.ts @@ -276,11 +276,8 @@ export class QueryInstance { __hasNext: () => this.hasNext, __isFetchingNext: () => this._fetchNextPromise !== undefined, }; - // The entity's key set changed, but consumers are deliberately not - // notified. This query's consumers get the proxy through the relay and - // snapshot it fresh, and other consumers of a shared entity have no use - // for this query's extras. Notifying would re-run them all and change - // their snapshot identity for unchanged data. + // Deliberately not notified: this query's consumers snapshot fresh via the + // relay, and other consumers of a shared entity don't need these extras. } return this.rootEntity.getProxy(def.statics.shape as unknown as EntityDef) as QueryResult; diff --git a/packages/fetchium/src/__tests__/entity-snapshot.test.ts b/packages/fetchium/src/__tests__/entity-snapshot.test.ts index 0facbe3..50a43a4 100644 --- a/packages/fetchium/src/__tests__/entity-snapshot.test.ts +++ b/packages/fetchium/src/__tests__/entity-snapshot.test.ts @@ -687,14 +687,12 @@ describe('Entity Snapshots', () => { try { client.applyMutationEvent({ type: 'update', typename: 'Token', data: { id: 'tok-1', price: 5 } }); - // The guard reports out of band instead of throwing inside the snapshot, - // and hands out what a full walk would have produced. const snap = read() as unknown as { tokens: { symbol: string }[] }; expect(snap.tokens[0].symbol).toBe('MUTATED'); expect(raised).toHaveLength(1); expect(raised[0].message).toMatch(/stale entity snapshot: Token:tok-0 field 'symbol'/); - // Raised once per field: a second recompute stays quiet. + // Raised once per field. client.applyMutationEvent({ type: 'update', typename: 'Token', data: { id: 'tok-1', price: 6 } }); read(); expect(raised).toHaveLength(1); @@ -717,7 +715,6 @@ describe('Entity Snapshots', () => { expect(Object.isFrozen(snap)).toBe(true); expect(Object.isFrozen(snap.tokens)).toBe(true); expect(Object.isFrozen(snap.tokens[0])).toBe(true); - // The classic "sort in render" bug throws where it happens, not later. expect(() => snap.tokens.sort()).toThrow(TypeError); expect(() => { snap.tokens[0].symbol = 'X'; diff --git a/packages/fetchium/src/__tests__/entity-write-counts.test.ts b/packages/fetchium/src/__tests__/entity-write-counts.test.ts index ed9df09..df689db 100644 --- a/packages/fetchium/src/__tests__/entity-write-counts.test.ts +++ b/packages/fetchium/src/__tests__/entity-write-counts.test.ts @@ -38,7 +38,6 @@ class GetList extends RESTQuery { result = { list: t.entity(List) }; } -/** Every entity write, whether it replaced the record or merged into it. */ function spyEntityWrites(store: { saveEntity: (...args: any[]) => void; mergeEntity: (...args: any[]) => void }) { const save = vi.spyOn(store, 'saveEntity'); const merge = vi.spyOn(store, 'mergeEntity'); @@ -159,8 +158,7 @@ describe('Entity Write Counts', () => { }); const saveEntity = spyEntityWrites(store); - // The item is already current in the store from the first query, so the - // apply declines its write. The insert must not put one back. + // i-1 is already current in the store, so the apply skips its write. The insert must not add one. client.applyMutationEvent({ type: 'update', typename: 'Item', diff --git a/packages/fetchium/src/__tests__/event-persistence.test.ts b/packages/fetchium/src/__tests__/event-persistence.test.ts index 7e2cfb0..824775a 100644 --- a/packages/fetchium/src/__tests__/event-persistence.test.ts +++ b/packages/fetchium/src/__tests__/event-persistence.test.ts @@ -12,11 +12,7 @@ import { AsyncQueryStore, type AsyncPersistentStore } from '../stores/async.js'; import { valueKeyFor, refCountKeyFor, refIdsKeyFor, queueKeyFor, updatedAtKeyFor } from '../stores/shared.js'; import { createMockFetch, setupTestClient, sleep } from './utils.js'; -/** - * What a streamed event leaves in the store: which records are written, which - * are not, and whether every reference on disk points at a record that exists. - * Complements `skip-write-and-snapshot-fixes.test.ts`. - */ +// What a streamed event leaves in the store, and whether every reference on disk resolves. function getDoc(kv: MemoryPersistentStore, key: number): Record | undefined { const value = kv.getString(valueKeyFor(key)); @@ -26,10 +22,7 @@ function refIds(kv: MemoryPersistentStore, key: number): number[] { return Array.from(kv.getBuffer(refIdsKeyFor(key)) ?? []); } -/** - * Every record must be referenced (a refcount) unless it is a query, and every - * reference must point at a record that exists. - */ +/** Every non-query record must be referenced, and every reference must resolve. */ function audit(kv: MemoryPersistentStore): { orphans: number[]; dangling: string[] } { const ids = new Set(); for (const k of kv.getAllKeys()) { @@ -60,14 +53,12 @@ function holdQuery(client: QueryClient, start: () => T): T { const K = (typename: string, id: string | number) => hashValue([typename, id]); -/** The keys of every entity write, whichever store method carried it. */ function spyWrites(store: SyncQueryStore): () => number[] { const saves = vi.spyOn(store, 'saveEntity'); const merges = vi.spyOn(store, 'mergeEntity'); return () => [...saves.mock.calls.map(c => c[0]), ...merges.mock.calls.map(c => c[0])]; } -/** Makes every write of the given key throw, whichever store method carries it. */ function failWritesOf(store: SyncQueryStore, key: number, shouldFail: () => boolean = () => true): void { const save = store.saveEntity.bind(store); const merge = store.mergeEntity.bind(store); @@ -81,10 +72,6 @@ function failWritesOf(store: SyncQueryStore, key: number, shouldFail: () => bool }); } -// ====================================================== -// Which records an event writes -// ====================================================== - describe('records a streamed event writes', () => { class Badge extends Entity { __typename = t.typename('Badge'); @@ -168,7 +155,7 @@ describe('records a streamed event writes', () => { await sleep(5); const u7 = client.entityMap.getEntity(K('User', 'u7'))!; - // The comment references u7 twice (author and mentions[0]) and u7 keeps its badge. + // u7 is referenced twice (author and mentions[0]). expect(client.entityMap.getEntity(K('Comment', 'c9'))!.entityRefs!.get(u7)).toBe(2); expect([...u7.entityRefs!.keys()].map(e => e.key)).toEqual([K('Badge', 'b7')]); expect(writes().filter(k => k === K('User', 'u7'))).toHaveLength(1); @@ -209,7 +196,6 @@ describe('records a streamed event writes', () => { expect(getDoc(kv, K('Badge', 'b1'))).toMatchObject({ label: 'gold' }); expect((q.value as any).comments.map((c: any) => c.author.badge.label)).toEqual(['gold', 'gold']); - // A refetch with the same payload changes nothing. mockFetch.get('/comments', payload()); await (q.value as any).__refetch(); expect(u1.refCount).toBe(2); @@ -253,8 +239,7 @@ describe('records a streamed event writes', () => { client.entityMap.getEntity(K('User', 'u1'))!.evict(); expect(client.entityMap.getEntity(K('User', 'u1'))).toBeUndefined(); - // The payload carries the required fields (an event for an entity that is - // not in memory is parsed as a full record) but not the optional badge. + // Required fields only: an event for an entity not in memory parses as a full record. client.applyMutationEvent({ type: 'update', typename: 'User', @@ -262,14 +247,12 @@ describe('records a streamed event writes', () => { }); await sleep(5); - // The badge reference the event did not carry survives. expect(getDoc(kv, K('User', 'u1'))).toMatchObject({ name: 'Alicia', karma: 10 }); expect(refIds(kv, K('User', 'u1'))).toEqual([K('Badge', 'b1')]); expect(getDoc(kv, K('Badge', 'b1'))).toMatchObject({ label: 'gold' }); expect(kv.getNumber(refCountKeyFor(K('Badge', 'b1')))).toBe(1); expect(client.entityMap.getEntity(K('User', 'u1'))).toBeUndefined(); - // A replaced child is written and the old one released. client.applyMutationEvent({ type: 'update', typename: 'User', @@ -288,7 +271,6 @@ describe('records a streamed event writes', () => { expect(getDoc(kv, K('Badge', 'b1'))).toBeUndefined(); expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); - // The next session hydrates the merged record. client.destroy(); const warn = vi.fn(); const second = new QueryClient({ @@ -326,9 +308,7 @@ describe('records a streamed event writes', () => { await sleep(5); expect(warn).toHaveBeenCalledWith('Failed to apply mutation event', expect.any(Error)); - // Memory and the store agree: the comment is in neither, and no record - // references it. The author written before the failing write stays behind - // unreferenced, as with any write that fails mid-walk. + // The author written before the failing write stays behind unreferenced. expect((postQ.value as any).post.comments).toEqual([]); expect((postQ.value as any).post.commentCount).toBe(0); expect(client.entityMap.getEntity(K('Comment', 'c9'))).toBeUndefined(); @@ -377,7 +357,6 @@ describe('records a streamed event writes', () => { expect((postQ.value as any).post.comments).toEqual([]); expect((postQ.value as any).post.commentCount).toBe(0); - // The transport retries once the store works again. fail = false; client.applyMutationEvent(event()); await sleep(5); @@ -408,7 +387,7 @@ describe('records a streamed event writes', () => { const threadQ = holdQuery(client, () => fetchQuery(GetThread, { id: 'p1' })); await threadQ; await holdQuery(client, () => fetchQuery(GetComments)); - // The comments query is collected from memory; its record stays on disk. + // Collected from memory. Its record stays on disk. client.entityMap.getEntity(K('Comment', 'c1'))!.evict(); client.applyMutationEvent({ @@ -470,7 +449,7 @@ describe('records a streamed event writes', () => { const artQ = holdQuery(client, () => fetchQuery(GetArticle, { id: 'a1' })); await artQ; - // No live collection routes notes; the payload's article points back at the note. + // The payload's article points back at the note. client.applyMutationEvent({ type: 'create', typename: 'Note', @@ -491,7 +470,6 @@ describe('records a streamed event writes', () => { expect(getDoc(kv, K('Note', 'n9'))).toMatchObject({ body: 'hi' }); expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); - // A later event for the note reaches the article's consumer. client.applyMutationEvent({ type: 'update', typename: 'Note', data: { id: 'n9', body: 'edited' } }); expect((artQ.value as any).article.latestNote.body).toBe('edited'); expect(getDoc(kv, K('Note', 'n9'))).toMatchObject({ body: 'edited' }); @@ -628,8 +606,7 @@ describe('records a streamed event writes', () => { }); await sleep(5); - // The topic's record was not written with a reference to a record that - // does not exist; memory holds the adoption and the next write heals it. + // Memory holds the adoption and the next write heals the record. expect(audit(kv).dangling).toEqual([]); expect((topicQ.value as any).topic.replies).toEqual([]); expect((topicQ.value as any).topic.latestComment.body).toBe('hi'); @@ -741,8 +718,7 @@ describe('records a streamed event writes', () => { client.entityMap.getEntity(K('Badge', 'b1'))?.evict(); client.entityMap.getEntity(K('Icon', 'i1'))?.evict(); - // The badge under the event is a created entity with a record on disk: - // its own merge keeps the icon the event did not mention. + // A created badge with a record on disk: its merge keeps the icon. client.applyMutationEvent({ type: 'update', typename: 'Profile', @@ -809,8 +785,7 @@ describe('records a streamed event writes', () => { it('a member routed into the live array of an entity built from events reaches its record', async () => { const { client, mockFetch, kv } = getClient(); - // A second class for Post without the live fields makes them optional in - // the merged definition, so an event for a post can omit them. + // Without the live fields, a second class makes them optional in events. class PostPreview extends Entity { __typename = t.typename('Post'); id = t.id; @@ -850,11 +825,9 @@ describe('records a streamed event writes', () => { const blogQ = holdQuery(client, () => fetchQuery(GetBlog, { id: 'b1' })); await blogQ; expect(refIds(kv, K('Post', 'p9'))).toEqual([K('Comment', 'c1')]); - // The post is collected from memory; its record stays on disk. client.entityMap.getEntity(K('Post', 'p9'))!.evict(); - // An event brings the post back into memory, retained by the blog's live - // array and built from the event alone: no comments, no count. + // Rebuilt from the event alone: no comments, no count. client.applyMutationEvent({ type: 'update', typename: 'Post', @@ -866,7 +839,6 @@ describe('records a streamed event writes', () => { expect(p9._partial).toBe(true); expect(getDoc(kv, K('Post', 'p9'))).toMatchObject({ title: 'T2', comments: [{ __entityRef: K('Comment', 'c1') }] }); - // A comment routed into its (empty in memory) live array reaches the record. client.applyMutationEvent({ type: 'create', typename: 'Comment', @@ -930,7 +902,6 @@ describe('records a streamed event writes', () => { const postQ = holdQuery(client, () => fetchQuery(GetPost, { id: 'p1' })); await postQ; await holdQuery(client, () => fetchQuery(GetUser, { id: 'u7' })); - // The author's record is on disk with its badge; the author is collected from memory. client.entityMap.getEntity(K('User', 'u7'))!.evict(); client.entityMap.getEntity(K('Badge', 'b7'))?.evict(); @@ -947,7 +918,6 @@ describe('records a streamed event writes', () => { }); await sleep(5); expect((postQ.value as any).post.comments.map((c: any) => c.author.name)).toEqual(['Newbie']); - // The event did not mention the badge: the record keeps it. expect(getDoc(kv, K('User', 'u7'))).toMatchObject({ name: 'Newbie', karma: 2, @@ -958,10 +928,6 @@ describe('records a streamed event writes', () => { }); }); -// ====================================================== -// Entities with an in-memory gcTime linger after release -// ====================================================== - describe('records a streamed event writes when released entities linger (gcTime)', () => { class Badge extends Entity { static cache = { gcTime: 1 }; @@ -1017,15 +983,13 @@ describe('records a streamed event writes when released entities linger (gcTime) }); await sleep(5); - // Nothing references what the event created. Had the author lingered - // until its gcTime, a later event could have written it as an orphan. + // Had the author lingered until gcTime, a later event could write it as an orphan. expect(client.entityMap.getEntity(K('Comment', 'c9'))).toBeUndefined(); expect(client.entityMap.getEntity(K('User', 'u7'))).toBeUndefined(); expect(client.entityMap.getEntity(K('Badge', 'b7'))).toBeUndefined(); expect(writes()).toEqual([]); expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); - // A second unrouted event for the same author still writes nothing. client.applyMutationEvent({ type: 'create', typename: 'Comment', @@ -1035,7 +999,6 @@ describe('records a streamed event writes when released entities linger (gcTime) expect(writes()).toEqual([]); expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); - // A routed create with the same author writes it, with its badge. client.applyMutationEvent({ type: 'create', typename: 'Comment', @@ -1049,7 +1012,6 @@ describe('records a streamed event writes when released entities linger (gcTime) expect(refIds(kv, K('User', 'u7'))).toEqual([K('Badge', 'b7')]); expect(audit(kv)).toEqual({ orphans: [], dangling: [] }); - // GC time passes; the entities are retained and survive. await sleep(300); expect(client.entityMap.getEntity(K('User', 'u7'))).toBeDefined(); expect(client.entityMap.getEntity(K('Badge', 'b7'))).toBeDefined(); @@ -1075,9 +1037,7 @@ describe('records a streamed event writes when released entities linger (gcTime) }); }); -// ====================================================== // AsyncQueryStore writer -// ====================================================== class AsyncDelegate implements AsyncPersistentStore { kv: Record = {}; @@ -1245,7 +1205,6 @@ describe('AsyncQueryStore writer', () => { expect(asyncDoc(delegate, K('Badge', 'b1'))).toMatchObject({ label: 'gold' }); expect(client.entityMap.getEntity(K('User', 'u1'))).toBeUndefined(); - // An entity the store has never seen leaves nothing behind. client.applyMutationEvent({ type: 'update', typename: 'User', @@ -1286,7 +1245,6 @@ describe('AsyncQueryStore writer', () => { const client = makeClient(store, mockFetch); await holdQuery(client, () => fetchQuery(GetUser, { id: 'u2' })); expect(store.hasEntity!(K('User', 'u1'))).toBeUndefined(); - // What this writer wrote itself is known even now. expect(store.hasEntity!(K('User', 'u2'))).toBe(true); const event = () => ({ type: 'update' as const, @@ -1392,7 +1350,7 @@ describe('AsyncQueryStore writer', () => { it('a store without mergeEntity gets the whole in-memory data of an event-built entity', async () => { const kv = new MemoryPersistentStore(); const inner = new SyncQueryStore(kv); - // A custom store forwarding the pre-0.6 surface only. + // A custom store with only the pre-0.6 surface. const custom = { loadQuery: inner.loadQuery.bind(inner), saveQuery: inner.saveQuery.bind(inner), @@ -1425,8 +1383,7 @@ describe('AsyncQueryStore writer', () => { data: { __typename: 'User', id: 'u1', name: 'Alicia', karma: 10 }, }); await sleep(5); - // Written whole, dropping the badge the event did not carry, never as a - // record holding only the event's fields. + // Written whole, never as a record of only the event's fields. expect(getDoc(kv, K('User', 'u1'))).toMatchObject({ __typename: 'User', id: 'u1', name: 'Alicia', karma: 10 }); expect(getDoc(kv, K('User', 'u1'))!.badge).toBeUndefined(); }); @@ -1452,8 +1409,7 @@ describe('AsyncQueryStore writer', () => { expect(store.isSettled()).toBe(false); await purged; - // The write was acknowledged before the purge dropped the record, so the - // client that heard both ends up with the record marked missing. + // Acknowledged before the purge dropped it, so the record ends up marked missing. expect(log).toEqual(['ack 5', 'delete 9', 'delete 5']); expect(delegate.kv[valueKeyFor(5)]).toBeUndefined(); expect(store.hasEntity!(5)).toBe(false); @@ -1531,7 +1487,6 @@ describe('AsyncQueryStore writer', () => { first.acknowledgeWrite(); expect(first._persisted).toBe(false); - // Two writes in flight: the record is current once both have landed. first.save(); first.save(); expect(first._pendingWrites).toBe(2); @@ -1557,7 +1512,6 @@ describe('AsyncQueryStore writer', () => { result = { list: t.entity(List) }; } - // Holds the write of one record until released. class GatedDelegate extends AsyncDelegate { gatedKey: string | undefined; gate: Promise | undefined; @@ -1600,10 +1554,6 @@ describe('AsyncQueryStore writer', () => { }); }); -// ====================================================== -// Store work per streamed event -// ====================================================== - describe('store work per streamed event', () => { class Author extends Entity { __typename = t.typename('Author'); @@ -1684,7 +1634,6 @@ describe('store work per streamed event', () => { await sleep(5); expect(merges.mock.calls.map(c => c[0])).toEqual([K('Note', 'n1')]); - // The field the event did not carry survives. expect(getDoc(kv, K('Note', 'n1'))).toMatchObject({ body: 'b', pinned: true }); }); }); @@ -1696,7 +1645,7 @@ describe('AsyncQueryStore writer: write skipping while writes are queued', () => }); class Badge extends Entity { - // Lingers in memory once released, so a later apply finds the same instance. + // Lingers once released, so a later apply finds the same instance. static cache = { gcTime: 60 }; __typename = t.typename('Badge'); id = t.id; @@ -1769,7 +1718,6 @@ describe('AsyncQueryStore writer: write skipping while writes are queued', () => it('an entity a queued write stops referencing is written again when it is referenced again', async () => { class Card extends Entity { - // Lingers in memory once released, so the next event finds the same instance. static cache = { gcTime: 60 }; __typename = t.typename('Card'); id = t.id; @@ -1801,8 +1749,7 @@ describe('AsyncQueryStore writer: write skipping while writes are queued', () => await drain(store); await scanned(store); - // The deck's write drops its reference to c1, the only one on disk. Before - // it is processed, the same card is created again with identical data. + // Recreated with identical data before the deck's write, which drops c1, lands. client.applyMutationEvent({ type: 'delete', typename: 'Card', data: 'c1' }); client.applyMutationEvent({ type: 'create', typename: 'Card', data: card }); await drain(store); @@ -1813,18 +1760,13 @@ describe('AsyncQueryStore writer: write skipping while writes are queued', () => }); }); -// ====================================================== -// A typename two classes share, across sessions -// ====================================================== - describe('an event for a typename whose other class is not registered this session', () => { class Tag extends Entity { __typename = t.typename('Tag'); id = t.id; label = t.string; } - // A list's class and a detail class of one typename: a list-shaped event - // carries every field of the list's class, but not the detail's. + // A list-shaped event carries every CoinRow field, but not the detail's. class CoinRow extends Entity { __typename = t.typename('Coin'); id = t.id; @@ -1939,7 +1881,6 @@ describe('an event for a typename whose other class is not registered this sessi typename: 'Market', data: { __typename: 'Market', id: 'm1', top: row('c1', 3) }, }); - // A later event for the in-memory entity still merges. client.applyMutationEvent({ type: 'update', typename: 'Coin', data: row('c1', 4) }); expect(getDoc(kv, K('Coin', 'c1'))).toEqual(detailRecord(4)); diff --git a/packages/fetchium/src/__tests__/same-value.test.ts b/packages/fetchium/src/__tests__/same-value.test.ts index 3fadb13..67ab7a9 100644 --- a/packages/fetchium/src/__tests__/same-value.test.ts +++ b/packages/fetchium/src/__tests__/same-value.test.ts @@ -43,8 +43,6 @@ describe('sameValue', () => { }); it('treats NaN as unchanged and -0 as a change (Object.is semantics)', () => { - // Re-applying a NaN field must not notify on every refetch, and the sign - // of zero is observable through `Object.is` on a proxy read. expect(sameValue(Number.NaN, Number.NaN)).toBe(true); expect(sameValue(0, -0)).toBe(false); }); diff --git a/packages/fetchium/src/__tests__/skip-write-and-snapshot-fixes.test.ts b/packages/fetchium/src/__tests__/skip-write-and-snapshot-fixes.test.ts index 1f17054..bbd14a3 100644 --- a/packages/fetchium/src/__tests__/skip-write-and-snapshot-fixes.test.ts +++ b/packages/fetchium/src/__tests__/skip-write-and-snapshot-fixes.test.ts @@ -13,10 +13,7 @@ import { AsyncQueryStore, type AsyncPersistentStore } from '../stores/async.js'; import { valueKeyFor, refIdsKeyFor, queueKeyFor, DEFAULT_MAX_COUNT } from '../stores/shared.js'; import { createMockFetch, setupTestClient, testWithClient, sleep } from './utils.js'; -/** - * Invariants the snapshot fast path, unchanged-apply skipping, single-write - * events and store deletion reporting must keep. - */ +// Invariants of the snapshot fast path, write skipping, events and store deletion reporting. function getDoc(kv: MemoryPersistentStore, key: number): Record | undefined { const value = kv.getString(valueKeyFor(key)); @@ -61,10 +58,6 @@ function makeClient(store: SyncQueryStore | AsyncQueryStore, mockFetch: ReturnTy return { client, warn }; } -// ====================================================== -// Cache hydration must not re-parse an entity that is already in memory -// ====================================================== - describe('hydration of a cached query whose entity is already in memory', () => { const getClient = setupTestClient(); class User extends Entity { @@ -92,13 +85,12 @@ describe('hydration of a cached query whose entity is already in memory', () => mockFetch.get('/a', { user: payload }); mockFetch.get('/b', { user: payload, extra: 'e' }); - // Session 1 persists both queries. const { client: first } = makeClient(store, mockFetch); await holdQuery(first, () => fetchQuery(GetA)); await holdQuery(first, () => fetchQuery(GetB)); first.destroy(); - // Session 2: the network is unreachable; A hydrates, then B hydrates the same entity. + // Session 2, offline: A hydrates, then B hydrates the same entity. mockFetch.get('/a', { user: payload }, { delay: 10_000 }); mockFetch.get('/b', { user: payload, extra: 'e' }, { delay: 10_000 }); const { client, warn } = makeClient(store, mockFetch); @@ -113,8 +105,7 @@ describe('hydration of a cached query whose entity is already in memory', () => const hB = snapshotHarness(client, () => fetchQuery(GetB)); await sleep(20); - // The in-memory data was not re-parsed: the proxy, the store record and - // both consumers agree, and A's consumer had no reason to recompute. + // Not re-parsed, so A's consumer had no reason to recompute. const instance = client.entityMap.getEntity(hashValue(['User', 1]))!; expect((instance.data.profile as { since: unknown }).since).toBeDefined(); expect((hA.query.value as any).user.profile.since).toBeInstanceOf(Date); @@ -166,7 +157,6 @@ describe('hydration of a cached query whose entity is already in memory', () => mockFetch.get('/wide', { doc: { __typename: 'Doc', id: 1, title: 'T', body: 'B2' } }); const wide = holdQuery(client, () => fetchQuery(GetWide)); await wide; - // The cache was dropped with a warning and the network answered instead. expect(warn).toHaveBeenCalledWith(expect.stringContaining('query cache may be corrupted'), expect.anything()); expect((wide.value as any).doc.body).toBe('B2'); client.destroy(); @@ -210,8 +200,7 @@ describe('hydration of a cached query whose entity is already in memory', () => await holdQuery(first, () => fetchQuery(GetFull)); first.destroy(); - // Session 2: the summary hydrates User:1 without `meta.owner`; the full - // query must not resolve from the cache with the owner missing. + // Session 2: the summary hydrates User:1 without `meta.owner`. mockFetch.get('/full', { user: { ...full, meta: { ...full.meta, x: 2 } } }, { delay: 50 }); const { client, warn } = makeClient(store, mockFetch); const summary = holdQuery(client, () => fetchQuery(GetSummary)); @@ -321,8 +310,7 @@ describe('hydration of a cached query whose entity is already in memory', () => await holdQuery(first, () => fetchQuery(GetTeam)); first.destroy(); - // u-1 is in memory without `email`; the members array narrows it out on - // read, so the cached team is still usable as it is. + // u-1 lacks `email`, but the members array narrows it out on read. const { client, warn } = makeClient(store, mockFetch); await holdQuery(client, () => fetchQuery(GetOwner)); mockFetch.get( @@ -374,8 +362,7 @@ describe('hydration of a cached query whose entity is already in memory', () => await holdQuery(first, () => fetchQuery(GetPost, { id: 'p1' })); first.destroy(); - // The post comes into memory through an event routed into the feed: - // title only. Then the cached post query hydrates the same entity. + // An event brings the post into memory with its title only. const { client, warn } = makeClient(store, mockFetch); const feed = holdQuery(client, () => fetchQuery(GetFeed, { id: 'f1' })); await feed; @@ -398,7 +385,6 @@ describe('hydration of a cached query whose entity is already in memory', () => const post = holdQuery(client, () => fetchQuery(GetPost, { id: 'p1' })); await sleep(20); expect(warn).not.toHaveBeenCalled(); - // The event's title stays; the fields it did not carry come from the record. expect((post.value as any).post.title).toBe('T2'); expect((post.value as any).post.body).toBe('hello'); expect((post.value as any).post.tags).toEqual(['a', 'b']); @@ -476,8 +462,7 @@ describe('hydration of a cached query whose entity is already in memory', () => note: { text: 'n' }, }); const before = JSON.stringify(body); - // Hands out the same frozen object on every call, as an adapter that - // keeps its last response may. + // The same frozen object on every call, as an adapter may hand out. const fetch = async () => ({ ok: true, status: 200, headers: new Headers(), json: async () => body }); const { client, warn } = makeClient( new SyncQueryStore(new MemoryPersistentStore()), @@ -497,7 +482,7 @@ describe('hydration of a cached query whose entity is already in memory', () => expect(h.read().doc.meta.views).toBe(2); expect(h.read().doc.meta.stats.likes).toBe(4); expect(h.read().doc.counts.a).toBe(2); - // The live array grows its own array, not the response's (empty, frozen) one. + // The live array grows its own array, not the frozen response's. client.applyMutationEvent({ type: 'create', typename: 'DocNote', @@ -517,10 +502,6 @@ describe('hydration of a cached query whose entity is already in memory', () => }); }); -// ====================================================== -// Streamed events: what an event changes in an existing entity is written -// ====================================================== - describe('mutation events and the store', () => { const getClient = setupTestClient(); @@ -621,7 +602,7 @@ describe('mutation events and the store', () => { it('a full-payload update for an entity on disk but not in memory refreshes its record', async () => { const { client, mockFetch, kv, store } = getClient(); await seed(client, mockFetch, false); - // Evict User u1 from memory while its record stays referenced by the cached query. + // Its record stays referenced by the cached query. client.entityMap.getEntity(u1Key)!.evict(); expect(client.entityMap.getEntity(u1Key)).toBeUndefined(); expect(getDoc(kv, u1Key)!.name).toBe('Alice'); @@ -637,7 +618,6 @@ describe('mutation events and the store', () => { expect(getDoc(kv, u1Key)).toMatchObject({ name: 'Alicia', karma: 12 }); expect(store.hasEntity(u1Key)).toBe(true); - // An update for an entity the store has never seen leaves nothing behind. client.applyMutationEvent({ type: 'update', typename: 'User', @@ -648,10 +628,6 @@ describe('mutation events and the store', () => { }); }); -// ====================================================== -// `_persisted` is only trusted while the store agrees -// ====================================================== - describe('write skipping stays consistent with the store', () => { class User extends Entity { __typename = t.typename('User'); @@ -687,8 +663,7 @@ describe('write skipping stays consistent with the store', () => { expect((relay.value as any)?.user?.name ?? client.entityMap.getEntity(userKey)!.data.name).toBe('Alicia'); expect(getDoc(kv, userKey)).toMatchObject({ name: 'Alice' }); - // The store recovers; the refetch returns the same data the entity already - // holds, so nothing changed in memory, yet the record must be rewritten. + // The refetch matches memory, yet the record must be rewritten. failing = false; await withContexts([[QueryClientContext, client]], () => fetchQuery(GetUser).value ?? undefined); await (client as any).queryInstances.values().next().value.refetch(); @@ -707,9 +682,7 @@ describe('write skipping stays consistent with the store', () => { }); }); -// ====================================================== // AsyncQueryStore writer -// ====================================================== class AsyncDelegate implements AsyncPersistentStore { readonly kv: Record = Object.create(null); @@ -810,8 +783,7 @@ describe('AsyncQueryStore writer used in-process', () => { await drain(store); expect(delegate.kv[valueKeyFor(userKey('1'))]).toBeDefined(); - // Q3's SaveQuery will evict Q1 and cascade to User 1 once processed. Before - // the writer gets there, Q1 refetches identical data. + // Q3's queued SaveQuery will evict Q1 and cascade to User 1. Q1 refetches first. await holdQuery(client, () => fetchQuery(GetProfile, { id: '3' })); expect(store.isSettled()).toBe(false); await (q1.value as any).__refetch(); @@ -820,7 +792,6 @@ describe('AsyncQueryStore writer used in-process', () => { expect(delegate.kv[valueKeyFor(userKey('1'))]).toBeDefined(); expect(warn).not.toHaveBeenCalled(); - // Cold start on the same store: Q1 hydrates from cache. client.destroy(); clients.pop(); mockFetch.reset(); @@ -898,7 +869,7 @@ describe('AsyncQueryStore writer used in-process', () => { const def = (maxCount: number) => ({ statics: { id: defId, cache: { maxCount } } }) as never; const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {}); try { - // A 4-slot queue from an older install, fully populated, with records behind each key. + // A full 4-slot queue from an older install. const old = new Uint32Array([4, 3, 2, 1]); delegate.kv[queueKeyFor(defId)] = old; for (const key of old) { @@ -914,8 +885,7 @@ describe('AsyncQueryStore writer used in-process', () => { expect(Array.from(queue)).toEqual([9, 4, 3, 2, 1, 0]); expect(consoleError).not.toHaveBeenCalled(); - // Shrink (after a restart, so the in-memory queue is not cached): the - // keys that no longer fit are evicted, not stranded. + // Shrink after a restart: keys that no longer fit are evicted, not stranded. const restarted = writerStore(delegate); restarted.activateQuery(def(3), 9); await drain(restarted); @@ -946,10 +916,6 @@ describe('AsyncQueryStore writer used in-process', () => { }); }); -// ====================================================== -// Snapshot fast path -// ====================================================== - describe('snapshot fast path follow-ups', () => { const getClient = setupTestClient(); @@ -1130,8 +1096,7 @@ describe('snapshot fast path follow-ups', () => { const computesBefore = wrapped.computes(); expect('__refetch' in before.profile).toBe(false); - // The entity-rooted query applies identical data. Its own consumer sees - // the extras, but the wrapped query's consumer is not re-run for them. + // The wrapped query's consumer is not re-run for the root query's extras. const root = snapshotHarness(client, () => fetchQuery(GetRoot)); await root.query; await sleep(5); @@ -1233,7 +1198,6 @@ describe('snapshot fast path follow-ups', () => { expect(Object.is(h.read().quote.history[0], -0)).toBe(true); expect(h.computes()).toBe(before + 1); - // The same data again is not a change. client.applyMutationEvent({ type: 'update', typename: 'Quote', data: { id: 'q', price: -0, history: [-0, 1] } }); expect(h.computes()).toBe(before + 1); }); @@ -1260,8 +1224,7 @@ describe('snapshot fast path follow-ups', () => { expect(Object.isFrozen(snap.meta)).toBe(true); const warn = vi.spyOn(client.getContext().log as { warn: (...args: unknown[]) => void }, 'warn'); - // An app forwards part of a snapshot as an event payload (or a mutation's - // `effects.updates`); the parser must not write parsed values into it. + // A snapshot forwarded as an event payload must not be written into. client.applyMutationEvent({ type: 'update', typename: 'Doc', @@ -1361,8 +1324,7 @@ describe('snapshot fast path follow-ups', () => { await h.query; expect(h.read().roster.members.size).toBe(3000); - // Each event recomputes the snapshot at an unchanged version of `roster`, - // so the dev guard compares the rebuilt Set with the previous one. + // Each event makes the dev guard compare a rebuilt Set with the previous one. const started = performance.now(); for (let i = 0; i < 20; i++) { client.applyMutationEvent({ type: 'update', typename: 'Roster', data: { id: 'o', label: `b${i}` } }); diff --git a/packages/fetchium/src/applyEntities.ts b/packages/fetchium/src/applyEntities.ts index 8d24bb5..c357ec4 100644 --- a/packages/fetchium/src/applyEntities.ts +++ b/packages/fetchium/src/applyEntities.ts @@ -28,11 +28,9 @@ export interface ApplyResult { } /** - * Whether an apply writes to the store. `true` writes every entity that needs - * it, and `false` writes nothing (cache hydration). `'existing'` writes only - * entities already in memory. Ones this apply creates are written when a - * written record references them (`EntityInstance.save` writes missing - * children first), or by the caller once it knows the root is retained. + * `false` writes nothing (cache hydration). `'existing'` writes only entities + * already in memory. Created ones are written once a written record references + * them, or by the caller once it knows the root is retained. */ export type PersistMode = boolean | 'existing'; @@ -60,9 +58,6 @@ export function applyEntityRefs( throw e; } - // Entities a written record referenced while they were still being reified - // (a payload that links back to an ancestor) are complete now, and so are - // the records that reference them. queryClient.flushDeferredWrites(); return { data, entityRefs }; @@ -88,8 +83,7 @@ function reifyAndApply( return applyEntity(entity, seen, queryClient, persist, entityRefs, appendMode, created); } - // Assign only slots whose value changes: values the parser did not declare - // (and so did not copy) may belong to the caller and be frozen. + // Assign only on change: values the parser didn't copy may be frozen. if (Array.isArray(value)) { for (let i = 0; i < value.length; i++) { const item = value[i]; @@ -135,10 +129,7 @@ function applyEntity( const { key, data, shape: entityShape, rawKeys, eventKeys, fillsPartial } = entity; const shapeFields = entityShape.shape; - // An entity can appear in several slots of one payload (under `author` and - // `mentions`), all sharing one parsed object. Applying it again would find - // its children already reified, count none and release every child ref, so - // only count this slot's reference. + // Already applied from another slot: re-applying would release every child ref. const applied = queryClient.entityMap.getEntity(key); if (applied !== undefined && applied.parseId === queryClient.currentParseId) { parentEntityRefs.set(applied, (parentEntityRefs.get(applied) ?? 0) + 1); @@ -158,9 +149,7 @@ function applyEntity( // A refetch or poll that returns identical data neither notifies nor writes. let changed = true; - // Until the fields are reified the data is not a record yet: a write that - // reaches this instance through a reference from inside its own subtree is - // deferred to the end of the walk. + // A write reaching this instance from inside its own subtree is deferred. entityInstance._applying = true; try { if (isUpdate) { @@ -179,32 +168,20 @@ function applyEntity( created, ); if ((persist === true && rawKeys === undefined) || fillsPartial) { - // A full payload (a fetch), or the record's fields merged in, makes - // the data a complete record again. entityInstance._partial = false; entityInstance._partialKeys = undefined; } else if (entityInstance._partial && rawKeys !== undefined) { - // Stays partial even after events carry every field of this class: - // another class sharing the typename may have written fields this one - // doesn't declare, and only a merge keeps them. + // Stays partial: another class sharing the typename may hold undeclared fields. for (const k of rawKeys) entityInstance._partialKeys!.add(k); } } else { initFields(shapeFields, data, entityInstance, data, seen, queryClient, persist, childRefs, appendMode, created); if (eventKeys !== undefined) { - // Streamed events are partial: what this entity holds is what its - // events carried, not necessarily the whole record. entityInstance._partial = true; entityInstance._partialKeys = new Set(eventKeys); - // A record always carries its typename and id, whether or not the - // event did (the parser fills the literal in). if (entityShape.typenameField !== undefined) entityInstance._partialKeys.add(entityShape.typenameField); if (typeof entityShape.idField === 'string') entityInstance._partialKeys.add(entityShape.idField); - // Partial even if the event carried every field of this class, for the - // same reason (the other class may not be registered this session). - // With no record to merge into, the first write makes it whole. } else if (persist === false) { - // Hydrated from the store: its record exists. entityInstance._recorded = true; } if (persist === 'existing') created?.add(entityInstance); @@ -232,10 +209,8 @@ function applyEntity( const newRefs = childRefs.size > 0 ? childRefs : undefined; const refsChanged = !sameRefs(entityInstance.entityRefs, newRefs); - // An entity hydrated from the store was never written, so there is no write - // to skip. One with a write still queued has its current data on the way. - // Skipping is only safe while the store has no deletion queued, which could - // drop the record this apply trusts. + // An entity hydrated from the store was never written, so there is no write to skip. + // A queued store deletion could drop the record this apply trusts. const needsPersist = changed || refsChanged || @@ -276,8 +251,6 @@ function mergeFields( const raw = data[fieldKey]; if (shouldReify(raw)) { - // Assign only on change: an object the parser kept as is (from a fetch - // result) may belong to the adapter and be frozen. const reified = reifyAndApply(raw, seen, queryClient, persist, childRefs, appendMode, created); if (reified !== raw) data[fieldKey] = reified; } @@ -320,8 +293,7 @@ function mergeFields( !(propShape.shape instanceof Set) ? (propShape.shape as Record) : undefined; - // An unchanged object kept from a fetch result may be frozen by the - // adapter: merge into a copy. + // May be frozen by the adapter. const target = Object.isFrozen(oldVal) ? { ...oldVal } : oldVal; if (nestedShape !== undefined) { if ( @@ -346,8 +318,8 @@ function mergeFields( existingData[fieldKey] = oldVal; } } else if (sameKeys(oldVal, newVal)) { - // Shapeless object (e.g. a record) with the same keys: copy field by field, keeping the - // existing value — and its identity — wherever it already matches. + // Shapeless object (e.g. a record) with the same keys: copy only the + // fields that differ, so matching values keep their identity. let merged = false; for (const k of Object.keys(newVal)) { if (!sameValue(oldVal[k], newVal[k])) { @@ -389,7 +361,6 @@ export function sameKeys(a: Record, b: Record) * they were built from, arrays and plain objects element by element. */ export function sameValue(a: unknown, b: unknown): boolean { - // `Object.is`: re-applying NaN is not a change, and `-0` over `0` is one. if (Object.is(a, b)) return true; if (typeof a !== 'object' || typeof b !== 'object' || a === null || b === null) return false; if (a instanceof FormattedValue || b instanceof FormattedValue) { @@ -459,8 +430,6 @@ function initFields( const raw = data[fieldKey]; if (shouldReify(raw)) { - // Assign only on change: an object the parser kept as is (from a fetch - // result) may belong to the adapter and be frozen. const reified = reifyAndApply(raw, seen, queryClient, persist, childRefs, appendMode, created); if (reified !== raw) data[fieldKey] = reified; } diff --git a/packages/fetchium/src/errors.ts b/packages/fetchium/src/errors.ts index b13059a..044db65 100644 --- a/packages/fetchium/src/errors.ts +++ b/packages/fetchium/src/errors.ts @@ -171,10 +171,8 @@ export function typeError(path: string, expectedType: InternalObjectFieldTypeDef } /** - * A cached query can't be hydrated as-is: an entity it holds is already in - * memory and doesn't satisfy the query's shape. Unlike a bad array item, this - * drops the whole cached query (which is refetched), since a list silently - * missing an item is worse. + * An in-memory entity doesn't satisfy a cached query's shape. Drops the whole + * cached query rather than filtering the item out. */ export class CachedEntityMismatchError extends Error { constructor(message: string) { diff --git a/packages/fetchium/src/parseEntities.ts b/packages/fetchium/src/parseEntities.ts index 15a6348..5dbec38 100644 --- a/packages/fetchium/src/parseEntities.ts +++ b/packages/fetchium/src/parseEntities.ts @@ -50,13 +50,13 @@ export interface ParsedEntity { data: Record; /** Set for partial event updates — restricts mergeFields to only these keys. */ rawKeys: Set | undefined; - /** The keys a streamed event carried, whether or not the entity was in memory. */ + /** The keys a streamed event carried. */ eventKeys: Set | undefined; - /** A cached record parsed to fill in the fields an entity built from events lacks. */ + /** A cached record filling in fields an entity built from events lacks. */ fillsPartial: boolean; } -/** Trust-memory verdicts: `true`/`false` once settled, `'pending'` while a check is in progress (a cycle). */ +/** `'pending'` while a check is in progress, so a cycle counts as satisfied. */ export type TrustMemo = Map, boolean | 'pending'>>; // ====================================================== @@ -72,13 +72,10 @@ export class ParseContext { isPartialEvent: boolean = false; seen: Map, ParsedEntity> | undefined = undefined; seenByKey: Map | undefined = undefined; - /** Verdicts of the trust-memory check, per instance and def, for one parse. */ trusted: TrustMemo | undefined = undefined; /** - * Whether nested objects and records always parse into a copy. On for input - * the client doesn't own and may see again (an event payload, mutation - * effects, a snapshot fed back). Off for a fetch result or cached record, - * which is copied only where a parsed value differs from it. + * On for input the client doesn't own and may see again (events, effects). + * Off for a fetch result or cached record, copied only where a value differs. */ copyInput: boolean = true; @@ -394,10 +391,7 @@ function parseUnionData( // ====================================================== function parseArrayData(array: unknown[], itemShape: ComplexTypeDef, ctx: ParseContext, path: string): unknown[] { - // Without `ctx.copyInput`, an array whose items all parse to themselves is - // returned as is, and the copy starts at the first item that differs or - // fails. An empty array is always new, since a live array grows the one it - // holds. + // An empty array is always new, since a live array grows the one it holds. let result: unknown[] | undefined = ctx.copyInput || array.length === 0 ? [] : undefined; for (let i = 0; i < array.length; i++) { @@ -420,10 +414,7 @@ function parseArrayData(array: unknown[], itemShape: ComplexTypeDef, ctx: ParseC return result ?? array; } -// Neither walker writes into its input. With `ctx.copyInput` they parse into -// a shallow copy. Without it, the copy is made only once a parsed value -// differs, and input whose values all parse to themselves is returned as is. -// The result keeps every key the input had. +// Neither walker writes into its input. function parseRecordData( record: Record, @@ -458,7 +449,6 @@ function parseObjectData( const value = obj[key]; const parsed = parseData(value, propShape as unknown as TypeDef, ctx, `${path}.${key}`); if (result !== undefined) result[key] = parsed; - // A shape key the input lacks is still set (to undefined), as a copy does. else if (parsed !== value || (value === undefined && !(key in obj))) (result = { ...obj })[key] = parsed; } @@ -510,22 +500,16 @@ function parseEntityData( if (preloadedEntities !== undefined) { const existing = queryClient.entityMap.getEntity(key); - // For an entity built from events: the fields they carried. The record - // supplies the rest. let fillKeys: Set | undefined; const preloaded = preloadedEntities.get(key); if (existing !== undefined && existing._partial && existing._partialKeys !== undefined && preloaded !== undefined) { - // Built from events, the in-memory data lacks what the record has: - // parse the record for those fields only and merge them in. + // Built from events: parse only the fields the record adds. fillKeys = existing._partialKeys; obj = preloaded; } else if (existing !== undefined) { - // A live entity is newer than the cache. Re-parsing its data would run - // already-parsed values (formatted values, parse results, child - // proxies) through the parser again and corrupt them. An empty partial - // entry merges nothing and hands the query the existing proxy. The data - // must still satisfy this query's shape, or the cached query is unusable. + // A live entity is newer than the cache, and re-parsing parsed values + // corrupts them. Merge nothing, but check it satisfies this shape. ctx.trusted ??= new Map(); if (!dataSatisfiesDef(existing.data, entityShape as unknown as ValidatorDef, queryClient, ctx.trusted)) { throw new CachedEntityMismatchError( @@ -613,11 +597,8 @@ function parseEntityData( // ====================================================== /** - * Whether already-parsed entity data can be handed to a query declaring `def` - * without re-parsing. Unlike `entitySatisfiesShape` (a top-level presence - * check for narrowing), this descends into nested values and child entities: - * a cached query missing a required nested field is worse than a refetch. - * Values that can't be judged are accepted, matching the parser's leniency. + * Whether parsed entity data satisfies `def` deeply, unlike the top-level + * `entitySatisfiesShape`. Values that can't be judged are accepted. */ export function dataSatisfiesDef( data: Record, @@ -686,8 +667,6 @@ function valueSatisfiesDef(value: unknown, fieldDef: unknown, queryClient: Query } if ((mask & Mask.UNION) !== 0) { - // Resolve the member the value belongs to; an unrecognised value is left - // to the union's own leniency. const members = fieldDef.shape as Record | undefined; if (members === undefined || members === null) return true; if (Array.isArray(value)) { @@ -741,12 +720,7 @@ function valueSatisfiesDef(value: unknown, fieldDef: unknown, queryClient: Query return (mask & maskOf(value)) !== 0; } -/** - * An entity field holds a proxy. One whose instance has left memory is - * accepted, since a re-parse would hand out its data too. A live one must - * satisfy the def. Verdicts are shared across the parse, and a cycle counts as - * satisfied. - */ +/** An instance no longer in memory is accepted, since a re-parse would hand out its data too. */ function entitySatisfies( value: object, def: ValidatorDef, @@ -767,11 +741,7 @@ function entitySatisfies( return verdict; } -/** - * When the item typename has several registered classes, the array is - * narrowed on read to members that satisfy the def, so a member that doesn't - * is not a mismatch. - */ +/** A shared-typename array is narrowed on read, so a non-matching member is not a mismatch. */ function arraySatisfies(value: unknown[], itemDef: unknown, queryClient: QueryClient, visiting: TrustMemo): boolean { let narrowed = false; if (itemDef instanceof ValidatorDef && (itemDef.mask & Mask.ENTITY) !== 0 && (itemDef.mask & Mask.UNION) === 0) { diff --git a/packages/fetchium/src/query-types.ts b/packages/fetchium/src/query-types.ts index 40e4357..c2365e7 100644 --- a/packages/fetchium/src/query-types.ts +++ b/packages/fetchium/src/query-types.ts @@ -112,11 +112,9 @@ export interface QueryStore { saveEntity(entityKey: number, value: unknown, refIds?: Set): void; /** - * Writes the fields streamed events supplied for an entity built from them. - * If the store holds a record for the key, it merges `fields` over it and - * derives references from the merged value. Otherwise `fields` becomes the - * record. Without this method the client calls `saveEntity` with the whole - * in-memory data, which drops the fields the events did not carry. + * Writes the fields streamed events supplied, merged over the stored record + * if one exists (references derived from the merged value). Without it the + * client calls `saveEntity`, dropping fields the events did not carry. */ mergeEntity?(entityKey: number, fields: unknown, refIds?: Set): void; @@ -128,41 +126,35 @@ export interface QueryStore { /** * Called with the key of every record the store drops on its own (eviction, - * cascade, purge). May return an unsubscribe function, which the client - * calls from `destroy()`. A store without this hook is written on every - * apply. + * cascade, purge). May return an unsubscribe, called from `destroy()`. + * Without it, entities are written on every apply. */ onDelete?(listener: (key: number) => void): void | (() => void); /** - * For stores that process writes asynchronously: called with each entity - * key once its write has been applied. The client then treats an entity as - * persisted only after this acknowledgement, so a dropped write is retried. + * Async stores: called with each entity key once its write is applied. The + * client treats an entity as persisted only after this acknowledgement. */ onPersisted?(listener: (key: number) => void): void | (() => void); /** - * For stores that process writes asynchronously: whether every operation - * handed to the store has been processed. While it returns `false` the - * client does not skip entity writes, since an operation still queued could - * delete the record the skip relies on. Absent means always settled. + * Async stores: whether every queued operation has been processed. While + * `false` the client does not skip unchanged entity writes. Absent means + * always settled. */ isSettled?(): boolean; /** - * For stores that process writes asynchronously, a narrower `isSettled`: - * whether an operation that could delete a record is still queued. The - * client's own entity writes don't count, since it rewrites any entity whose - * reference they drop. When present it replaces `isSettled`, so a burst of - * entity writes doesn't disable write skipping for every other entity. + * Async stores: a narrower `isSettled` that ignores the client's own entity + * writes, reporting only queued operations that could delete a record. + * Takes precedence over `isSettled` when present. */ hasQueuedDeletes?(): boolean; /** - * Whether the store holds a record for this entity key, or `undefined` if - * it can't tell yet (it hasn't finished reading what it holds). A streamed - * event for an entity not in memory refreshes the record only when this - * returns `true`. Without this hook, such events are not written. + * Whether the store holds a record for this key, or `undefined` if it can't + * tell yet. An event for an entity not in memory is written only when this + * returns `true`. */ hasEntity?(key: number): boolean | undefined; } diff --git a/packages/fetchium/src/stores/async.ts b/packages/fetchium/src/stores/async.ts index 4db29d7..c512f4d 100644 --- a/packages/fetchium/src/stores/async.ts +++ b/packages/fetchium/src/stores/async.ts @@ -51,7 +51,7 @@ export type StoreMessage = value: unknown; updatedAt: number; cacheTime: number; - /** Absent on the wire from readers built before 0.6; the writer then keeps the persisted queue's size. */ + /** Absent from older readers. The writer then keeps the persisted queue's size. */ maxCount?: number; refIds?: number[]; } @@ -60,7 +60,7 @@ export type StoreMessage = entityKey: number; value: unknown; refIds?: number[]; - /** `value` holds only some fields; the writer merges them over the stored record, if any. */ + /** `value` holds only some fields, merged over the stored record. */ merge?: boolean; } | { type: StoreMessageType.ActivateQuery; queryDefId: string; queryKey: number; cacheTime: number; maxCount?: number } @@ -74,11 +74,7 @@ export interface AsyncQueryStoreConfig { delegate?: AsyncPersistentStore; // Only provided for writer } -/** - * Writer-internal work run through the same serial queue as wire messages, so - * it is ordered with the writes and deletions around it. Never sent over the - * wire. - */ +/** Writer-internal work, serialized with wire messages. Never sent over the wire. */ class InternalWork { constructor( readonly run: () => Promise, @@ -89,7 +85,6 @@ class InternalWork { type QueuedWork = StoreMessage | InternalWork; -/** Whether something received on the channel is a message this writer handles. */ function isStoreMessage(msg: unknown): msg is StoreMessage { if (typeof msg !== 'object' || msg === null) return false; const type = (msg as { type?: unknown }).type; @@ -125,19 +120,15 @@ export class AsyncQueryStore implements QueryStore { private deleteListeners: Array<(key: number) => void> = []; private persistedListeners: Array<(key: number) => void> = []; private processing = false; - // Queued work that can delete a record: everything except this writer's - // own client's entity writes, whose dropped references the client tracks. + // Everything except this client's entity writes, whose dropped refs the client tracks. private queuedDeletes = 0; private readonly ownEntityWrites = new WeakSet(); - // Ids whose value the delegate holds (queued writes included), mirrored so - // `hasEntity` can answer synchronously. Read from the delegate once at - // start without blocking the queue. Writes and deletions before that read - // lands are noted aside and folded in. + // Mirror of the delegate's value keys so `hasEntity` answers synchronously. + // Changes before the initial scan lands are noted aside and folded in. private heldKeys: Set | undefined; private heldSinceScan: Set | undefined = new Set(); private droppedSinceScan: Set | undefined = new Set(); - // Only the writer sees deletions and completed writes. A reader offers no - // hooks, so the client writes every apply. + // Writer only. A reader offers no hooks, so its client writes every apply. onDelete?: (listener: (key: number) => void) => () => void; onPersisted?: (listener: (key: number) => void) => () => void; hasEntity?: (key: number) => boolean | undefined; @@ -152,7 +143,6 @@ export class AsyncQueryStore implements QueryStore { this.hasQueuedDeletes = () => this.queuedDeletes > 0; this.hasEntity = key => { if (this.heldKeys !== undefined) return this.heldKeys.has(key); - // Not read yet: only what this writer itself wrote is known. return this.heldSinceScan!.has(key) ? true : undefined; }; } @@ -171,7 +161,6 @@ export class AsyncQueryStore implements QueryStore { } } - /** Queues writer-internal work behind everything already queued. */ private runInternal(run: () => Promise): Promise { return new Promise((resolve, reject) => { this.enqueueMessage(new InternalWork(run, resolve, reject)); @@ -232,7 +221,7 @@ export class AsyncQueryStore implements QueryStore { try { held = await this.delegate!.has(valueKeyFor(id)); } catch { - // Unknown: treat it as missing. + // Treat as missing. } if (held) this.noteHeld(id); else this.noteDropped(id); @@ -268,8 +257,7 @@ export class AsyncQueryStore implements QueryStore { } } catch (error) { console.error('Error processing message:', error); - // A failed entity write leaves the record missing or stale. Report it - // as dropped so the client writes the entity again on its next apply. + // Report as dropped so the client rewrites it on its next apply. if (!(msg instanceof InternalWork) && msg.type === StoreMessageType.SaveEntity) { await this.recheckHeld(msg.entityKey); for (const listener of this.deleteListeners.slice()) listener(msg.entityKey); @@ -281,12 +269,7 @@ export class AsyncQueryStore implements QueryStore { } } - /** - * Reads which records the delegate holds, alongside the queue rather than - * ahead of it. Writes and deletions processed meanwhile are folded in. - * A failed read is retried. If it keeps failing nothing is assumed held, so - * an event for an entity not in memory is not written. - */ + /** Runs alongside the queue, not ahead of it. On repeated failure nothing is assumed held. */ private async scanHeldKeys(): Promise { const held = new Set(); for (let attempt = 0; ; attempt++) { @@ -312,10 +295,6 @@ export class AsyncQueryStore implements QueryStore { this.heldSinceScan = this.droppedSinceScan = undefined; } - /** - * Whether every operation handed to this store has been applied. Readers - * hand their operations to a writer elsewhere and never skip writes anyway. - */ isSettled(): boolean { return !this.isWriter || (this.messageQueue.length === 0 && !this.processing); } @@ -388,8 +367,7 @@ export class AsyncQueryStore implements QueryStore { } for (const entityId of entityIds) { - // Records can reference each other in a cycle (a file whose record - // links back to its folder); one read per record. + // Records can reference each other in a cycle. if (preloaded.has(entityId)) continue; const entityValue = await this.delegate.getString(valueKeyFor(entityId)); @@ -523,11 +501,8 @@ export class AsyncQueryStore implements QueryStore { queue = new Uint32Array(maxCount ?? DEFAULT_MAX_COUNT); await this.delegate!.setBuffer(queueKey, queue); } else if (maxCount !== undefined && queue.length !== maxCount) { - // `maxCount` changed since the queue was written (writers before 0.6 - // always used the default of 50). A view over the old buffer can't - // grow, and a shorter one would strand the keys it drops: copy what - // fits and evict the rest. The key being activated moves to the - // front, so it is kept wherever it sits. + // A view over the old buffer can't grow, and a shorter one would strand + // the keys it drops. The activated key moves to the front, so keep it. const resized = new Uint32Array(maxCount); resized.set(queue.subarray(0, Math.min(queue.length, maxCount))); for (let i = maxCount; i < queue.length; i++) { @@ -568,11 +543,7 @@ export class AsyncQueryStore implements QueryStore { } } - /** - * Drops every query definition's cached queries once the definition has - * gone unused for its `cacheTime`. Runs through the writer's serial queue, - * so a record it drops cannot be reported as written afterwards. - */ + /** Runs through the writer's queue, so a record it drops can't be reported as written afterwards. */ purgeStaleQueries(): Promise { if (!this.delegate) return Promise.resolve(); if (!this.isWriter) return this.purgeStaleQueriesNow(); diff --git a/packages/fetchium/src/stores/shared.ts b/packages/fetchium/src/stores/shared.ts index 3d2dcb2..0fff5c8 100644 --- a/packages/fetchium/src/stores/shared.ts +++ b/packages/fetchium/src/stores/shared.ts @@ -17,11 +17,7 @@ export const DEFAULT_MAX_COUNT = 50; export const DEFAULT_CACHE_TIME = 60 * 24; // 24 hours in minutes export const DEFAULT_GC_TIME = 5; // 5 minutes - in-memory eviction default -/** - * Merges the fields of a partial record over a stored one, at the JSON level - * (proxies have already been serialized to `{ __entityRef }` markers), and - * derives the merged record's references from the markers it contains. - */ +/** Merges at the JSON level and derives refs from the `__entityRef` markers. */ export function mergeStoredRecord( stored: string, partial: unknown, @@ -32,8 +28,6 @@ export function mergeStoredRecord( } catch { return undefined; } - // A non-object record can't be merged into. The caller writes the fields as - // the record instead, which heals it. if (typeof record !== 'object' || record === null || Array.isArray(record)) return undefined; const value = { ...(record as Record), diff --git a/packages/fetchium/src/stores/sync.ts b/packages/fetchium/src/stores/sync.ts index b9edf49..801286f 100644 --- a/packages/fetchium/src/stores/sync.ts +++ b/packages/fetchium/src/stores/sync.ts @@ -71,11 +71,7 @@ export class MemoryPersistentStore implements SyncPersistentStore { } } -/** - * Delete listeners are keyed by the backing kv, not the store instance: two - * stores over one kv delete each other's records, and a client behind either - * must hear about it. - */ +/** Keyed by kv, not store: two stores over one kv delete each other's records. */ const deleteListenersByKv = new WeakMap void>>(); export class SyncQueryStore implements QueryStore { @@ -137,8 +133,7 @@ export class SyncQueryStore implements QueryStore { private preloadEntities(entityIds: Uint32Array, preloaded: PreloadedEntityMap): void { for (const entityId of entityIds) { - // Records can reference each other in a cycle (a file whose record - // links back to its folder); one read per record. + // Records can reference each other in a cycle. if (preloaded.has(entityId)) continue; const entityValue = this.kv.getString(valueKeyFor(entityId)); @@ -197,10 +192,8 @@ export class SyncQueryStore implements QueryStore { queue = new Uint32Array(maxCount); this.kv.setBuffer(queueKeyFor(queryDefId), queue); } else if (queue.length !== maxCount) { - // `maxCount` changed since the queue was written. A view over the old - // buffer can't grow, and a shorter one would strand the keys it drops: - // copy what fits and evict the rest. The key being activated moves to - // the front, so it is kept wherever it sits. + // A view over the old buffer can't grow, and a shorter one would strand + // the keys it drops. The activated key moves to the front, so keep it. const resized = new Uint32Array(maxCount); resized.set(queue.subarray(0, Math.min(queue.length, maxCount))); for (let i = maxCount; i < queue.length; i++) { @@ -326,7 +319,7 @@ export class SyncQueryStore implements QueryStore { kv.delete(valueKeyFor(id)); kv.delete(refCountKeyFor(id)); - // A copy: a listener may unsubscribe while being notified. + // A listener may unsubscribe while being notified. for (const listener of this.deleteListeners.slice()) listener(id); const refIds = kv.getBuffer(refIdsKeyFor(id)); From 96a5200c0a4e4cda1f30763420b5e07dbe36711c Mon Sep 17 00:00:00 2001 From: Adam Azbel-Bedell Date: Wed, 7 Oct 2026 13:01:13 -0400 Subject: [PATCH 15/15] fix(fetchium): keep event writes and store listeners safe when the store can't answer A failed write whose has() recheck also fails now retries, then leaves the key held. An event refresh is sent as an ifStored merge, which the writer drops and reports when no record exists, so a wrongly held key never writes partial fields as a record. Events for entities not in memory are refreshed during the writer's key scan and from readers again. A throwing store listener is logged instead of stopping the writer or a deletion cascade. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/src/app/api/fetchium/page.md | 10 +- docs/src/app/api/stores-async/page.md | 26 ++-- packages/fetchium/src/EntityInstance.ts | 9 +- packages/fetchium/src/EntityStore.ts | 18 ++- packages/fetchium/src/QueryClient.ts | 16 ++- .../src/__tests__/event-persistence.test.ts | 94 +++++++++++++-- .../skip-write-and-snapshot-fixes.test.ts | 111 ++++++++++++++++++ .../src/__tests__/store-on-delete.test.ts | 88 ++++++++++++++ packages/fetchium/src/query-types.ts | 8 +- packages/fetchium/src/stores/async.ts | 51 +++++--- packages/fetchium/src/stores/shared.ts | 11 ++ packages/fetchium/src/stores/sync.ts | 4 +- 12 files changed, 379 insertions(+), 67 deletions(-) diff --git a/docs/src/app/api/fetchium/page.md b/docs/src/app/api/fetchium/page.md index 3d6f76e..5120ea2 100644 --- a/docs/src/app/api/fetchium/page.md +++ b/docs/src/app/api/fetchium/page.md @@ -710,8 +710,8 @@ The `t` object provides a declarative type definition DSL for describing query p ### Store types -| Type | Definition | Description | -| -------------------- | ----------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `QueryStore` | Interface | Storage backend interface. See [stores/sync](/api/stores-sync) and [stores/async](/api/stores-async). Besides the required methods, a store may implement `onDelete` (the client then skips writing entities whose records are current, and writes one again once the store reports its record dropped), `onPersisted` (for a store that applies writes later), `isSettled`, `hasEntity` (lets a streamed event refresh the record of an entity that is not in memory) and `mergeEntity` (merges the fields streamed events supplied for an entity built from them over its record; without it, such an entity is written whole through `saveEntity`). A store without `onDelete` is written on every apply, as before 0.6; without `hasEntity`, an event for an entity that is not in memory and that no live array retains is not written. | -| `CachedQuery` | `{ value: unknown; refIds: Set \| undefined; updatedAt: number; preloadedEntities?: PreloadedEntityMap }` | Cached query data returned by `QueryStore.loadQuery()`. | -| `PreloadedEntityMap` | `Map>` | Pre-loaded entity data for hydration. | +| Type | Definition | Description | +| -------------------- | ----------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `QueryStore` | Interface | Storage backend interface. See [stores/sync](/api/stores-sync) and [stores/async](/api/stores-async). Besides the required methods, a store may implement `onDelete` (the client then skips writing entities whose records are current, and writes one again once the store reports its record dropped), `onPersisted` (for a store that applies writes later), `isSettled`, `hasEntity` (lets a streamed event refresh the record of an entity that is not in memory) and `mergeEntity` (merges the fields streamed events supplied for an entity built from them over its record; without it, such an entity is written whole through `saveEntity`). A store without `onDelete` is written on every apply, as before 0.6; when `hasEntity` is missing or answers `undefined`, an event for an entity that is not in memory and that no live array retains is written only as a `mergeEntity` call with `ifStored`, which writes nothing when the store holds no record, and not at all without `mergeEntity`. | +| `CachedQuery` | `{ value: unknown; refIds: Set \| undefined; updatedAt: number; preloadedEntities?: PreloadedEntityMap }` | Cached query data returned by `QueryStore.loadQuery()`. | +| `PreloadedEntityMap` | `Map>` | Pre-loaded entity data for hydration. | diff --git a/docs/src/app/api/stores-async/page.md b/docs/src/app/api/stores-async/page.md index bbe2c6b..ab56842 100644 --- a/docs/src/app/api/stores-async/page.md +++ b/docs/src/app/api/stores-async/page.md @@ -41,19 +41,19 @@ new AsyncQueryStore(config: AsyncQueryStoreConfig) #### Methods -| Method | Signature | Description | -| ------------------- | -------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `loadQuery` | `(queryDef: QueryDefinition, queryKey: number): Promise` | Loads a cached query. Only works when a `delegate` is available (writer mode). Returns `undefined` if no delegate, if the entry has expired, or if it does not exist. Preloads referenced entities. | -| `saveQuery` | `(queryDef: QueryDefinition, queryKey: number, value: unknown, updatedAt: number, refIds?: Set): void` | Dispatches a save-query message. In writer mode, enqueues for serial processing. In reader mode, sends via `sendMessage`. | -| `saveEntity` | `(entityKey: number, value: unknown, refIds?: Set): void` | Dispatches a save-entity message. | -| `mergeEntity` | `(entityKey: number, fields: unknown, refIds?: Set): void` | Dispatches a save-entity message with `merge: true`; the writer merges `fields` over the stored record, if there is one (see `SyncQueryStore.mergeEntity`). | -| `activateQuery` | `(queryDef: QueryDefinition, queryKey: number): void` | Dispatches an activate-query message to update the LRU queue. A queue persisted with a different `maxCount` is resized by the writer: growing reallocates it, shrinking evicts the keys that no longer fit. | -| `deleteQuery` | `(queryKey: number): void` | Dispatches a delete-query message. | -| `purgeStaleQueries` | `(): Promise` | Scans all stored query classes and removes expired entries. In writer mode it runs through the same serial queue as the messages, after everything queued before it; a reader with its own `delegate` runs it directly. Only operates when a `delegate` is available. | -| `onDelete` | `(listener: (key: number) => void): () => void` | Writer mode only. Registers a listener called with the key of every record the writer drops on its own (an LRU eviction, a reference-count cascade, a purge) or fails to write. Returns an unsubscribe function. | -| `onPersisted` | `(listener: (key: number) => void): () => void` | Writer mode only. Registers a listener called with the key of every entity record a queued write has stored. Returns an unsubscribe function. | -| `isSettled` | `(): boolean` | Whether every operation handed to the writer has been applied (nothing queued, nothing mid-flight). Always `true` in reader mode. | -| `hasEntity` | `(key: number): boolean \| undefined` | Writer mode only. Whether the delegate holds a record for the key, answered from a set the writer reads from the delegate when it starts (alongside its queue, not ahead of it) and keeps current with every write it queues and every deletion it performs. `undefined` for a key it has not written itself until that read lands; after a read that keeps failing, nothing is assumed held. | +| Method | Signature | Description | +| ------------------- | -------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `loadQuery` | `(queryDef: QueryDefinition, queryKey: number): Promise` | Loads a cached query. Only works when a `delegate` is available (writer mode). Returns `undefined` if no delegate, if the entry has expired, or if it does not exist. Preloads referenced entities. | +| `saveQuery` | `(queryDef: QueryDefinition, queryKey: number, value: unknown, updatedAt: number, refIds?: Set): void` | Dispatches a save-query message. In writer mode, enqueues for serial processing. In reader mode, sends via `sendMessage`. | +| `saveEntity` | `(entityKey: number, value: unknown, refIds?: Set): void` | Dispatches a save-entity message. | +| `mergeEntity` | `(entityKey: number, fields: unknown, refIds?: Set, ifStored?: boolean): void` | Dispatches a save-entity message with `merge: true`; the writer merges `fields` over the stored record, if there is one (see `SyncQueryStore.mergeEntity`). With `ifStored` and no stored record, the writer writes nothing and reports the key to `onDelete` listeners. | +| `activateQuery` | `(queryDef: QueryDefinition, queryKey: number): void` | Dispatches an activate-query message to update the LRU queue. A queue persisted with a different `maxCount` is resized by the writer: growing reallocates it, shrinking evicts the keys that no longer fit. | +| `deleteQuery` | `(queryKey: number): void` | Dispatches a delete-query message. | +| `purgeStaleQueries` | `(): Promise` | Scans all stored query classes and removes expired entries. In writer mode it runs through the same serial queue as the messages, after everything queued before it; a reader with its own `delegate` runs it directly. Only operates when a `delegate` is available. | +| `onDelete` | `(listener: (key: number) => void): () => void` | Writer mode only. Registers a listener called with the key of every record the writer drops on its own (an LRU eviction, a reference-count cascade, a purge) or fails to write. Returns an unsubscribe function. | +| `onPersisted` | `(listener: (key: number) => void): () => void` | Writer mode only. Registers a listener called with the key of every entity record a queued write has stored. Returns an unsubscribe function. | +| `isSettled` | `(): boolean` | Whether every operation handed to the writer has been applied (nothing queued, nothing mid-flight). Always `true` in reader mode. | +| `hasEntity` | `(key: number): boolean \| undefined` | Writer mode only. Whether the delegate holds a record for the key, answered from a set the writer reads from the delegate when it starts (alongside its queue, not ahead of it) and keeps current with every write it queues and every deletion it performs. `undefined` for a key it has not written itself until that read lands; after a read that keeps failing, nothing is assumed held. An event for an entity not in memory is sent as an `ifStored` merge while the answer is `undefined`, and always from a reader. | --- diff --git a/packages/fetchium/src/EntityInstance.ts b/packages/fetchium/src/EntityInstance.ts index ae28ec2..041b733 100644 --- a/packages/fetchium/src/EntityInstance.ts +++ b/packages/fetchium/src/EntityInstance.ts @@ -618,8 +618,11 @@ export class EntityInstance { return false; } - /** `storeHolds`: the store's `hasEntity` answer, if the caller already has it. */ - save(storeHolds?: boolean): void { + /** + * `storeHolds`: the store's `hasEntity` answer, if the caller already has it. + * `ifStored`: a merge only refreshes a stored record, never creates one. + */ + save(storeHolds?: boolean, ifStored?: boolean): void { const client = this._queryClient; if (this._saving) return; this._saving = true; @@ -660,7 +663,7 @@ export class EntityInstance { // Counted before the call: a store may acknowledge synchronously. if (client.storeAcksWrites) this._pendingWrites++; try { - client.entityMap.save(this, this._partial ? this._partialKeys : undefined); + client.entityMap.save(this, this._partial ? this._partialKeys : undefined, ifStored); } catch (e) { this.markUnwritten(); throw e; diff --git a/packages/fetchium/src/EntityStore.ts b/packages/fetchium/src/EntityStore.ts index 45410a7..32f8522 100644 --- a/packages/fetchium/src/EntityStore.ts +++ b/packages/fetchium/src/EntityStore.ts @@ -3,14 +3,20 @@ import type { QueryClient } from './QueryClient.js'; import { EntityInstance } from './EntityInstance.js'; import { ValidatorDef } from './typeDefs.js'; +type PersistEntity = ( + key: number, + data: Record, + refKeys?: Set, + merge?: boolean, + ifStored?: boolean, +) => void; + export class EntityStore { private instances = new Map(); - private persistEntity: (key: number, data: Record, refKeys?: Set, merge?: boolean) => void; + private persistEntity: PersistEntity; mergesEntities: boolean = false; - constructor( - persistEntity: (key: number, data: Record, refKeys?: Set, merge?: boolean) => void, - ) { + constructor(persistEntity: PersistEntity) { this.persistEntity = persistEntity; } @@ -62,7 +68,7 @@ export class EntityStore { } /** With `mergeKeys`, only those fields are sent, merged over the stored record. */ - save(instance: EntityInstance, mergeKeys?: Set): void { + save(instance: EntityInstance, mergeKeys?: Set, ifStored?: boolean): void { let refKeys: Set | undefined; if (instance.entityRefs) { refKeys = new Set(); @@ -75,6 +81,6 @@ export class EntityStore { value = {}; for (const k of mergeKeys) value[k] = instance.data[k]; } - this.persistEntity(instance.key, value, refKeys, merge); + this.persistEntity(instance.key, value, refKeys, merge, ifStored); } } diff --git a/packages/fetchium/src/QueryClient.ts b/packages/fetchium/src/QueryClient.ts index 5b7942e..f6cbb9f 100644 --- a/packages/fetchium/src/QueryClient.ts +++ b/packages/fetchium/src/QueryClient.ts @@ -188,8 +188,8 @@ export class QueryClient { config.gcManager ?? (this.isServer ? new NoOpGcManager() : new GcManager(this.handleEviction, evictionMultiplier)); this.networkManager = config.networkManager ?? new NetworkManager(); - this.entityMap = new EntityStore((key, data, refs, merge) => { - if (merge) this.store.mergeEntity!(key, data, refs); + this.entityMap = new EntityStore((key, data, refs, merge, ifStored) => { + if (merge) this.store.mergeEntity!(key, data, refs, ifStored); else this.store.saveEntity(key, data, refs); }); this.entityMap.mergesEntities = typeof this.store.mergeEntity === 'function'; @@ -747,8 +747,8 @@ export class QueryClient { return; } - // Write a created root if a live array will retain it or the store already - // holds it. Before routing, so a failed write routes nothing. + // Write a created root if a live array will retain it, else only refresh a + // record the store may hold. Before routing, so a failed write routes nothing. let matched = false; let retains = false; this.routeEvent( @@ -767,9 +767,13 @@ export class QueryClient { let held: boolean | undefined; if (!entity._persisted && entity._pendingWrites === 0) { if (!retains) held = this.store.hasEntity?.(key); - if (retains || held === true) { + // Unknown: entities created under the root would be written even if the root isn't. + const refresh = + held === true || + (held === undefined && entity._partial && created!.size === 1 && this.entityMap.mergesEntities); + if (retains || refresh) { try { - entity.save(held); + entity.save(held, !retains); } catch (e) { this.context.log?.warn?.('Failed to apply mutation event', e); this.evictUnlessAdopted(entity, created!); diff --git a/packages/fetchium/src/__tests__/event-persistence.test.ts b/packages/fetchium/src/__tests__/event-persistence.test.ts index 824775a..eafeeaf 100644 --- a/packages/fetchium/src/__tests__/event-persistence.test.ts +++ b/packages/fetchium/src/__tests__/event-persistence.test.ts @@ -8,7 +8,7 @@ import { RESTQueryAdapter } from '../rest/RESTQueryAdapter.js'; import { fetchQuery } from '../query.js'; import { QueryClient, QueryClientContext } from '../QueryClient.js'; import { SyncQueryStore, MemoryPersistentStore } from '../stores/sync.js'; -import { AsyncQueryStore, type AsyncPersistentStore } from '../stores/async.js'; +import { AsyncQueryStore, type AsyncPersistentStore, type StoreMessage } from '../stores/async.js'; import { valueKeyFor, refCountKeyFor, refIdsKeyFor, queueKeyFor, updatedAtKeyFor } from '../stores/shared.js'; import { createMockFetch, setupTestClient, sleep } from './utils.js'; @@ -1214,7 +1214,7 @@ describe('AsyncQueryStore writer', () => { expect(asyncDoc(delegate, K('User', 'u404'))).toBeUndefined(); }); - it('does not write an event for an entity it cannot yet say it holds, and refreshes it once it can', async () => { + it('refreshes a record it holds from an event that lands before it has read its keys', async () => { const delegate = new AsyncDelegate(1); const mockFetch = createMockFetch(); mockFetch.get('/user/u1', { @@ -1246,28 +1246,96 @@ describe('AsyncQueryStore writer', () => { await holdQuery(client, () => fetchQuery(GetUser, { id: 'u2' })); expect(store.hasEntity!(K('User', 'u1'))).toBeUndefined(); expect(store.hasEntity!(K('User', 'u2'))).toBe(true); - const event = () => ({ - type: 'update' as const, - typename: 'User', - data: { __typename: 'User', id: 'u1', name: 'Alicia', karma: 10 }, - }); - client.applyMutationEvent(event()); + const update = (data: Record) => + client.applyMutationEvent({ type: 'update', typename: 'User', data: { __typename: 'User', karma: 10, ...data } }); + // Not written: the badge it creates would be, whether or not a record of u1 exists. + update({ id: 'u1', name: 'Al', badge: { __typename: 'Badge', id: 'b2', label: 'new' } }); + update({ id: 'u1', name: 'Alicia' }); + update({ id: 'u404', name: 'Nobody' }); expect(client.entityMap.getEntity(K('User', 'u1'))).toBeUndefined(); releaseScan(); await scanned(store); await drain(store); - // Not written: nothing referenced it and the store could not say it held it. - expect(asyncDoc(delegate, K('User', 'u1'))).toMatchObject({ name: 'Alice' }); - expect(store.hasEntity!(K('User', 'u1'))).toBe(true); - client.applyMutationEvent(event()); - await drain(store); expect(asyncDoc(delegate, K('User', 'u1'))).toMatchObject({ name: 'Alicia', karma: 10, badge: { __entityRef: K('Badge', 'b1') }, }); expect(asyncDoc(delegate, K('Badge', 'b1'))).toMatchObject({ label: 'gold' }); + expect(asyncDoc(delegate, K('User', 'u404'))).toBeUndefined(); + expect(asyncDoc(delegate, K('Badge', 'b2'))).toBeUndefined(); + expect(store.hasEntity!(K('User', 'u404'))).toBe(false); + }); + + it('a reader sends an event for an entity on disk but not in memory to the writer as a merge', async () => { + const delegate = new AsyncDelegate(); + let toWriter: (msg: StoreMessage) => void = () => {}; + const writer = new AsyncQueryStore({ + isWriter: true, + delegate, + connect: handle => ((toWriter = handle), { sendMessage: () => {} }), + }); + const reader = new AsyncQueryStore({ isWriter: false, connect: () => ({ sendMessage: msg => toWriter(msg) }) }); + const mockFetch = createMockFetch(); + mockFetch.get('/user/u1', { + user: { + __typename: 'User', + id: 'u1', + name: 'Alice', + karma: 10, + badge: { __typename: 'Badge', id: 'b1', label: 'gold' }, + }, + }); + const client = makeClient(reader, mockFetch); + await holdQuery(client, () => fetchQuery(GetUser, { id: 'u1' })); + await drain(writer); + client.entityMap.getEntity(K('User', 'u1'))!.evict(); + + const update = (id: string, name: string) => + client.applyMutationEvent({ type: 'update', typename: 'User', data: { __typename: 'User', id, name, karma: 3 } }); + update('u1', 'Alicia'); + update('u404', 'Nobody'); + await drain(writer); + + expect(asyncDoc(delegate, K('User', 'u1'))).toMatchObject({ + name: 'Alicia', + karma: 3, + badge: { __entityRef: K('Badge', 'b1') }, + }); + expect(asyncDoc(delegate, K('User', 'u404'))).toBeUndefined(); + }); + + it("a reader's write of an entity created under one in memory is written whole, so the reference resolves", async () => { + const delegate = new AsyncDelegate(); + let toWriter: (msg: StoreMessage) => void = () => {}; + const writer = new AsyncQueryStore({ + isWriter: true, + delegate, + connect: handle => ((toWriter = handle), { sendMessage: () => {} }), + }); + const reader = new AsyncQueryStore({ isWriter: false, connect: () => ({ sendMessage: msg => toWriter(msg) }) }); + const mockFetch = createMockFetch(); + mockFetch.get('/user/u1', { user: { __typename: 'User', id: 'u1', name: 'Alice', karma: 10 } }); + const client = makeClient(reader, mockFetch); + await holdQuery(client, () => fetchQuery(GetUser, { id: 'u1' })); + await drain(writer); + + client.applyMutationEvent({ + type: 'update', + typename: 'User', + data: { + __typename: 'User', + id: 'u1', + name: 'Alice', + karma: 10, + badge: { __typename: 'Badge', id: 'b9', label: 'x' }, + }, + }); + await drain(writer); + + expect(asyncDoc(delegate, K('User', 'u1'))).toMatchObject({ badge: { __entityRef: K('Badge', 'b9') } }); + expect(asyncDoc(delegate, K('Badge', 'b9'))).toEqual({ __typename: 'Badge', id: 'b9', label: 'x' }); }); it('ignores what it does not understand on the channel and keeps processing', async () => { diff --git a/packages/fetchium/src/__tests__/skip-write-and-snapshot-fixes.test.ts b/packages/fetchium/src/__tests__/skip-write-and-snapshot-fixes.test.ts index bbd14a3..ce9c86f 100644 --- a/packages/fetchium/src/__tests__/skip-write-and-snapshot-fixes.test.ts +++ b/packages/fetchium/src/__tests__/skip-write-and-snapshot-fixes.test.ts @@ -687,11 +687,16 @@ describe('write skipping stays consistent with the store', () => { class AsyncDelegate implements AsyncPersistentStore { readonly kv: Record = Object.create(null); failNextSetString: string | undefined; + failHas = 0; private async tick(): Promise { await new Promise(resolve => setTimeout(resolve, 0)); } async has(key: string) { await this.tick(); + if (this.failHas > 0) { + this.failHas--; + throw new Error('injected failure'); + } return key in this.kv; } async getString(key: string) { @@ -862,6 +867,112 @@ describe('AsyncQueryStore writer used in-process', () => { } }); + it('a rejected SaveEntity whose recheck also fails keeps the record held', async () => { + class Person extends Entity { + __typename = t.typename('Person'); + id = t.id; + name = t.string; + role = t.optional(t.string); + } + class GetPerson extends RESTQuery { + path = '/person'; + result = { person: t.entity(Person) }; + } + const key = hashValue(['Person', '1']); + const delegate = new AsyncDelegate(); + const store = writerStore(delegate); + const mockFetch = createMockFetch(); + mockFetch.get('/person', { person: { __typename: 'Person', id: '1', name: 'Alice', role: 'admin' } }); + const { client } = makeClient(store, mockFetch); + clients.push(client); + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {}); + const update = (name: string) => + client.applyMutationEvent({ type: 'update', typename: 'Person', data: { __typename: 'Person', id: '1', name } }); + try { + await holdQuery(client, () => fetchQuery(GetPerson)); + await drain(store); + client.entityMap.getEntity(key)!.evict(); + + delegate.failNextSetString = valueKeyFor(key); + delegate.failHas = 3; + update('Bob'); + await drain(store); + expect(store.hasEntity!(key)).toBe(true); + + update('Carol'); + await drain(store); + expect(JSON.parse(delegate.kv[valueKeyFor(key)] as string)).toMatchObject({ name: 'Carol', role: 'admin' }); + } finally { + consoleError.mockRestore(); + } + }); + + it('an event refresh of a record the writer wrongly holds writes nothing, and the entity is rewritten whole', async () => { + class Task extends Entity { + __typename = t.typename('Task'); + id = t.id; + boardId = t.string; + title = t.string; + done = t.optional(t.boolean); + } + class Board extends Entity { + __typename = t.typename('Board'); + id = t.id; + edits = t.liveValue(t.number, Task, { + constraints: { boardId: (this as any).id }, + onCreate: (n: number) => n, + onUpdate: (n: number) => n + 1, + onDelete: (n: number) => n, + }); + } + class GetBoard extends RESTQuery { + path = '/board'; + result = { board: t.entity(Board) }; + } + class GetTask extends RESTQuery { + path = '/task'; + result = { task: t.entity(Task) }; + } + const key = hashValue(['Task', 't1']); + const delegate = new AsyncDelegate(); + const store = writerStore(delegate); + const deleted: number[] = []; + store.onDelete!(k => deleted.push(k)); + const mockFetch = createMockFetch(); + mockFetch.get('/board', { board: { __typename: 'Board', id: 'b1', edits: 0 } }); + mockFetch.get('/task', { task: { __typename: 'Task', id: 't1', boardId: 'b1', title: 'A', done: false } }); + const { client } = makeClient(store, mockFetch); + clients.push(client); + const update = (title: string) => + client.applyMutationEvent({ + type: 'update', + typename: 'Task', + data: { __typename: 'Task', id: 't1', boardId: 'b1', title }, + }); + + await holdQuery(client, () => fetchQuery(GetBoard)); + await holdQuery(client, () => fetchQuery(GetTask)); + await drain(store); + // The record is lost behind the writer's back, so it still counts it held. + delete delegate.kv[valueKeyFor(key)]; + client.entityMap.getEntity(key)!.evict(); + + update('B'); + await drain(store); + expect(delegate.kv[valueKeyFor(key)]).toBeUndefined(); + expect(deleted).toContain(key); + expect(store.hasEntity!(key)).toBe(false); + + update('C'); + await drain(store); + expect(JSON.parse(delegate.kv[valueKeyFor(key)] as string)).toEqual({ + __typename: 'Task', + id: 't1', + boardId: 'b1', + title: 'C', + }); + }); + it('resizes a persisted LRU queue in both directions and defaults a missing maxCount', async () => { const delegate = new AsyncDelegate(); const store = writerStore(delegate); diff --git a/packages/fetchium/src/__tests__/store-on-delete.test.ts b/packages/fetchium/src/__tests__/store-on-delete.test.ts index fad9cf7..a6c2d6c 100644 --- a/packages/fetchium/src/__tests__/store-on-delete.test.ts +++ b/packages/fetchium/src/__tests__/store-on-delete.test.ts @@ -141,6 +141,94 @@ describe('QueryStore.onDelete', () => { await sleep(20); expect(deleted).toEqual([q1]); }); + + it('SyncQueryStore finishes a cascade and notifies every listener when one throws', () => { + const kv = new MemoryPersistentStore(); + const store = new SyncQueryStore(kv); + const deleted: number[] = []; + store.onDelete(() => { + throw new Error('listener failed'); + }); + store.onDelete(key => deleted.push(key)); + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {}); + try { + const q1 = hashValue(['GET:/t', { id: '1' }]); + const e1 = hashValue(['User', 1]); + store.saveEntity(e1, { id: 1 }); + store.saveQuery(queryDef('GET:/t', 10), q1, { __entityRef: e1 }, Date.now(), new Set([e1])); + store.deleteQuery(q1); + expect(deleted).toEqual([q1, e1]); + expect(kv.has(valueKeyFor(e1))).toBe(false); + expect(consoleError).toHaveBeenCalled(); + } finally { + consoleError.mockRestore(); + } + }); + + it('AsyncQueryStore writer finishes a cascade when a listener throws', async () => { + const delegate = new MockAsyncPersistentStore(); + const writer = new AsyncQueryStore({ + isWriter: true, + delegate, + connect: (_handle: (msg: StoreMessage) => void) => ({ sendMessage: () => {} }), + }); + const deleted: number[] = []; + writer.onDelete!(() => { + throw new Error('listener failed'); + }); + writer.onDelete!(key => deleted.push(key)); + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {}); + try { + const q1 = hashValue(['GET:/w', { id: '1' }]); + const e1 = hashValue(['User', 1]); + writer.saveEntity(e1, { id: 1 }); + writer.saveQuery(queryDef('GET:/w', 10), q1, { __entityRef: e1 }, Date.now(), new Set([e1])); + writer.deleteQuery(q1); + await sleep(20); + expect(deleted).toEqual([q1, e1]); + expect(await delegate.has(valueKeyFor(e1))).toBe(false); + } finally { + consoleError.mockRestore(); + } + }); + + it('AsyncQueryStore writer keeps processing when a listener throws over a failed write', async () => { + class FlakyStore extends MockAsyncPersistentStore { + failNext: string | undefined; + override async setString(key: string, value: string) { + if (key === this.failNext) { + this.failNext = undefined; + throw new Error('injected failure'); + } + return super.setString(key, value); + } + } + const delegate = new FlakyStore(); + const writer = new AsyncQueryStore({ + isWriter: true, + delegate, + connect: (_handle: (msg: StoreMessage) => void) => ({ sendMessage: () => {} }), + }); + const deleted: number[] = []; + writer.onDelete!(() => { + throw new Error('listener failed'); + }); + writer.onDelete!(key => deleted.push(key)); + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {}); + try { + const e1 = hashValue(['User', 1]); + const e2 = hashValue(['User', 2]); + delegate.failNext = valueKeyFor(e1); + writer.saveEntity(e1, { id: 1 }); + writer.saveEntity(e2, { id: 2 }); + await sleep(20); + expect(deleted).toEqual([e1]); + expect(await delegate.has(valueKeyFor(e2))).toBe(true); + expect(writer.isSettled()).toBe(true); + } finally { + consoleError.mockRestore(); + } + }); }); describe('stores without onDelete', () => { diff --git a/packages/fetchium/src/query-types.ts b/packages/fetchium/src/query-types.ts index c0756d2..5b70c45 100644 --- a/packages/fetchium/src/query-types.ts +++ b/packages/fetchium/src/query-types.ts @@ -103,8 +103,10 @@ export interface QueryStore { * Writes the fields streamed events supplied, merged over the stored record * if one exists (references derived from the merged value). Without it the * client calls `saveEntity`, dropping fields the events did not carry. + * With `ifStored`, nothing is written when no record is stored, and a store + * with `onDelete` reports the key as dropped. */ - mergeEntity?(entityKey: number, fields: unknown, refIds?: Set): void; + mergeEntity?(entityKey: number, fields: unknown, refIds?: Set, ifStored?: boolean): void; activateQuery(queryDef: QueryDefinition, storageKey: number): void; @@ -141,8 +143,8 @@ export interface QueryStore { /** * Whether the store holds a record for this key, or `undefined` if it can't - * tell yet. An event for an entity not in memory is written only when this - * returns `true`. + * tell yet. An event for an entity not in memory is written as an `ifStored` + * merge unless this returns `false`. Without `mergeEntity`, only when `true`. */ hasEntity?(key: number): boolean | undefined; } diff --git a/packages/fetchium/src/stores/async.ts b/packages/fetchium/src/stores/async.ts index c512f4d..2c6518c 100644 --- a/packages/fetchium/src/stores/async.ts +++ b/packages/fetchium/src/stores/async.ts @@ -7,6 +7,7 @@ import { LAST_USED_PREFIX, lastUsedKeyFor, mergeStoredRecord, + notifyListeners, queueKeyFor, refCountKeyFor, refIdsKeyFor, @@ -62,6 +63,8 @@ export type StoreMessage = refIds?: number[]; /** `value` holds only some fields, merged over the stored record. */ merge?: boolean; + /** With `merge`: dropped, and reported deleted, if no record is stored. */ + ifStored?: boolean; } | { type: StoreMessageType.ActivateQuery; queryDefId: string; queryKey: number; cacheTime: number; maxCount?: number } | { type: StoreMessageType.DeleteQuery; queryKey: number }; @@ -96,7 +99,6 @@ function isStoreMessage(msg: unknown): msg is StoreMessage { ); } -/** Callers notify over a copy, since a listener may unsubscribe while being notified. */ function subscribe(listeners: Array<(key: number) => void>, listener: (key: number) => void): () => void { listeners.push(listener); return () => { @@ -215,16 +217,22 @@ export class AsyncQueryStore implements QueryStore { } } - /** The previous record may have survived a failed write, so ask the delegate. */ + /** The previous record may have survived a failed write. Held unless the delegate says otherwise. */ private async recheckHeld(id: number): Promise { - let held = false; - try { - held = await this.delegate!.has(valueKeyFor(id)); - } catch { - // Treat as missing. + for (let attempt = 0; ; attempt++) { + try { + if (await this.delegate!.has(valueKeyFor(id))) this.noteHeld(id); + else this.noteDropped(id); + return; + } catch (error) { + if (attempt >= 2) { + console.error('Could not check whether the store holds a record:', error); + this.noteHeld(id); + return; + } + await new Promise(resolve => setTimeout(resolve, 50 << attempt)); + } } - if (held) this.noteHeld(id); - else this.noteDropped(id); } private startQueueProcessor(): void { @@ -260,7 +268,7 @@ export class AsyncQueryStore implements QueryStore { // Report as dropped so the client rewrites it on its next apply. if (!(msg instanceof InternalWork) && msg.type === StoreMessageType.SaveEntity) { await this.recheckHeld(msg.entityKey); - for (const listener of this.deleteListeners.slice()) listener(msg.entityKey); + notifyListeners(this.deleteListeners, msg.entityKey); } } finally { this.processing = false; @@ -313,8 +321,11 @@ export class AsyncQueryStore implements QueryStore { ); break; case StoreMessageType.SaveEntity: - await this.writerSaveEntity(msg.entityKey, msg.value, msg.refIds, msg.merge === true); - for (const listener of this.persistedListeners.slice()) listener(msg.entityKey); + if ( + await this.writerSaveEntity(msg.entityKey, msg.value, msg.refIds, msg.merge === true, msg.ifStored === true) + ) { + notifyListeners(this.persistedListeners, msg.entityKey); + } break; case StoreMessageType.ActivateQuery: await this.writerActivateQuery(msg.queryDefId, msg.queryKey, msg.cacheTime, msg.maxCount); @@ -417,13 +428,14 @@ export class AsyncQueryStore implements QueryStore { }); } - mergeEntity(entityKey: number, fields: unknown, refIds?: Set): void { + mergeEntity(entityKey: number, fields: unknown, refIds?: Set, ifStored?: boolean): void { this.dispatch({ type: StoreMessageType.SaveEntity, entityKey, value: fields, refIds: refIds ? Array.from(refIds) : undefined, merge: true, + ifStored, }); } @@ -469,16 +481,23 @@ export class AsyncQueryStore implements QueryStore { value: unknown, refIds: number[] | undefined, merge: boolean, - ): Promise { + ifStored: boolean, + ): Promise { if (merge) { const stored = await this.delegate!.getString(valueKeyFor(entityKey)); + if (stored === undefined && ifStored) { + this.noteDropped(entityKey); + notifyListeners(this.deleteListeners, entityKey); + return false; + } const merged = stored !== undefined ? mergeStoredRecord(stored, value) : undefined; if (merged !== undefined) { await this.setValue(entityKey, merged.value, merged.refIds); - return; + return true; } } await this.setValue(entityKey, value, refIds ? new Set(refIds) : undefined); + return true; } private async writerActivateQuery( @@ -640,7 +659,7 @@ export class AsyncQueryStore implements QueryStore { await delegate.delete(valueKeyFor(id)); this.noteDropped(id); await delegate.delete(refCountKeyFor(id)); - for (const listener of this.deleteListeners.slice()) listener(id); + notifyListeners(this.deleteListeners, id); const refIds = await delegate.getBuffer(refIdsKey); await delegate.delete(refIdsKey); // Clean up the refIds key diff --git a/packages/fetchium/src/stores/shared.ts b/packages/fetchium/src/stores/shared.ts index 0fff5c8..93f6fcf 100644 --- a/packages/fetchium/src/stores/shared.ts +++ b/packages/fetchium/src/stores/shared.ts @@ -51,3 +51,14 @@ function collectEntityRefs(value: unknown, into: Set): void { } for (const key in value) collectEntityRefs((value as Record)[key], into); } + +/** Notifies over a copy, since a listener may unsubscribe while being notified. */ +export function notifyListeners(listeners: Array<(key: number) => void>, key: number): void { + for (const listener of listeners.slice()) { + try { + listener(key); + } catch (error) { + console.error('A store listener threw:', error); + } + } +} diff --git a/packages/fetchium/src/stores/sync.ts b/packages/fetchium/src/stores/sync.ts index 801286f..f2df858 100644 --- a/packages/fetchium/src/stores/sync.ts +++ b/packages/fetchium/src/stores/sync.ts @@ -12,6 +12,7 @@ import { updatedAtKeyFor, valueKeyFor, mergeStoredRecord, + notifyListeners, } from './shared.js'; export interface SyncPersistentStore { @@ -319,8 +320,7 @@ export class SyncQueryStore implements QueryStore { kv.delete(valueKeyFor(id)); kv.delete(refCountKeyFor(id)); - // A listener may unsubscribe while being notified. - for (const listener of this.deleteListeners.slice()) listener(id); + notifyListeners(this.deleteListeners, id); const refIds = kv.getBuffer(refIdsKeyFor(id)); kv.delete(refIdsKeyFor(id)); // Clean up the refIds key