From b2963ef3d55c9ea45cd2172966b9cb981ec3ef60 Mon Sep 17 00:00:00 2001 From: lizhixuan Date: Mon, 26 Oct 2026 23:31:08 +0800 Subject: [PATCH 1/3] fix(provider-polyhaven): rank by match quality and popularity; cache the asset list --- .changeset/polyhaven-relevance.md | 7 + packages/provider-polyhaven/README.md | 12 + .../src/__tests__/polyhaven.test.ts | 218 +++++++++++++++++- packages/provider-polyhaven/src/index.ts | 122 +++++++--- 4 files changed, 330 insertions(+), 29 deletions(-) create mode 100644 .changeset/polyhaven-relevance.md diff --git a/.changeset/polyhaven-relevance.md b/.changeset/polyhaven-relevance.md new file mode 100644 index 0000000..abc6859 --- /dev/null +++ b/.changeset/polyhaven-relevance.md @@ -0,0 +1,7 @@ +--- +'@refkit/provider-polyhaven': patch +--- + +Poly Haven results are ranked by how well they match the query, then by popularity. A query word found as a word of an asset's name or id counts most, then a tag or category, then a match inside a longer word; ties go to the most-downloaded asset. Before, most matches tied and kept Poly Haven's alphabetical order, so "sunset HDRI" returned `aarfontein_dawn_2`, `aarfontein_dusk` and the like instead of the popular sunset skies. A query with no words now lists the most-downloaded assets first. + +The asset list (about 1 MB per asset type) is downloaded once and reused for an hour by each provider instance, instead of on every search. Change that with the new `listTtlMs` option; `0` turns the cache off. diff --git a/packages/provider-polyhaven/README.md b/packages/provider-polyhaven/README.md index d9fe771..b4537c7 100644 --- a/packages/provider-polyhaven/README.md +++ b/packages/provider-polyhaven/README.md @@ -30,6 +30,18 @@ const refkit = createRefkit({ providers: [polyhaven(), polyhaven({ assetType: 'h const hdris = await refkit.search({ query: 'sunset', modalities: ['image'], sources: ['polyhaven-hdri'] }) ``` +### Ranking + +Poly Haven's list endpoint has no search parameter, so `polyhaven()` downloads the whole list and ranks it itself. Each query word adds the score of its best match in an asset: 3 if it is a word of the asset's name or id, 2 if it is a tag or category or a word of one (`sunset` in `sunrise-sunset`), 1 if it only appears inside one of those fields. Assets that match no word are dropped. Equal scores go to the most-downloaded asset first. A query with no words returns every asset, most-downloaded first. `page` steps through this ranked list in windows of `maxAssets`. + +### Asset list cache + +The list is about 1 MB, so each provider instance keeps the copy it downloaded for `listTtlMs` milliseconds (default `3_600_000`, one hour) and ranks later searches against it. Searches that run at the same time share one download, and a failed download is not kept. Set `listTtlMs: 0` to download the list on every search. + +```ts +polyhaven({ assetType: 'hdris', listTtlMs: 10 * 60_000 }) // re-download the HDRI list every 10 minutes +``` + ### ambientCG sibling factory `ambientcg()` lives in the same package and returns the same CC0-normalized image references, hitting ambientCG's API instead: diff --git a/packages/provider-polyhaven/src/__tests__/polyhaven.test.ts b/packages/provider-polyhaven/src/__tests__/polyhaven.test.ts index 3721286..0fe500c 100644 --- a/packages/provider-polyhaven/src/__tests__/polyhaven.test.ts +++ b/packages/provider-polyhaven/src/__tests__/polyhaven.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from 'vitest' +import { afterEach, describe, expect, it, vi } from 'vitest' import { createRefkit, evaluateUse, type ProviderContext } from '@refkit/core' import { polyhaven } from '../index' @@ -93,3 +93,219 @@ describe('polyhaven provider', () => { expect(refkit.providers.map(p => p.id)).toEqual(['polyhaven', 'polyhaven-hdri']) }) }) + +// — relevance ranking — + +// HDRI fixtures: /files/ answers every id with a tonemapped preview, so the +// ranking alone decides what comes back, in what order. +const hdriCtx = (list: unknown): ProviderContext => ({ + fetch: (async (input: string) => { + const u = String(input) + if (u.includes('/assets')) return new Response(JSON.stringify(list), { status: 200 }) + const m = u.match(/\/files\/([^/?]+)/) + return new Response(JSON.stringify({ tonemapped: { url: `https://dl.polyhaven.org/${m?.[1]}.jpg` } }), { status: 200 }) + }) as typeof fetch, +}) +const ids = (refs: { sourceUrl: string }[]) => refs.map(r => r.sourceUrl.replace('https://polyhaven.com/a/', '')) +const hdriSearch = (list: unknown, text: string, extra: object = {}, config: object = {}) => + polyhaven({ assetType: 'hdris', ...config }).search({ text, modalities: ['image'], ...extra }, hdriCtx(list)) + +describe('polyhaven relevance ranking', () => { + // The list endpoint serves assets alphabetically by id; every fixture puts the + // less relevant asset first so list order and relevance disagree. + it('ranks an exact tag or category word above a substring-only match ("sunset HDRI")', async () => { + const list = { + amber_sunsets: { type: 0, name: 'Amber Sunsets', categories: ['outdoor'], download_count: 900_000 }, // "sunset" only inside "sunsets" + brick_studio: { type: 0, name: 'Brick Studio', categories: ['indoor'] }, // matches nothing + zwartkops_evening: { type: 0, name: 'Zwartkops Evening', categories: ['outdoor', 'sunrise-sunset'], download_count: 10 }, + } + // "hdri" matches no asset and adds nothing; a non-matching asset stays out. + expect(ids(await hdriSearch(list, 'sunset HDRI'))).toEqual(['zwartkops_evening', 'amber_sunsets']) + }) + + it('ranks a word of the name or id above a tag or category word, whatever the downloads', async () => { + const list = { + alpine_meadow: { type: 0, name: 'Alpine Meadow', categories: ['sunrise-sunset'], tags: ['sunset'], download_count: 800_000 }, + sunset_jhbcentral: { type: 0, name: 'JHB Central', categories: ['urban'], download_count: 1 }, // word of the id only + venice_sunset: { type: 0, name: 'Venice Sunset', categories: ['urban'], download_count: 5 }, + } + expect(ids(await hdriSearch(list, 'sunset'))).toEqual(['venice_sunset', 'sunset_jhbcentral', 'alpine_meadow']) + }) + + it('counts each token once, at its best match, not once per field it appears in', async () => { + const list = { + // "sunset" is in the name, a category word and a tag: still worth 3, not 3+2+2 + aa_sunset_peak: { type: 0, name: 'Sunset Peak', categories: ['sunrise-sunset'], tags: ['sunset'], download_count: 10 }, + bb_sunset_bay: { type: 0, name: 'Sunset Bay', download_count: 20 }, + } + expect(ids(await hdriSearch(list, 'sunset'))).toEqual(['bb_sunset_bay', 'aa_sunset_peak']) + }) + + it('breaks equal scores by download_count (missing counts as 0), then by list order', async () => { + const list = { + aarfontein_dawn_2: { type: 0, name: 'Aarfontein Dawn 2', categories: ['sunrise-sunset'] }, // no download_count + lilienstein: { type: 0, name: 'Lilienstein', categories: ['sunrise-sunset'], download_count: 769_081 }, + spruit_sunrise: { type: 0, name: 'Spruit Sunrise', categories: ['sunrise-sunset'], download_count: 551_025 }, + the_sky_is_on_fire: { type: 0, name: 'The Sky Is On Fire', categories: ['sunrise-sunset'], download_count: 551_025 }, + } + expect(ids(await hdriSearch(list, 'sunset HDRI'))).toEqual(['lilienstein', 'spruit_sunrise', 'the_sky_is_on_fire', 'aarfontein_dawn_2']) + }) + + it('windows pages over the ranked list, not the list order', async () => { + // a..e listed in that order, popularity rising: ranked order is e, d, c, b, a + const list = Object.fromEntries(['a', 'b', 'c', 'd', 'e'].map((id, i) => + [`${id}_dusk`, { type: 0, name: `${id.toUpperCase()} Dusk`, categories: ['sunrise-sunset'], download_count: (i + 1) * 100 }])) + const page = (n: number) => hdriSearch(list, 'sunset', { controls: { page: n } }, { maxAssets: 2 }) + expect(ids(await page(1))).toEqual(['e_dusk', 'd_dusk']) + expect(ids(await page(2))).toEqual(['c_dusk', 'b_dusk']) + expect(ids(await page(3))).toEqual(['a_dusk']) + }) + + it('keeps every asset for a query with no tokens, most-downloaded first', async () => { + const list = { + a_room: { type: 0, name: 'A Room', download_count: 5 }, + b_room: { type: 0, name: 'B Room' }, // no download_count + c_room: { type: 0, name: 'C Room', download_count: 50 }, + d_room: { type: 0, name: 'D Room', download_count: 5 }, + } + for (const text of ['', ' ?! ']) { + expect(ids(await hdriSearch(list, text))).toEqual(['c_room', 'a_room', 'd_room', 'b_room']) + } + }) +}) + +// — asset list cache — + +interface Deferred { promise: Promise; resolve: (v: T) => void } +function deferred(): Deferred { + let resolve!: (v: T) => void + const promise = new Promise((r) => { resolve = r }) + return { promise, resolve } +} +const listResponse = () => new Response(JSON.stringify(LIST), { status: 200 }) + +/** A Poly Haven stand-in that counts /assets calls. `replies[i]` scripts the i-th + * list response (default: LIST). Like the real fetch, a request rejects with the + * signal's reason when its signal aborts. */ +function listServer(replies: Array<() => Response | Promise> = []) { + let listCalls = 0 + const fetchImpl = (async (input: string, init?: RequestInit) => { + const u = String(input) + let reply: Promise + if (u.includes('/assets')) { + const scripted = replies[listCalls++] + reply = Promise.resolve().then(scripted ?? listResponse) + } else { + const m = u.match(/\/files\/([^/?]+)/) + const files = m ? (FILES_TEX as Record)[m[1]] : undefined + reply = Promise.resolve(files ? new Response(JSON.stringify(files), { status: 200 }) : new Response('null', { status: 404 })) + } + const signal = init?.signal + if (!signal) return reply + return new Promise((resolve, reject) => { + if (signal.aborted) return reject(signal.reason) + signal.addEventListener('abort', () => reject(signal.reason), { once: true }) + reply.then(resolve, reject) + }) + }) as typeof fetch + return { + ctx: (signal?: AbortSignal): ProviderContext => ({ fetch: fetchImpl, ...(signal ? { signal } : {}) }), + listCalls: () => listCalls, + } +} +const q = (text: string) => ({ text, modalities: ['image' as const] }) + +describe('polyhaven asset list cache', () => { + afterEach(() => { vi.useRealTimers() }) + + it('downloads the list once for two searches on the same provider instance', async () => { + const server = listServer() + const p = polyhaven() + expect(ids(await p.search(q('forest rock'), server.ctx()))).toEqual(['mossy_forest_rock', 'forest_floor']) + expect(ids(await p.search(q('asphalt'), server.ctx()))).toEqual(['aerial_asphalt_01']) + expect(server.listCalls()).toBe(1) + // the cache belongs to the instance: another instance downloads its own list + await polyhaven().search(q('asphalt'), server.ctx()) + expect(server.listCalls()).toBe(2) + }) + + it('refetches the list once the default one-hour TTL has passed', async () => { + vi.useFakeTimers({ toFake: ['Date'] }) + const t0 = new Date('2026-10-26T00:00:00Z').getTime() + vi.setSystemTime(t0) + const server = listServer() + const p = polyhaven() + await p.search(q('forest'), server.ctx()) + vi.setSystemTime(t0 + 3_600_000 - 1) + await p.search(q('forest'), server.ctx()) + expect(server.listCalls()).toBe(1) + vi.setSystemTime(t0 + 3_600_000) + await p.search(q('forest'), server.ctx()) + expect(server.listCalls()).toBe(2) + await p.search(q('forest'), server.ctx()) // the fresh download is cached in turn + expect(server.listCalls()).toBe(2) + }) + + it('honours a custom listTtlMs', async () => { + vi.useFakeTimers({ toFake: ['Date'] }) + vi.setSystemTime(0) + const server = listServer() + const p = polyhaven({ listTtlMs: 5_000 }) + await p.search(q('forest'), server.ctx()) + vi.setSystemTime(4_999) + await p.search(q('forest'), server.ctx()) + expect(server.listCalls()).toBe(1) + vi.setSystemTime(5_000) + await p.search(q('forest'), server.ctx()) + expect(server.listCalls()).toBe(2) + }) + + it('does not cache a failed download: a network error or a non-OK status is retried by the next search', async () => { + const server = listServer([ + () => { throw new TypeError('fetch failed') }, + () => new Response('busy', { status: 503 }), + ]) + const p = polyhaven() + await expect(p.search(q('forest'), server.ctx())).rejects.toThrow('fetch failed') + await expect(p.search(q('forest'), server.ctx())).rejects.toThrow('polyhaven list failed: 503') + expect(ids(await p.search(q('forest'), server.ctx()))).toEqual(['forest_floor', 'mossy_forest_rock']) + await p.search(q('rock'), server.ctx()) // the successful download is the one cached + expect(server.listCalls()).toBe(3) + }) + + it('fetches the list on every search with listTtlMs: 0', async () => { + const server = listServer() + const p = polyhaven({ listTtlMs: 0 }) + await p.search(q('forest'), server.ctx()) + await p.search(q('forest'), server.ctx()) + expect(server.listCalls()).toBe(2) + }) + + it('shares one in-flight download between concurrent searches', async () => { + const gate = deferred() + const server = listServer([() => gate.promise]) + const p = polyhaven() + const searches = Promise.all([p.search(q('forest'), server.ctx()), p.search(q('asphalt'), server.ctx())]) + gate.resolve(listResponse()) + const [forest, asphalt] = await searches + expect(ids(forest)).toEqual(['forest_floor', 'mossy_forest_rock']) + expect(ids(asphalt)).toEqual(['aerial_asphalt_01']) + expect(server.listCalls()).toBe(1) + }) + + it("rejects an aborted search at once without failing the shared download for anyone else", async () => { + const gate = deferred() + const server = listServer([() => gate.promise]) + const p = polyhaven() + const ctrl = new AbortController() + const aborted = p.search(q('forest'), server.ctx(ctrl.signal)) + const other = p.search(q('asphalt'), server.ctx()) + ctrl.abort(new Error('caller gave up')) + await expect(aborted).rejects.toThrow('caller gave up') // while the list is still downloading + gate.resolve(listResponse()) + expect(ids(await other)).toEqual(['aerial_asphalt_01']) + // the download the aborted search started is the one later searches reuse + expect(ids(await p.search(q('rock'), server.ctx()))).toEqual(['mossy_forest_rock']) + expect(server.listCalls()).toBe(1) + }) +}) diff --git a/packages/provider-polyhaven/src/index.ts b/packages/provider-polyhaven/src/index.ts index d2077dd..c8c2498 100644 --- a/packages/provider-polyhaven/src/index.ts +++ b/packages/provider-polyhaven/src/index.ts @@ -14,6 +14,12 @@ export interface PolyHavenConfig { assetType?: 'textures' | 'hdris' /** Max assets resolved per search; each costs one /files/ call (N+1). Default 12. */ maxAssets?: number + /** How long, in ms, this provider instance reuses the asset list it downloaded. + * Poly Haven's list endpoint has no query or paging, so every search ranks the + * whole list (about 1 MB); caching it saves that download on later searches. + * Concurrent searches share one in-flight download, and a failed download is + * not cached. Default 3_600_000 (one hour); 0 disables the cache. */ + listTtlMs?: number } interface PolyHavenAsset { @@ -23,13 +29,68 @@ interface PolyHavenAsset { tags?: string[] authors?: Record thumbnail_url?: string + download_count?: number } type PolyHavenList = Record + // /files tree: maps/resolutions/formats → { url }. Loosely typed; we walk known image paths only. type PolyHavenFiles = Record interface PhFileLeaf { url?: string } +const DEFAULT_LIST_TTL_MS = 3_600_000 + +/** Lowercase words of a string, split on anything that isn't alphanumeric — the + * rule for both the query tokens and the asset fields they are matched against. + * Splitting on punctuation, not just whitespace, keeps "forest, rock." from + * gluing a comma onto a token. */ +const words = (s: string): string[] => s.toLowerCase().split(/[^a-z0-9]+/).filter(Boolean) + +/** How well an asset matches the query: each token adds its best match only — + * 3 for a word of the name or id, 2 for a tag or category or a word of one + * ("sunset" in `sunrise-sunset`), 1 for a substring of any of those fields, + * 0 for none. A token that matches nothing (e.g. "hdri") adds 0. */ +function relevance(id: string, asset: PolyHavenAsset, tokens: string[]): number { + const labels = [...(asset.categories ?? []), ...(asset.tags ?? [])] + const nameWords = new Set([...words(id), ...words(asset.name ?? '')]) + // A token has no separators, so it can equal a whole label only when the label + // is one word: the label's own words cover both cases. + const labelWords = new Set(labels.flatMap(words)) + const fields = [id, asset.name ?? '', ...labels].map(f => f.toLowerCase()) + let score = 0 + for (const t of tokens) { + score += nameWords.has(t) ? 3 : labelWords.has(t) ? 2 : fields.some(f => f.includes(t)) ? 1 : 0 + } + return score +} + +/** Rank the list for a query — the list endpoint has no query param, so this is + * the search. Keeps assets that match at least one token and orders them by + * score, then download_count, then the list's own (alphabetical) order. A query + * with no tokens keeps every asset, so the same order reduces to most-downloaded + * first: the alphabetical order means nothing, and it would show the same + * `aarfontein_*` assets to every query-less search. */ +function rankAssets(list: PolyHavenList, tokens: string[]): [string, PolyHavenAsset][] { + const downloads = ({ download_count: d }: PolyHavenAsset) => (typeof d === 'number' && Number.isFinite(d) ? d : 0) + return Object.entries(list) + .map(([id, asset]) => ({ id, asset, score: relevance(id, asset, tokens) })) + .filter(e => tokens.length === 0 || e.score > 0) + // stable sort: equal score and downloads keep the list's own order + .sort((x, y) => y.score - x.score || downloads(y.asset) - downloads(x.asset)) + .map(e => [e.id, e.asset]) +} + +/** Settle with `p`, or reject with the signal's reason as soon as it aborts. */ +function raceAbort(p: Promise, signal: AbortSignal | undefined): Promise { + if (!signal) return p + return new Promise((resolve, reject) => { + const onAbort = () => reject(signal.reason ?? new Error('aborted')) + if (signal.aborted) return onAbort() + signal.addEventListener('abort', onAbort, { once: true }) + p.then(resolve, reject).finally(() => signal.removeEventListener('abort', onAbort)) + }) +} + /** First image URL for a texture: Diffuse (then a couple of fallbacks) → smallest res → jpg/png. */ function textureImageUrl(files: PolyHavenFiles): string | undefined { for (const mapKey of ['Diffuse', 'diff', 'Color', 'albedo']) { @@ -82,6 +143,32 @@ function toReference(id: string, asset: PolyHavenAsset, imageUrl: string, kind: export function polyhaven(config: PolyHavenConfig = {}) { const assetType = config.assetType ?? 'textures' + const listTtlMs = config.listTtlMs ?? DEFAULT_LIST_TTL_MS + const listUrl = new URL(`${PH_BASE}/assets`) + listUrl.searchParams.set('t', assetType) + const downloadList = (ctx: ProviderContext, signal?: AbortSignal) => + ctx.fetch(listUrl.toString(), { signal }).then(res => okJson(res, 'polyhaven list')) + + // The cached list: an in-flight download until `fetchedAt` is set, fresh for + // listTtlMs after that. + let cached: { list: Promise; fetchedAt?: number } | undefined + const assetList = (ctx: ProviderContext): Promise => { + if (!(listTtlMs > 0)) return downloadList(ctx, ctx.signal) + if (!cached || (cached.fetchedAt !== undefined && Date.now() - cached.fetchedAt >= listTtlMs)) { + // Download WITHOUT this search's signal and race the signal per caller + // instead: an aborted search rejects at once but cannot fail the download + // other searches share, and a download that outlives the deadline of the + // search that started it still lands in the cache for the next search. + const entry: NonNullable = { list: downloadList(ctx) } + cached = entry + entry.list.then( + () => { entry.fetchedAt = Date.now() }, + () => { if (cached === entry) cached = undefined }, // a failed download is not cached + ) + } + return raceAbort(cached.list, ctx.signal) + } + return defineProvider({ id: assetType === 'hdris' ? 'polyhaven-hdri' : 'polyhaven', modalities: ['image'], @@ -91,35 +178,14 @@ export function polyhaven(config: PolyHavenConfig = {}) { : 'CC0 PBR textures for 3D work (Poly Haven)', capabilities: { controls: ['page'] }, async search(q: NormalizedQuery, ctx: ProviderContext): Promise { - const listUrl = new URL(`${PH_BASE}/assets`) - listUrl.searchParams.set('t', assetType) - const res = await ctx.fetch(listUrl.toString(), { signal: ctx.signal }) - const list = await okJson(res, 'polyhaven list') - let entries = Object.entries(list) - // Client-side keyword filter — the list endpoint has no query param. Match - // PER TOKEN, not on the whole phrase: asset metadata is single-word (id, - // name, categories, tags), so "forest rock" as one substring matches - // nothing while both of its words describe real assets. Assets covering - // more of the query rank first. Split on anything that isn't alphanumeric - // (not just whitespace) so punctuation from a natural-language query - // ("forest, rock.") doesn't glue onto a token and prevent it matching. - const tokens = (q.text ?? '').toLowerCase().split(/[^a-z0-9]+/).filter(Boolean) - const fields = (id: string, a: PolyHavenAsset) => [id, a.name ?? '', ...(a.categories ?? []), ...(a.tags ?? [])].map(s => s.toLowerCase()) - if (tokens.length > 0) { - entries = entries - // `fields` is built once per asset, not once per (asset, token) pair — - // the full list is thousands of assets wide. - .map(([id, a]) => { - const haystack = fields(id, a) - return { id, a, hits: tokens.filter(t => haystack.some(f => f.includes(t))).length } - }) - .filter(e => e.hits > 0) - // stable sort — assets with equally many hits keep the list's own order - .sort((x, y) => y.hits - x.hits) - .map(e => [e.id, e.a] as [string, PolyHavenAsset]) - } + // Client-side ranking — the list endpoint has no query param. Match PER + // TOKEN, not on the whole phrase: asset metadata is single-word (id, name, + // categories, tags), so "forest rock" as one substring matches nothing + // while both of its words describe real assets. The ranking must happen + // here, before the maxAssets window: core reranks only what we return. + const entries = rankAssets(await assetList(ctx), words(q.text ?? '')) const n = Math.min(config.maxAssets ?? q.limit ?? 12, 30) - // the list endpoint returns everything — page = a window over the filtered list + // the list endpoint returns everything — page = a window over the ranked list const from = offsetForPage(q.controls?.page, n) ?? 0 const picked = entries.slice(from, from + n) const refs = await Promise.all(picked.map(async ([id, asset]) => { From f8f9b5b23f26ef92c6fe8f64773165e2123e731f Mon Sep 17 00:00:00 2001 From: lizhixuan Date: Mon, 26 Oct 2026 23:36:56 +0800 Subject: [PATCH 2/3] fix(provider-polyhaven): time out the shared list download; copy raw; document listTtlMs bounds --- packages/provider-polyhaven/README.md | 2 +- .../src/__tests__/polyhaven.test.ts | 52 +++++++++++++++++++ packages/provider-polyhaven/src/index.ts | 37 +++++++++++-- 3 files changed, 86 insertions(+), 5 deletions(-) diff --git a/packages/provider-polyhaven/README.md b/packages/provider-polyhaven/README.md index b4537c7..9f3bbbf 100644 --- a/packages/provider-polyhaven/README.md +++ b/packages/provider-polyhaven/README.md @@ -36,7 +36,7 @@ Poly Haven's list endpoint has no search parameter, so `polyhaven()` downloads t ### Asset list cache -The list is about 1 MB, so each provider instance keeps the copy it downloaded for `listTtlMs` milliseconds (default `3_600_000`, one hour) and ranks later searches against it. Searches that run at the same time share one download, and a failed download is not kept. Set `listTtlMs: 0` to download the list on every search. +The list is about 1 MB, so each provider instance keeps the copy it downloaded for `listTtlMs` milliseconds (default `3_600_000`, one hour) and ranks later searches against it. Searches that run at the same time share one download. A failed download is not kept, and neither is one still unfinished after 30 seconds: its searches fail and the next search starts a new download. Set `listTtlMs` to `0`, a negative number or `NaN` to download the list on every search; `Infinity` keeps the list for as long as the provider instance lives. ```ts polyhaven({ assetType: 'hdris', listTtlMs: 10 * 60_000 }) // re-download the HDRI list every 10 minutes diff --git a/packages/provider-polyhaven/src/__tests__/polyhaven.test.ts b/packages/provider-polyhaven/src/__tests__/polyhaven.test.ts index 0fe500c..0139fb3 100644 --- a/packages/provider-polyhaven/src/__tests__/polyhaven.test.ts +++ b/packages/provider-polyhaven/src/__tests__/polyhaven.test.ts @@ -281,6 +281,58 @@ describe('polyhaven asset list cache', () => { expect(server.listCalls()).toBe(2) }) + it('turns the cache off for a negative or NaN listTtlMs, like 0', async () => { + for (const listTtlMs of [-1, Number.NaN]) { + const server = listServer() + const p = polyhaven({ listTtlMs }) + await p.search(q('forest'), server.ctx()) + await p.search(q('forest'), server.ctx()) + expect(server.listCalls()).toBe(2) + } + }) + + it('keeps the list for the instance lifetime with listTtlMs: Infinity', async () => { + vi.useFakeTimers({ toFake: ['Date'] }) + vi.setSystemTime(0) + const server = listServer() + const p = polyhaven({ listTtlMs: Infinity }) + await p.search(q('forest'), server.ctx()) + vi.setSystemTime(365 * 24 * 3_600_000) + await p.search(q('forest'), server.ctx()) + expect(server.listCalls()).toBe(1) + }) + + it('times out a stalled shared download at 30 s: its searches reject and the next search downloads afresh', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] }) + const server = listServer([() => new Promise(() => {})]) // the first list request never answers + const p = polyhaven() + const stalled = [p.search(q('forest'), server.ctx()), p.search(q('rock'), server.ctx())] + let settled = false + void Promise.allSettled(stalled).then(() => { settled = true }) + const flush = () => new Promise(r => setImmediate(r)) // setImmediate is not faked: drains pending promise work + await vi.advanceTimersByTimeAsync(29_999) + await flush() + expect(settled).toBe(false) + await vi.advanceTimersByTimeAsync(1) + await flush() + expect(settled).toBe(true) + for (const s of stalled) await expect(s).rejects.toThrow('polyhaven list timed out after 30000ms') + expect(ids(await p.search(q('forest'), server.ctx()))).toEqual(['forest_floor', 'mossy_forest_rock']) + expect(server.listCalls()).toBe(2) + expect(vi.getTimerCount()).toBe(0) // a settled download leaves no timer behind to hold the process open + }) + + it('emits a copy of the cached asset as raw, so mutating it cannot change the cache', async () => { + const server = listServer() + const p = polyhaven() + const [first] = await p.search(q('asphalt'), server.ctx()) + Object.assign(first.raw as Record, { name: 'Tampered', tags: ['tampered'] }) + const [second] = await p.search(q('asphalt'), server.ctx()) + expect(server.listCalls()).toBe(1) // served from the cache + expect(second.title).toBe('Aerial Asphalt 01') + expect(second.raw).toEqual(LIST.aerial_asphalt_01) + }) + it('shares one in-flight download between concurrent searches', async () => { const gate = deferred() const server = listServer([() => gate.promise]) diff --git a/packages/provider-polyhaven/src/index.ts b/packages/provider-polyhaven/src/index.ts index c8c2498..e824aed 100644 --- a/packages/provider-polyhaven/src/index.ts +++ b/packages/provider-polyhaven/src/index.ts @@ -17,8 +17,10 @@ export interface PolyHavenConfig { /** How long, in ms, this provider instance reuses the asset list it downloaded. * Poly Haven's list endpoint has no query or paging, so every search ranks the * whole list (about 1 MB); caching it saves that download on later searches. - * Concurrent searches share one in-flight download, and a failed download is - * not cached. Default 3_600_000 (one hour); 0 disables the cache. */ + * Concurrent searches share one in-flight download; a failed download, or one + * still unfinished after 30 s, is not cached. Default 3_600_000 (one hour). + * 0, a negative value or NaN turns the cache off (every search downloads the + * list); Infinity keeps the list for the lifetime of the instance. */ listTtlMs?: number } @@ -39,6 +41,29 @@ type PolyHavenFiles = Record interface PhFileLeaf { url?: string } const DEFAULT_LIST_TTL_MS = 3_600_000 +/** The shared list download runs without any search's signal, so it gets a + * deadline of its own: a stalled download must not hold every search until + * each one's deadline (Node's fetch gives up only after 300 s). */ +const LIST_TIMEOUT_MS = 30_000 + +/** Run `start(signal)` under a fixed deadline: at `ms` the signal aborts and the + * result rejects, even if the work ignores its signal. The timer is cleared as + * soon as the work settles. Built on setTimeout rather than AbortSignal.timeout + * so fake timers drive it in tests. */ +function withDeadline(start: (signal: AbortSignal) => Promise, ms: number, label: string): Promise { + const ctrl = new AbortController() + let timer: ReturnType | undefined + const expired = new Promise((_, reject) => { + timer = setTimeout(() => { + const err = new Error(`${label} timed out after ${ms}ms`) + ctrl.abort(err) + reject(err) + }, ms) + }) + // a synchronous throw from `start` becomes a rejection, so the timer is still cleared + const work = new Promise(resolve => resolve(start(ctrl.signal))) + return Promise.race([work, expired]).finally(() => clearTimeout(timer)) +} /** Lowercase words of a string, split on anything that isn't alphanumeric — the * rule for both the query tokens and the asset fields they are matched against. @@ -137,7 +162,8 @@ function toReference(id: string, asset: PolyHavenAsset, imageUrl: string, kind: // textureImageUrl may resolve a .png fallback — derive the MIME from the extension // (core imageMediaType) rather than hardcoding jpeg (mislabeling a PNG as JPEG). preview: { url: imageUrl, mediaType: imageMediaType(undefined, imageUrl) }, - raw: asset, + // a copy: `asset` belongs to the cached list, which later searches reuse + raw: { ...asset }, } } @@ -159,7 +185,10 @@ export function polyhaven(config: PolyHavenConfig = {}) { // instead: an aborted search rejects at once but cannot fail the download // other searches share, and a download that outlives the deadline of the // search that started it still lands in the cache for the next search. - const entry: NonNullable = { list: downloadList(ctx) } + // LIST_TIMEOUT_MS bounds it instead; expiry is just another failure. + const entry: NonNullable = { + list: withDeadline(signal => downloadList(ctx, signal), LIST_TIMEOUT_MS, 'polyhaven list'), + } cached = entry entry.list.then( () => { entry.fetchedAt = Date.now() }, From fe107e7ecac9cfaa11f92915264dac9a415fe914 Mon Sep 17 00:00:00 2001 From: lizhixuan Date: Mon, 26 Oct 2026 23:56:15 +0800 Subject: [PATCH 3/3] =?UTF-8?q?fix(provider-polyhaven):=20review=20follow-?= =?UTF-8?q?ups=20=E2=80=94=20validated=20list=20body,=20pinned=20cache/abo?= =?UTF-8?q?rt=20behaviour,=20deep=20raw=20copy,=20Unicode=20tokens?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .changeset/polyhaven-relevance.md | 4 +- packages/provider-polyhaven/README.md | 2 +- .../src/__tests__/polyhaven.test.ts | 142 +++++++++++++++++- packages/provider-polyhaven/src/index.ts | 61 ++++++-- 4 files changed, 186 insertions(+), 23 deletions(-) diff --git a/.changeset/polyhaven-relevance.md b/.changeset/polyhaven-relevance.md index abc6859..9f1cd21 100644 --- a/.changeset/polyhaven-relevance.md +++ b/.changeset/polyhaven-relevance.md @@ -2,6 +2,6 @@ '@refkit/provider-polyhaven': patch --- -Poly Haven results are ranked by how well they match the query, then by popularity. A query word found as a word of an asset's name or id counts most, then a tag or category, then a match inside a longer word; ties go to the most-downloaded asset. Before, most matches tied and kept Poly Haven's alphabetical order, so "sunset HDRI" returned `aarfontein_dawn_2`, `aarfontein_dusk` and the like instead of the popular sunset skies. A query with no words now lists the most-downloaded assets first. +Poly Haven results are ranked by how well they match the query, then by popularity. A query word found as a word of an asset's name or id counts most, then a tag or category, then a match inside a longer word; ties go to the most-downloaded asset. Before, most matches tied and kept Poly Haven's alphabetical order, so "sunset HDRI" returned `aarfontein_dawn_2`, `aarfontein_dusk` and the like instead of the popular sunset skies. An empty query now lists the most-downloaded assets first. Query words may be in any script (`crème`, `日落`); a query that matches no asset, including one made only of punctuation, returns nothing. -The asset list (about 1 MB per asset type) is downloaded once and reused for an hour by each provider instance, instead of on every search. Change that with the new `listTtlMs` option; `0` turns the cache off. +The asset list (about 1 MB per asset type) is downloaded once and reused for an hour by each provider instance, instead of on every search. Change that with the new `listTtlMs` option; `0` turns the cache off. A shared list download still unfinished after 30 seconds fails the searches waiting on it; if a host allows Poly Haven searches longer than that, `listTtlMs: 0` restores one download per search, bounded only by the host's own timeout. diff --git a/packages/provider-polyhaven/README.md b/packages/provider-polyhaven/README.md index 9f3bbbf..ef3aed3 100644 --- a/packages/provider-polyhaven/README.md +++ b/packages/provider-polyhaven/README.md @@ -32,7 +32,7 @@ const hdris = await refkit.search({ query: 'sunset', modalities: ['image'], sour ### Ranking -Poly Haven's list endpoint has no search parameter, so `polyhaven()` downloads the whole list and ranks it itself. Each query word adds the score of its best match in an asset: 3 if it is a word of the asset's name or id, 2 if it is a tag or category or a word of one (`sunset` in `sunrise-sunset`), 1 if it only appears inside one of those fields. Assets that match no word are dropped. Equal scores go to the most-downloaded asset first. A query with no words returns every asset, most-downloaded first. `page` steps through this ranked list in windows of `maxAssets`. +Poly Haven's list endpoint has no search parameter, so `polyhaven()` downloads the whole list and ranks it itself. A word is a run of Unicode letters, combining marks and digits, compared case-insensitively, so `crème` and `日落` are single words. Each query word adds the score of its best match in an asset: 3 if it is a word of the asset's name or id, 2 if it is a tag or category or a word of one (`sunset` in `sunrise-sunset`), 1 if it only appears inside one of those fields. Assets that match no word are dropped. Equal scores go to the most-downloaded asset first. An empty or whitespace-only query returns every asset, most-downloaded first. Any other query that matches nothing returns nothing: that includes a query in a script no asset uses, and one made only of punctuation or emoji. `page` steps through this ranked list in windows of `maxAssets`. ### Asset list cache diff --git a/packages/provider-polyhaven/src/__tests__/polyhaven.test.ts b/packages/provider-polyhaven/src/__tests__/polyhaven.test.ts index 0139fb3..d188c0a 100644 --- a/packages/provider-polyhaven/src/__tests__/polyhaven.test.ts +++ b/packages/provider-polyhaven/src/__tests__/polyhaven.test.ts @@ -1,3 +1,4 @@ +import { getEventListeners } from 'node:events' import { afterEach, describe, expect, it, vi } from 'vitest' import { createRefkit, evaluateUse, type ProviderContext } from '@refkit/core' import { polyhaven } from '../index' @@ -161,17 +162,43 @@ describe('polyhaven relevance ranking', () => { expect(ids(await page(3))).toEqual(['a_dusk']) }) - it('keeps every asset for a query with no tokens, most-downloaded first', async () => { + it('keeps every asset for an empty or whitespace-only query, most-downloaded first', async () => { const list = { a_room: { type: 0, name: 'A Room', download_count: 5 }, b_room: { type: 0, name: 'B Room' }, // no download_count c_room: { type: 0, name: 'C Room', download_count: 50 }, d_room: { type: 0, name: 'D Room', download_count: 5 }, } - for (const text of ['', ' ?! ']) { + for (const text of ['', ' ', ' \t\n ']) { expect(ids(await hdriSearch(list, text))).toEqual(['c_room', 'a_room', 'd_room', 'b_room']) } }) + + it('returns nothing for text with no matchable word: a non-Latin query, punctuation or an emoji', async () => { + const list = { + a_room: { type: 0, name: 'A Room', download_count: 5 }, + b_sunset: { type: 0, name: 'B Sunset', categories: ['sunrise-sunset'], download_count: 50 }, + } + for (const text of ['日落', '日落 hdri', ' ?! ', '🌅']) { + expect(ids(await hdriSearch(list, text)), text).toEqual([]) + } + }) + + it('tokenises on Unicode letters, marks and digits: "crème" is one word, not "cr" + "me"', async () => { + const list = { + a_cr_me: { type: 0, name: 'Cr Me', download_count: 900 }, // what "cr" + "me" would match + b_creme_wall: { type: 0, name: 'Crème Wall', download_count: 1 }, + c_beach: { type: 0, name: '日落 Beach', download_count: 1 }, + // Devanagari vowel signs are marks with no composed form: split on them and + // "सूर्यास्त" (sunset) falls apart into letters that "सरसों" (mustard) also has + d_ghat: { type: 0, name: 'सूर्यास्त Ghat', download_count: 1 }, + e_field: { type: 0, name: 'सरसों Field', download_count: 900 }, + } + expect(ids(await hdriSearch(list, 'crème'))).toEqual(['b_creme_wall']) + expect(ids(await hdriSearch(list, 'cre\u0300me'))).toEqual(['b_creme_wall']) // decomposed (NFD) input + expect(ids(await hdriSearch(list, '日落'))).toEqual(['c_beach']) + expect(ids(await hdriSearch(list, 'सूर्यास्त'))).toEqual(['d_ghat']) + }) }) // — asset list cache — @@ -322,14 +349,19 @@ describe('polyhaven asset list cache', () => { expect(vi.getTimerCount()).toBe(0) // a settled download leaves no timer behind to hold the process open }) - it('emits a copy of the cached asset as raw, so mutating it cannot change the cache', async () => { + it('emits a deep copy of the cached asset as raw, so mutating it cannot change the cache', async () => { const server = listServer() const p = polyhaven() const [first] = await p.search(q('asphalt'), server.ctx()) - Object.assign(first.raw as Record, { name: 'Tampered', tags: ['tampered'] }) + const raw = first.raw as { name: string; tags: string[]; authors: Record } + raw.name = 'Tampered' + raw.tags.push('tampered') + raw.authors.Mallory = 'All' const [second] = await p.search(q('asphalt'), server.ctx()) expect(server.listCalls()).toBe(1) // served from the cache expect(second.title).toBe('Aerial Asphalt 01') + expect(second.tags).toEqual(['asphalt', 'road', 'flat']) + expect(second.rights.author).toBe('Rob Tuytel') expect(second.raw).toEqual(LIST.aerial_asphalt_01) }) @@ -361,3 +393,105 @@ describe('polyhaven asset list cache', () => { expect(server.listCalls()).toBe(1) }) }) + +// — list body validation — + +describe('polyhaven list body validation', () => { + const body = (b: unknown) => () => new Response(JSON.stringify(b), { status: 200 }) + + it('rejects a null body and does not cache it: the next search downloads again and gets results', async () => { + const server = listServer([body(null)]) + const p = polyhaven() + await expect(p.search(q('forest'), server.ctx())).rejects.toThrow('polyhaven list: unexpected response body') + expect(ids(await p.search(q('forest'), server.ctx()))).toEqual(['forest_floor', 'mossy_forest_rock']) + expect(server.listCalls()).toBe(2) + }) + + it('rejects an array, a non-object or an error-object body and does not cache it', async () => { + const bad = [[], 'maintenance', { error: 'rate limited' }] + const server = listServer(bad.map(body)) + const p = polyhaven() + for (const b of bad) { + await expect(p.search(q('forest'), server.ctx()), JSON.stringify(b)).rejects.toThrow('polyhaven list: unexpected response body') + } + expect(ids(await p.search(q('forest'), server.ctx()))).toEqual(['forest_floor', 'mossy_forest_rock']) + expect(server.listCalls()).toBe(4) + }) + + it('skips list entries that are not asset objects', async () => { + const server = listServer([body({ broken: null, note: 'forest', ...LIST })]) + expect(ids(await polyhaven().search(q('forest'), server.ctx()))).toEqual(['forest_floor', 'mossy_forest_rock']) + }) +}) + +// — cache and abort behaviour, pinned — + +/** A fetch stub that never touches a request's signal (so it adds no abort + * listeners of its own) and records the signal each /assets request carried. + * The list reply can be delayed on the (fakeable) setTimeout. */ +function quietServer(listDelayMs = 0) { + const listSignals: Array = [] + const fetchImpl = (async (input: string, init?: RequestInit) => { + const u = String(input) + if (u.includes('/assets')) { + listSignals.push(init?.signal) + if (listDelayMs > 0) await new Promise(r => setTimeout(r, listDelayMs)) + return listResponse() + } + const m = u.match(/\/files\/([^/?]+)/) + const files = m ? (FILES_TEX as Record)[m[1]] : undefined + return files ? new Response(JSON.stringify(files), { status: 200 }) : new Response('null', { status: 404 }) + }) as typeof fetch + return { fetch: fetchImpl, listSignals, listCalls: () => listSignals.length } +} + +describe('polyhaven cache and abort behaviour', () => { + afterEach(() => { vi.useRealTimers() }) + + it('removes its abort listener from the search signal once each search settles', async () => { + const server = quietServer() + const c = new AbortController() + const p = polyhaven() + for (const text of ['forest', 'rock', 'asphalt']) await p.search(q(text), { fetch: server.fetch, signal: c.signal }) + expect(getEventListeners(c.signal, 'abort')).toHaveLength(0) + }) + + it("rejects a search whose signal is already aborted with the signal's reason", async () => { + const server = quietServer() + const c = new AbortController() + c.abort(new Error('cancelled')) + await expect(polyhaven().search(q('forest'), { fetch: server.fetch, signal: c.signal })).rejects.toThrow('cancelled') + }) + + it('starts no list download for a search whose signal is already aborted', async () => { + for (const config of [{}, { listTtlMs: 0 }]) { + const server = quietServer() + const c = new AbortController() + c.abort(new Error('cancelled')) + await expect(polyhaven(config).search(q('forest'), { fetch: server.fetch, signal: c.signal })).rejects.toThrow('cancelled') + expect(server.listCalls(), JSON.stringify(config)).toBe(0) + } + }) + + it('counts the TTL from when the download finished, not from when it started', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] }) + vi.setSystemTime(0) + const server = quietServer(20_000) // the list takes 20 s to arrive + const p = polyhaven({ listTtlMs: 5_000 }) + const first = p.search(q('forest'), { fetch: server.fetch }) + await vi.advanceTimersByTimeAsync(20_000) + await first + await vi.advanceTimersByTimeAsync(4_999) // t = 24.999 s: 4.999 s after the download finished + const second = p.search(q('forest'), { fetch: server.fetch }) // a refetch would be recorded synchronously + expect(server.listCalls()).toBe(1) + expect(ids(await second)).toEqual(['forest_floor', 'mossy_forest_rock']) + }) + + it("passes the search's own signal to the list request when the cache is off", async () => { + const server = quietServer() + const c = new AbortController() + await polyhaven({ listTtlMs: 0 }).search(q('forest'), { fetch: server.fetch, signal: c.signal }) + expect(server.listSignals).toHaveLength(1) + expect(server.listSignals[0]).toBe(c.signal) + }) +}) diff --git a/packages/provider-polyhaven/src/index.ts b/packages/provider-polyhaven/src/index.ts index e824aed..01c9205 100644 --- a/packages/provider-polyhaven/src/index.ts +++ b/packages/provider-polyhaven/src/index.ts @@ -65,11 +65,29 @@ function withDeadline(start: (signal: AbortSignal) => Promise, ms: number, return Promise.race([work, expired]).finally(() => clearTimeout(timer)) } -/** Lowercase words of a string, split on anything that isn't alphanumeric — the - * rule for both the query tokens and the asset fields they are matched against. - * Splitting on punctuation, not just whitespace, keeps "forest, rock." from - * gluing a comma onto a token. */ -const words = (s: string): string[] => s.toLowerCase().split(/[^a-z0-9]+/).filter(Boolean) +/** NFC + lowercase, so composed and decomposed input ("crème" either way) compare equal. */ +const fold = (s: string): string => s.normalize('NFC').toLowerCase() + +/** Words of a string: runs of Unicode letters, combining marks and digits, after + * `fold` — the rule for both the query tokens and the asset fields they are + * matched against. Splitting on punctuation, not just whitespace, keeps + * "forest, rock." from gluing a comma onto a token; Unicode classes keep "crème" + * and "日落" whole instead of cutting them at every non-ASCII letter (marks are + * part of a word in Indic scripts and in decomposed Latin). */ +const words = (s: string): string[] => fold(s).split(/[^\p{L}\p{M}\p{N}]+/u).filter(Boolean) + +const isObject = (v: unknown): v is Record => typeof v === 'object' && v !== null && !Array.isArray(v) + +/** The list body is id → asset object. Anything else — null, an array, a string, + * or an object with entries but no asset among them such as `{"error": …}` — + * rejects, so a bad 200 fails its searches and is not cached for the TTL. */ +function checkedList(body: unknown): PolyHavenList { + const values = isObject(body) ? Object.values(body) : undefined + if (!values || (values.length > 0 && !values.some(isObject))) { + throw new Error('polyhaven list: unexpected response body') + } + return body as PolyHavenList +} /** How well an asset matches the query: each token adds its best match only — * 3 for a word of the name or id, 2 for a tag or category or a word of one @@ -81,7 +99,7 @@ function relevance(id: string, asset: PolyHavenAsset, tokens: string[]): number // A token has no separators, so it can equal a whole label only when the label // is one word: the label's own words cover both cases. const labelWords = new Set(labels.flatMap(words)) - const fields = [id, asset.name ?? '', ...labels].map(f => f.toLowerCase()) + const fields = [id, asset.name ?? '', ...labels].map(fold) let score = 0 for (const t of tokens) { score += nameWords.has(t) ? 3 : labelWords.has(t) ? 2 : fields.some(f => f.includes(t)) ? 1 : 0 @@ -91,15 +109,23 @@ function relevance(id: string, asset: PolyHavenAsset, tokens: string[]): number /** Rank the list for a query — the list endpoint has no query param, so this is * the search. Keeps assets that match at least one token and orders them by - * score, then download_count, then the list's own (alphabetical) order. A query - * with no tokens keeps every asset, so the same order reduces to most-downloaded - * first: the alphabetical order means nothing, and it would show the same - * `aarfontein_*` assets to every query-less search. */ -function rankAssets(list: PolyHavenList, tokens: string[]): [string, PolyHavenAsset][] { + * score, then download_count, then the list's own (alphabetical) order. + * + * Only blank text (empty or whitespace) browses: it keeps every asset, so the + * same order reduces to most-downloaded first — the alphabetical order means + * nothing and would show every browse the same `aarfontein_*` assets. Text that + * is not blank but yields no word (punctuation, an emoji such as 🌅) still asked + * for something, and we cannot tell "?!" from a symbol that means "sunrise", so + * it returns nothing rather than unrelated popular assets dressed as matches. */ +function rankAssets(list: PolyHavenList, text: string): [string, PolyHavenAsset][] { + const tokens = words(text) + const browse = text.trim() === '' + if (!browse && tokens.length === 0) return [] const downloads = ({ download_count: d }: PolyHavenAsset) => (typeof d === 'number' && Number.isFinite(d) ? d : 0) return Object.entries(list) + .filter(([, asset]) => isObject(asset)) // one malformed entry must not sink the search .map(([id, asset]) => ({ id, asset, score: relevance(id, asset, tokens) })) - .filter(e => tokens.length === 0 || e.score > 0) + .filter(e => browse || e.score > 0) // stable sort: equal score and downloads keep the list's own order .sort((x, y) => y.score - x.score || downloads(y.asset) - downloads(x.asset)) .map(e => [e.id, e.asset]) @@ -162,8 +188,9 @@ function toReference(id: string, asset: PolyHavenAsset, imageUrl: string, kind: // textureImageUrl may resolve a .png fallback — derive the MIME from the extension // (core imageMediaType) rather than hardcoding jpeg (mislabeling a PNG as JPEG). preview: { url: imageUrl, mediaType: imageMediaType(undefined, imageUrl) }, - // a copy: `asset` belongs to the cached list, which later searches reuse - raw: { ...asset }, + // a deep copy: `asset` belongs to the cached list, which later searches reuse, + // and its nested tags and authors feed their ranking, tags and attribution + raw: structuredClone(asset), } } @@ -173,12 +200,14 @@ export function polyhaven(config: PolyHavenConfig = {}) { const listUrl = new URL(`${PH_BASE}/assets`) listUrl.searchParams.set('t', assetType) const downloadList = (ctx: ProviderContext, signal?: AbortSignal) => - ctx.fetch(listUrl.toString(), { signal }).then(res => okJson(res, 'polyhaven list')) + ctx.fetch(listUrl.toString(), { signal }).then(res => okJson(res, 'polyhaven list')).then(checkedList) // The cached list: an in-flight download until `fetchedAt` is set, fresh for // listTtlMs after that. let cached: { list: Promise; fetchedAt?: number } | undefined const assetList = (ctx: ProviderContext): Promise => { + // already cancelled: reject before any download starts (or any cache entry exists) + if (ctx.signal?.aborted) return Promise.reject(ctx.signal.reason ?? new Error('aborted')) if (!(listTtlMs > 0)) return downloadList(ctx, ctx.signal) if (!cached || (cached.fetchedAt !== undefined && Date.now() - cached.fetchedAt >= listTtlMs)) { // Download WITHOUT this search's signal and race the signal per caller @@ -212,7 +241,7 @@ export function polyhaven(config: PolyHavenConfig = {}) { // categories, tags), so "forest rock" as one substring matches nothing // while both of its words describe real assets. The ranking must happen // here, before the maxAssets window: core reranks only what we return. - const entries = rankAssets(await assetList(ctx), words(q.text ?? '')) + const entries = rankAssets(await assetList(ctx), q.text ?? '') const n = Math.min(config.maxAssets ?? q.limit ?? 12, 30) // the list endpoint returns everything — page = a window over the ranked list const from = offsetForPage(q.controls?.page, n) ?? 0