From c8a3d5c6eb26e17cf35143ac1a93397c408587e5 Mon Sep 17 00:00:00 2001 From: Ian Macartney <366683+ianmacartney@users.noreply.github.com> Date: Thu, 17 Sep 2026 23:23:28 -0700 Subject: [PATCH 1/5] perf: benchmark batch row refetching before patching --- BENCHMARK.md | 99 ++++++++++++ README.md | 5 + benchmark.mjs | 241 +++++++++++++++++++++++++++++ example/README.md | 39 +++++ example/convex/_generated/api.d.ts | 2 + example/convex/benchmark.ts | 230 +++++++++++++++++++++++++++ example/convex/schema.ts | 9 ++ 7 files changed, 625 insertions(+) create mode 100644 BENCHMARK.md create mode 100644 benchmark.mjs create mode 100644 example/convex/benchmark.ts diff --git a/BENCHMARK.md b/BENCHMARK.md new file mode 100644 index 0000000..c7857ac --- /dev/null +++ b/BENCHMARK.md @@ -0,0 +1,99 @@ +# Re-fetching before patching + +This benchmark compares two ways of processing a batch when no rows change +between selection and processing: + +- **Direct patch:** a snapshot query returns `{ id, value }`; the mutation + trusts that value and patches `{ processed: true, result: value + 1 }`. +- **Re-fetch:** a snapshot query returns IDs; the mutation calls `db.get` for + each ID, checks that the row exists and is unprocessed, then applies the same + patch using the current value. + +Both variants scan the same index, use `ctx.runQuery` with +`useStaleSnapshot: true`, and invoke the worker through `ctx.runMutation` in the +same outer mutation, matching the component's query/mutation calling pattern. +Both use sequential patches; the re-fetch variant adds a sequential point read +and eligibility check before each patch. Queries pass only fields needed for +processing, so artificial document padding is not passed in mutation arguments. + +## Results + +Measured on 2026-09-18 UTC (2026-09-17 PDT). All 600 measured executions +produced the expected results, with no reported OCC retries or execution errors. +Times below are milliseconds per complete query-plus-mutation transaction. + +| Rows per batch | Padding per row | Direct patch mean | Re-fetch mean | Mean difference | Paired 95% interval | +| -------------: | --------------: | ----------------: | ------------: | --------------: | ------------------: | +| 1 | 0 B | 14.62 | 14.63 | +0.01 | -0.70 to +0.72 | +| 25 | 0 B | 87.46 | 86.80 | -0.66 | -4.65 to +3.33 | +| 100 | 0 B | 335.50 | 408.84 | +73.34 | -12.87 to +159.54 | +| 1 | 4096 B | 15.84 | 15.16 | -0.68 | -2.01 to +0.65 | +| 25 | 4096 B | 92.08 | 91.65 | -0.43 | -5.42 to +4.57 | +| 100 | 4096 B | 298.61 | 322.84 | +24.23 | +4.14 to +44.33 | + +The 1- and 25-row cases did not show a clear latency difference. At 100 rows +with 4096 bytes of padding, re-fetching increased the mean by 8.1% in this run. +The 100-row case without padding had large latency outliers and an inconclusive +mean difference: medians were 312.94 ms versus 318.41 ms, while p95 values were +530.29 ms versus 994.53 ms. Its 21.9% mean increase should not be treated as a +stable overhead estimate. + +Database usage was deterministic across trials: + +| Rows per batch | Padding per row | Direct patch read docs | Re-fetch read docs | Direct patch read bytes | Re-fetch read bytes | Write bytes, either variant | +| -------------: | --------------: | ---------------------: | -----------------: | ----------------------: | ------------------: | --------------------------: | +| 1 | 0 B | 2 | 3 | 380 | 527 | 282 | +| 25 | 0 B | 50 | 75 | 9,575 | 13,275 | 7,100 | +| 100 | 0 B | 200 | 300 | 38,600 | 53,500 | 28,600 | +| 1 | 4096 B | 2 | 3 | 8,581 | 12,827 | 4,384 | +| 25 | 4096 B | 50 | 75 | 214,600 | 320,775 | 109,650 | +| 100 | 4096 B | 200 | 300 | 858,700 | 1,283,500 | 438,800 | + +Re-fetching added one reported document read per row: **50% more read documents +for the full query-plus-mutation transaction**, with identical writes. This is a +resource-usage measurement, not a claim of a 50% increase in total cost or +latency. Byte counts include stored row fields and metadata, not just padding. + +Run ID: `478bc6c6-a4f9-40f6-bd71-1f6c15fead8e`. Raw evidence is in +`.context/benchmark-results/results.json`, `cases.json`, and `logs.jsonl`. + +## Method + +Run on the cloud dev deployment `first-lobster-65` with Convex JS 1.43.0. For +each combination of batch size (1, 25, 100) and document padding (0, 4096 +bytes), discard five warmup pairs, then measure 50 paired trials with +alternating variant order. This gives 600 measured executions plus 60 warmup +executions. Each pair uses the same rows, reset before each variant, with no +concurrent writers to those rows. All results are verified after every +execution. Fixtures are scoped to each case and removed in a `finally` block. + +Server time and database usage come from the outer mutation's Convex completion +log. Time includes the snapshot query and nested mutation, but excludes fixture +setup, resets, verification, cleanup, action orchestration, scheduler overhead, +and network latency to the CLI. The harness separately records the action's +`runMutation` round-trip time. No timers run inside mutations, where +`Date.now()` is fixed. + +Paired differences are re-fetch minus direct patch. The reported 95% intervals +use a normal approximation from the standard error of the paired differences; +they describe variation within this run, not variation across deployments. + +## Reproduce + +```sh +npm ci +npx convex dev --once +node benchmark.mjs +``` + +The harness requires a dev deployment in `.env.local`. Configuration options are +documented in +[example/README.md](./example/README.md#benchmark-point-reads-before-patching). +Machine-readable results, action timings, and raw server logs are written to +`.context/benchmark-results/`. + +This is a warm, uncontended workload on one deployment. It does not benchmark +cold database reads, scheduler throughput, simultaneous writers, or deliberately +different snapshots. Re-fetching is needed for correctness when eligibility or +values can change, regardless of its measured overhead here. The direct-patch +variant is an intentional benchmark baseline, not general application guidance. diff --git a/README.md b/README.md index 2b672a7..65a9a73 100644 --- a/README.md +++ b/README.md @@ -552,6 +552,11 @@ The demo is hosted on Convex itself via published to `https://.convex.site`. See [example/README.md](./example/README.md#deploy) for the deploy commands. +For a real-deployment comparison of returning values and patching directly +versus returning IDs and re-fetching before patching, see +[the benchmark results](./BENCHMARK.md) and +[how to run the benchmark](./example/README.md#benchmark-point-reads-before-patching). + ### How it works | Table | Written by | Read by | diff --git a/benchmark.mjs b/benchmark.mjs new file mode 100644 index 0000000..ebc570a --- /dev/null +++ b/benchmark.mjs @@ -0,0 +1,241 @@ +#!/usr/bin/env node +// Real-deployment comparison of query values -> patch vs query IDs -> get+patch. +// Usage: node benchmark.mjs [--trials 50] [--batch-sizes 1,25,100] +import { spawn, execFile } from "node:child_process"; +import { readFileSync, mkdirSync, writeFileSync } from "node:fs"; +import { randomUUID } from "node:crypto"; +import { parseArgs, parseEnv, promisify } from "node:util"; + +const { values } = parseArgs({ + options: { + trials: { type: "string", default: "50" }, + warmups: { type: "string", default: "5" }, + "batch-sizes": { type: "string", default: "1,25,100" }, + "padding-bytes": { type: "string", default: "0,4096" }, + output: { type: "string", default: ".context/benchmark-results" }, + }, +}); +const env = parseEnv(readFileSync(".env.local", "utf8")); +const deployment = env.CONVEX_DEPLOYMENT?.match(/^dev:([\w-]+)$/)?.[1]; +if (!deployment || process.env.CONVEX_DEPLOY_KEY || env.CONVEX_DEPLOY_KEY) { + throw new Error( + "This harness requires a dev deployment in .env.local and no deploy-key override.", + ); +} +const trials = Number(values.trials); +const warmups = Number(values.warmups); +const batchSizes = values["batch-sizes"].split(",").map(Number); +const paddingSizes = values["padding-bytes"].split(",").map(Number); +for (const [numbers, min, max] of [ + [batchSizes, 1, 500], + [paddingSizes, 0, 16384], + [[trials], 1, 100], + [[warmups], 0, 20], +]) { + if (numbers.some((n) => !Number.isInteger(n) || n < min || n > max)) { + throw new Error(`Expected integers from ${min} to ${max}`); + } +} +const outDir = values.output; +mkdirSync(outDir, { recursive: true }); +const runId = randomUUID(); +const cli = "node_modules/convex/bin/main.js"; +const exec = promisify(execFile); +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +const events = []; +const cases = []; +const tagPattern = + /batch-read-bench\|([\w-]+)\|(\d+)\|(\d+)\|(-?\d+)\|(patch|refetch)\|(true|false)/; + +function completions() { + const unique = new Map(); + for (const event of events) { + if ( + event.kind !== "Completion" || + event.identifier?.replace(".js:", ":") !== "benchmark:iteration" + ) + continue; + const tag = JSON.stringify(event.logLines).match(tagPattern); + if (!tag || !tag[1].startsWith(runId)) continue; + unique.set(event.executionId, { + ...event, + caseId: tag[1], + batchSize: +tag[2], + paddingBytes: +tag[3], + trial: +tag[4], + mode: tag[5], + warmup: tag[6] === "true", + }); + } + return [...unique.values()]; +} + +function stats(xs) { + if (!xs.length) return null; + const sorted = [...xs].sort((a, b) => a - b); + const mean = xs.reduce((a, b) => a + b, 0) / xs.length; + return { + n: xs.length, + mean, + p50: sorted[Math.floor(xs.length * 0.5)], + p95: sorted[Math.min(xs.length - 1, Math.floor(xs.length * 0.95))], + }; +} + +function summarize(samples) { + return { + executionMs: stats(samples.map((e) => e.executionTime * 1000)), + readDocuments: stats( + samples.map((e) => e.usageStats.databaseReadDocuments), + ), + readBytes: stats(samples.map((e) => e.usageStats.databaseReadBytes)), + writeBytes: stats(samples.map((e) => e.usageStats.databaseWriteBytes)), + }; +} + +console.log( + `Target: dev (${deployment}); ${trials} paired trials, ${warmups} warmup pairs per case.`, +); +const logProcess = spawn( + process.execPath, + [cli, "logs", "--deployment", deployment, "--jsonl", "--success"], + { stdio: ["ignore", "pipe", "pipe"] }, +); +let buffer = ""; +let logError = ""; +logProcess.stderr.on("data", (chunk) => { + logError += chunk.toString(); +}); +logProcess.stdout.on("data", (chunk) => { + buffer += chunk.toString(); + let newline; + while ((newline = buffer.indexOf("\n")) >= 0) { + const line = buffer.slice(0, newline).trim(); + buffer = buffer.slice(newline + 1); + if (line.startsWith("{")) events.push(JSON.parse(line)); + } +}); + +try { + await sleep(2000); + if (logProcess.exitCode !== null) + throw new Error(`Log stream exited: ${logError}`); + for (const paddingBytes of paddingSizes) { + for (const batchSize of batchSizes) { + const caseId = `${runId}-${batchSize}-${paddingBytes}`; + const args = { runId: caseId, batchSize, paddingBytes, trials, warmups }; + const { stdout } = await exec( + process.execPath, + [ + cli, + "run", + "--deployment", + deployment, + "benchmark:run", + JSON.stringify(args), + ], + { maxBuffer: 32 * 1024 * 1024 }, + ); + const actionSamples = JSON.parse(stdout); + cases.push({ caseId, batchSize, paddingBytes, actionSamples }); + writeFileSync(`${outDir}/cases.json`, JSON.stringify(cases, null, 2)); + console.log( + `Verified ${batchSize} rows/batch, ${paddingBytes} padding bytes: ${actionSamples.length} measured executions; fixtures cleaned up.`, + ); + } + } + const expected = cases.length * trials * 2; + const deadline = Date.now() + 30000; + while ( + completions().filter((e) => !e.warmup && !e.error && !e.willRetry).length < + expected && + Date.now() < deadline + ) + await sleep(500); + const measured = completions().filter( + (e) => !e.warmup && !e.error && !e.willRetry, + ); + if (measured.length !== expected) + throw new Error( + `Expected ${expected} completion logs, got ${measured.length}. ${logError}`, + ); + const results = cases.map((spec) => { + const samples = measured.filter((e) => e.caseId === spec.caseId); + const patch = samples + .filter((e) => e.mode === "patch") + .sort((a, b) => a.trial - b.trial); + const refetch = samples + .filter((e) => e.mode === "refetch") + .sort((a, b) => a.trial - b.trial); + for (const rows of [patch, refetch]) { + if (rows.length !== trials || rows.some((e, i) => e.trial !== i)) + throw new Error("Missing or duplicated trial logs"); + } + const diffs = refetch.map( + (e, i) => (e.executionTime - patch[i].executionTime) * 1000, + ); + const mean = stats(diffs).mean; + const se = + trials > 1 + ? Math.sqrt( + diffs.reduce((sum, d) => sum + (d - mean) ** 2, 0) / + (trials - 1) / + trials, + ) + : null; + return { + batchSize: spec.batchSize, + paddingBytes: spec.paddingBytes, + patch: summarize(patch), + refetch: summarize(refetch), + pairedDifferenceMs: { + mean, + normalApprox95CI: + se === null ? null : [mean - 1.96 * se, mean + 1.96 * se], + }, + actionRoundTripMs: Object.fromEntries( + ["patch", "refetch"].map((mode) => [ + mode, + stats( + spec.actionSamples + .filter((s) => s.mode === mode) + .map((s) => s.elapsedMs), + ), + ]), + ), + }; + }); + const report = { + createdAt: new Date().toISOString(), + deployment, + runId, + trials, + warmups, + convexVersion: JSON.parse( + readFileSync("node_modules/convex/package.json", "utf8"), + ).version, + errorsOrRetries: completions().filter( + (e) => e.error || e.willRetry || e.occInfo?.retryCount, + ).length, + results, + }; + writeFileSync(`${outDir}/results.json`, JSON.stringify(report, null, 2)); + console.table( + results.map((r) => ({ + batch: r.batchSize, + padding: r.paddingBytes, + "patch mean ms": r.patch.executionMs.mean.toFixed(3), + "refetch mean ms": r.refetch.executionMs.mean.toFixed(3), + "difference ms": r.pairedDifferenceMs.mean.toFixed(3), + "patch reads": r.patch.readDocuments.mean, + "refetch reads": r.refetch.readDocuments.mean, + })), + ); + console.log(`Wrote ${outDir}/results.json`); +} finally { + logProcess.kill("SIGTERM"); + writeFileSync( + `${outDir}/logs.jsonl`, + events.map((e) => JSON.stringify(e)).join("\n") + "\n", + ); +} diff --git a/example/README.md b/example/README.md index 1ef22b6..c9be6f5 100644 --- a/example/README.md +++ b/example/README.md @@ -30,6 +30,45 @@ npm run dev # backend + component watcher npm run dev:frontend # Vite dev server (second terminal) ``` +## Benchmark point reads before patching + +After pushing the backend to your dev deployment, run from the repo root: + +```sh +npx convex dev --once +node benchmark.mjs +``` + +This compares a snapshot query returning `{ id, value }` followed by direct +patches against a snapshot query returning IDs followed by `db.get`, an +eligibility check, and the same patches. Both paths use sequential patches. Only +the fields needed for processing are passed through the batch; optional document +padding measures the effect of larger stored rows. + +The default run measures 50 paired trials after five warmup pairs for batches of +1, 25, and 100 rows, with 0 or 4096 padding bytes per row. Trial order +alternates. There are no concurrent writers, and every trial verifies that all +rows received identical results. Setup, reset, verification, and cleanup are +outside the measured transaction. Fixtures live in a separate table and are +deleted after each case. All benchmark functions are internal. + +Results and raw server completion logs are saved to +`.context/benchmark-results/`. Server execution time includes the snapshot query +and nested worker mutation, but excludes scheduling and network latency to the +CLI. The action's `runMutation` round-trip timing is also recorded. This +measures the overhead of re-fetching when rows have not changed; it does not +simulate stale snapshots or measure conflict retries under load. + +For a shorter run or different batch sizes: + +```sh +node benchmark.mjs --trials 10 --warmups 2 --batch-sizes 25,100 --padding-bytes 0 +``` + +The harness requires a `dev:` deployment in `.env.local` and refuses a +deploy-key override. See [BENCHMARK.md](../BENCHMARK.md) for recorded results +and limitations. + ## Deploy The demo is hosted on Convex with diff --git a/example/convex/_generated/api.d.ts b/example/convex/_generated/api.d.ts index bfd09a4..f752eda 100644 --- a/example/convex/_generated/api.d.ts +++ b/example/convex/_generated/api.d.ts @@ -9,6 +9,7 @@ */ import type * as aggregates from "../aggregates.js"; +import type * as benchmark from "../benchmark.js"; import type * as e2e from "../e2e.js"; import type * as example from "../example.js"; import type * as rateLimited from "../rateLimited.js"; @@ -21,6 +22,7 @@ import type { declare const fullApi: ApiFromModules<{ aggregates: typeof aggregates; + benchmark: typeof benchmark; e2e: typeof e2e; example: typeof example; rateLimited: typeof rateLimited; diff --git a/example/convex/benchmark.ts b/example/convex/benchmark.ts new file mode 100644 index 0000000..5e2a147 --- /dev/null +++ b/example/convex/benchmark.ts @@ -0,0 +1,230 @@ +import { v } from "convex/values"; +import { internal } from "./_generated/api.js"; +import { + internalAction, + internalMutation, + internalQuery, +} from "./_generated/server.js"; +import type { Id } from "./_generated/dataModel.js"; + +// Run with benchmark.mjs. All endpoints are internal, and fixtures are scoped +// to a unique run ID. No production queues or worker state are touched. +const vIds = v.array(v.id("benchmarkItems")); +const vItems = v.array( + v.object({ id: v.id("benchmarkItems"), value: v.number() }), +); +const vMode = v.union(v.literal("patch"), v.literal("refetch")); +const vCase = { + runId: v.string(), + batchSize: v.number(), + paddingBytes: v.number(), +}; + +function boundedInteger(value: number, min: number, max: number) { + if (!Number.isInteger(value) || value < min || value > max) { + throw new Error(`Expected an integer from ${min} to ${max}, got ${value}`); + } +} + +export const seed = internalMutation({ + args: vCase, + returns: vIds, + handler: async (ctx, { runId, batchSize, paddingBytes }) => { + boundedInteger(batchSize, 1, 500); + boundedInteger(paddingBytes, 0, 16384); + const ids: Id<"benchmarkItems">[] = []; + for (let i = 0; i < batchSize; i++) { + ids.push( + await ctx.db.insert("benchmarkItems", { + runId, + value: i, + padding: "x".repeat(paddingBytes), + processed: false, + }), + ); + } + return ids; + }, +}); + +export const reset = internalMutation({ + args: { ids: vIds }, + returns: v.null(), + handler: async (ctx, { ids }) => { + for (const id of ids) { + await ctx.db.patch("benchmarkItems", id, { + processed: false, + result: undefined, + }); + } + return null; + }, +}); + +const vQueryArgs = { runId: v.string(), batchSize: v.number() }; + +export const batchIds = internalQuery({ + args: vQueryArgs, + returns: vIds, + handler: async (ctx, { runId, batchSize }) => { + boundedInteger(batchSize, 1, 500); + const rows = await ctx.db + .query("benchmarkItems") + .withIndex("by_runId_processed", (q) => + q.eq("runId", runId).eq("processed", false), + ) + .take(batchSize); + return rows.map((row) => row._id); + }, +}); + +export const batchValues = internalQuery({ + args: vQueryArgs, + returns: vItems, + handler: async (ctx, { runId, batchSize }) => { + boundedInteger(batchSize, 1, 500); + const rows = await ctx.db + .query("benchmarkItems") + .withIndex("by_runId_processed", (q) => + q.eq("runId", runId).eq("processed", false), + ) + .take(batchSize); + // Carry only needed fields, not the artificial padding. + return rows.map((row) => ({ id: row._id, value: row.value })); + }, +}); + +export const patch = internalMutation({ + args: { items: vItems }, + returns: v.number(), + handler: async (ctx, { items }) => { + // Intentionally assumes the query's values and eligibility are unchanged. + // This is the comparison baseline, not the recommended general pattern. + for (const { id, value } of items) { + await ctx.db.patch("benchmarkItems", id, { + processed: true, + result: value + 1, + }); + } + return items.length; + }, +}); + +export const refetch = internalMutation({ + args: { ids: vIds }, + returns: v.number(), + handler: async (ctx, { ids }) => { + let processed = 0; + // Match the baseline's sequential patch order, adding one point read and + // eligibility check per row. Both paths write exactly the same fields. + for (const id of ids) { + const row = await ctx.db.get("benchmarkItems", id); + if (!row || row.processed) continue; + await ctx.db.patch("benchmarkItems", id, { + processed: true, + result: row.value + 1, + }); + processed++; + } + return processed; + }, +}); + +export const iteration = internalMutation({ + args: { ...vCase, mode: vMode, trial: v.number(), warmup: v.boolean() }, + returns: v.number(), + handler: async ( + ctx, + { runId, batchSize, paddingBytes, mode, trial, warmup }, + ): Promise => { + // Tag the root completion log so the harness can extract server execution + // time and database usage, excluding setup/verification and action overhead. + console.log( + `batch-read-bench|${runId}|${batchSize}|${paddingBytes}|${trial}|${mode}|${warmup}`, + ); + const args = { runId, batchSize }; + if (mode === "patch") { + const items = await ctx.runQuery(internal.benchmark.batchValues, args, { + useStaleSnapshot: true, + }); + return await ctx.runMutation(internal.benchmark.patch, { items }); + } + const ids = await ctx.runQuery(internal.benchmark.batchIds, args, { + useStaleSnapshot: true, + }); + return await ctx.runMutation(internal.benchmark.refetch, { ids }); + }, +}); + +export const verify = internalQuery({ + args: { ids: vIds }, + returns: v.null(), + handler: async (ctx, { ids }) => { + for (const id of ids) { + const row = await ctx.db.get("benchmarkItems", id); + if (!row || !row.processed || row.result !== row.value + 1) { + throw new Error(`Incorrect benchmark result for ${id}`); + } + } + return null; + }, +}); + +export const cleanup = internalMutation({ + args: { runId: v.string() }, + returns: v.number(), + handler: async (ctx, { runId }) => { + const rows = await ctx.db + .query("benchmarkItems") + .withIndex("by_runId_processed", (q) => q.eq("runId", runId)) + .take(500); + for (const row of rows) await ctx.db.delete("benchmarkItems", row._id); + return rows.length; + }, +}); + +export const run = internalAction({ + args: { ...vCase, trials: v.number(), warmups: v.number() }, + returns: v.array( + v.object({ mode: vMode, trial: v.number(), elapsedMs: v.number() }), + ), + handler: async (ctx, { trials, warmups, ...spec }) => { + boundedInteger(trials, 1, 100); + boundedInteger(warmups, 0, 20); + const samples: { + mode: "patch" | "refetch"; + trial: number; + elapsedMs: number; + }[] = []; + try { + const ids = await ctx.runMutation(internal.benchmark.seed, spec); + for (let trial = -warmups; trial < trials; trial++) { + // Alternate which variant runs first to reduce drift/order bias. + const modes = + trial % 2 === 0 + ? (["patch", "refetch"] as const) + : (["refetch", "patch"] as const); + for (const mode of modes) { + await ctx.runMutation(internal.benchmark.reset, { ids }); + // Date.now advances in actions; it is fixed inside mutations. Server + // completion logs provide a second, more precise execution metric. + const start = Date.now(); + const count = await ctx.runMutation(internal.benchmark.iteration, { + ...spec, + mode, + trial, + warmup: trial < 0, + }); + const elapsedMs = Date.now() - start; + if (count !== spec.batchSize) + throw new Error(`Expected ${spec.batchSize} patches, got ${count}`); + await ctx.runQuery(internal.benchmark.verify, { ids }); + if (trial >= 0) samples.push({ mode, trial, elapsedMs }); + } + } + return samples; + } finally { + await ctx.runMutation(internal.benchmark.cleanup, { runId: spec.runId }); + } + }, +}); diff --git a/example/convex/schema.ts b/example/convex/schema.ts index 244a190..eb2cef0 100644 --- a/example/convex/schema.ts +++ b/example/convex/schema.ts @@ -60,4 +60,13 @@ export default defineSchema({ oldestLatencyMs: v.number(), newestLatencyMs: v.number(), }), + + // Isolated fixtures for benchmark.mjs; each run cleans up its own rows. + benchmarkItems: defineTable({ + runId: v.string(), + value: v.number(), + padding: v.string(), + processed: v.boolean(), + result: v.optional(v.number()), + }).index("by_runId_processed", ["runId", "processed"]), }); From a3d37ccebd7e26a034dbf6ce42598e85572d289b Mon Sep 17 00:00:00 2001 From: Ian Macartney <366683+ianmacartney@users.noreply.github.com> Date: Thu, 17 Sep 2026 23:53:27 -0700 Subject: [PATCH 2/5] perf: benchmark parallel batch reads with sequential patches --- BENCHMARK.md | 93 ++++++++++++++++++++++++++----------- benchmark.mjs | 5 +- example/README.md | 19 ++++---- example/convex/benchmark.ts | 12 +++-- 4 files changed, 87 insertions(+), 42 deletions(-) diff --git a/BENCHMARK.md b/BENCHMARK.md index c7857ac..a61b923 100644 --- a/BENCHMARK.md +++ b/BENCHMARK.md @@ -5,18 +5,54 @@ between selection and processing: - **Direct patch:** a snapshot query returns `{ id, value }`; the mutation trusts that value and patches `{ processed: true, result: value + 1 }`. -- **Re-fetch:** a snapshot query returns IDs; the mutation calls `db.get` for - each ID, checks that the row exists and is unprocessed, then applies the same - patch using the current value. +- **Re-fetch:** a snapshot query returns IDs; the mutation fetches the rows with + `Promise.all(ids.map(id => ctx.db.get(...)))`, then checks eligibility and + applies the same patches in a sequential `for` loop using current values. Both variants scan the same index, use `ctx.runQuery` with `useStaleSnapshot: true`, and invoke the worker through `ctx.runMutation` in the same outer mutation, matching the component's query/mutation calling pattern. -Both use sequential patches; the re-fetch variant adds a sequential point read -and eligibility check before each patch. Queries pass only fields needed for -processing, so artificial document padding is not passed in mutation arguments. - -## Results +Both use sequential patches. Only the gets are parallelized, matching the +application examples. Queries pass only fields needed for processing, so +artificial document padding is not passed in mutation arguments. + +## Results: parallel gets, sequential patches + +Measured on 2026-09-18 UTC (2026-09-17 PDT) on dev `enchanted-cardinal-63`. Both +variants were measured afresh in this run. All 600 measured executions produced +the expected results, with no reported OCC retries or execution errors. The +benchmark table was confirmed empty after cleanup. Times are milliseconds per +complete query-plus-mutation transaction. + +| Rows per batch | Padding per row | Direct patch mean | Parallel-get mean | Mean difference | Paired 95% interval | +| -------------: | --------------: | ----------------: | ----------------: | --------------: | ------------------: | +| 1 | 0 B | 16.13 | 15.72 | -0.42 | -2.08 to +1.25 | +| 25 | 0 B | 98.30 | 48.71 | -49.59 | -70.46 to -28.72 | +| 100 | 0 B | 337.08 | 94.90 | -242.18 | -351.62 to -132.74 | +| 1 | 4096 B | 14.21 | 14.67 | +0.46 | -0.80 to +1.72 | +| 25 | 4096 B | 79.72 | 31.44 | -48.29 | -53.01 to -43.56 | +| 100 | 4096 B | 292.50 | 87.74 | -204.77 | -226.71 to -182.82 | + +With patches still sequential in both variants, parallel gets reduced mean +server time by **50–61% for 25 rows** and **70–72% for 100 rows** in this run. +All four of those paired intervals exclude zero. Single-row cases showed no +clear difference. The medians also favor parallel gets: at 100 rows they were +280.26 ms versus 79.95 ms without padding, and 285.47 ms versus 82.35 ms with +4096 bytes of padding. The benchmark measures the complete transaction; it does +not isolate the database-internal reason for the improvement. + +Run ID: `5b565bfd-2962-44b6-a408-856f1ef7b942`. Raw evidence is in +`.context/benchmark-parallel-results/results.json`, `cases.json`, and +`logs.jsonl`. + +## Earlier results: sequential gets + +The results below used the original implementation on dev `first-lobster-65`, +which awaited one get and one patch at a time. They do not measure `Promise.all` +gets. That implementation is available in commit `066836b`. Since the parallel +run uses a different deployment, compare each re-fetch variant with its own +direct-patch baseline; these runs do not isolate the speedup from changing +sequential gets to parallel gets. Measured on 2026-09-18 UTC (2026-09-17 PDT). All 600 measured executions produced the expected results, with no reported OCC retries or execution errors. @@ -38,7 +74,13 @@ mean difference: medians were 312.94 ms versus 318.41 ms, while p95 values were 530.29 ms versus 994.53 ms. Its 21.9% mean increase should not be treated as a stable overhead estimate. -Database usage was deterministic across trials: +Run ID: `478bc6c6-a4f9-40f6-bd71-1f6c15fead8e`. Raw evidence is in +`.context/benchmark-results/results.json`, `cases.json`, and `logs.jsonl`. + +## Database usage + +The read-document, read-byte, and write-byte measurements were identical in the +parallel-get and sequential-get runs, and deterministic across trials: | Rows per batch | Padding per row | Direct patch read docs | Re-fetch read docs | Direct patch read bytes | Re-fetch read bytes | Write bytes, either variant | | -------------: | --------------: | ---------------------: | -----------------: | ----------------------: | ------------------: | --------------------------: | @@ -54,18 +96,15 @@ for the full query-plus-mutation transaction**, with identical writes. This is a resource-usage measurement, not a claim of a 50% increase in total cost or latency. Byte counts include stored row fields and metadata, not just padding. -Run ID: `478bc6c6-a4f9-40f6-bd71-1f6c15fead8e`. Raw evidence is in -`.context/benchmark-results/results.json`, `cases.json`, and `logs.jsonl`. - ## Method -Run on the cloud dev deployment `first-lobster-65` with Convex JS 1.43.0. For -each combination of batch size (1, 25, 100) and document padding (0, 4096 -bytes), discard five warmup pairs, then measure 50 paired trials with -alternating variant order. This gives 600 measured executions plus 60 warmup -executions. Each pair uses the same rows, reset before each variant, with no -concurrent writers to those rows. All results are verified after every -execution. Fixtures are scoped to each case and removed in a `finally` block. +Both runs use Convex JS 1.43.0. For each combination of batch size (1, 25, 100) +and document padding (0, 4096 bytes), discard five warmup pairs, then measure 50 +paired trials with alternating variant order. This gives 600 measured executions +plus 60 warmup executions. Each pair uses the same rows, reset before each +variant, with no concurrent writers to those rows. All results are verified +after every execution. Fixtures are scoped to each case and removed in a +`finally` block. Server time and database usage come from the outer mutation's Convex completion log. Time includes the snapshot query and nested mutation, but excludes fixture @@ -90,10 +129,12 @@ The harness requires a dev deployment in `.env.local`. Configuration options are documented in [example/README.md](./example/README.md#benchmark-point-reads-before-patching). Machine-readable results, action timings, and raw server logs are written to -`.context/benchmark-results/`. - -This is a warm, uncontended workload on one deployment. It does not benchmark -cold database reads, scheduler throughput, simultaneous writers, or deliberately -different snapshots. Re-fetching is needed for correctness when eligibility or -values can change, regardless of its measured overhead here. The direct-patch -variant is an intentional benchmark baseline, not general application guidance. +`.context/benchmark-parallel-results/`. The original sequential-get run is +retained in `.context/benchmark-results/`. + +Each run is a warm, uncontended workload on one deployment. It does not +benchmark cold database reads, scheduler throughput, simultaneous writers, or +deliberately different snapshots. Re-fetching is needed for correctness when +eligibility or values can change, regardless of its measured overhead here. The +direct-patch variant is an intentional benchmark baseline, not general +application guidance. diff --git a/benchmark.mjs b/benchmark.mjs index ebc570a..6281b25 100644 --- a/benchmark.mjs +++ b/benchmark.mjs @@ -1,5 +1,5 @@ #!/usr/bin/env node -// Real-deployment comparison of query values -> patch vs query IDs -> get+patch. +// Query values -> sequential patches vs query IDs -> Promise.all gets -> sequential patches. // Usage: node benchmark.mjs [--trials 50] [--batch-sizes 1,25,100] import { spawn, execFile } from "node:child_process"; import { readFileSync, mkdirSync, writeFileSync } from "node:fs"; @@ -12,7 +12,7 @@ const { values } = parseArgs({ warmups: { type: "string", default: "5" }, "batch-sizes": { type: "string", default: "1,25,100" }, "padding-bytes": { type: "string", default: "0,4096" }, - output: { type: "string", default: ".context/benchmark-results" }, + output: { type: "string", default: ".context/benchmark-parallel-results" }, }, }); const env = parseEnv(readFileSync(".env.local", "utf8")); @@ -211,6 +211,7 @@ try { runId, trials, warmups, + refetchStrategy: "Promise.all gets, sequential patches", convexVersion: JSON.parse( readFileSync("node_modules/convex/package.json", "utf8"), ).version, diff --git a/example/README.md b/example/README.md index c9be6f5..3071dee 100644 --- a/example/README.md +++ b/example/README.md @@ -40,10 +40,11 @@ node benchmark.mjs ``` This compares a snapshot query returning `{ id, value }` followed by direct -patches against a snapshot query returning IDs followed by `db.get`, an -eligibility check, and the same patches. Both paths use sequential patches. Only -the fields needed for processing are passed through the batch; optional document -padding measures the effect of larger stored rows. +patches against a snapshot query returning IDs followed by `Promise.all` over +the `db.get` calls, eligibility checks, and the same patches. Both paths use +sequential patches in a `for` loop. Only the fields needed for processing are +passed through the batch; optional document padding measures the effect of +larger stored rows. The default run measures 50 paired trials after five warmup pairs for batches of 1, 25, and 100 rows, with 0 or 4096 padding bytes per row. Trial order @@ -53,11 +54,11 @@ outside the measured transaction. Fixtures live in a separate table and are deleted after each case. All benchmark functions are internal. Results and raw server completion logs are saved to -`.context/benchmark-results/`. Server execution time includes the snapshot query -and nested worker mutation, but excludes scheduling and network latency to the -CLI. The action's `runMutation` round-trip timing is also recorded. This -measures the overhead of re-fetching when rows have not changed; it does not -simulate stale snapshots or measure conflict retries under load. +`.context/benchmark-parallel-results/`. Server execution time includes the +snapshot query and nested worker mutation, but excludes scheduling and network +latency to the CLI. The action's `runMutation` round-trip timing is also +recorded. This measures the overhead of re-fetching when rows have not changed; +it does not simulate stale snapshots or measure conflict retries under load. For a shorter run or different batch sizes: diff --git a/example/convex/benchmark.ts b/example/convex/benchmark.ts index 5e2a147..0c8aefc 100644 --- a/example/convex/benchmark.ts +++ b/example/convex/benchmark.ts @@ -114,13 +114,15 @@ export const refetch = internalMutation({ args: { ids: vIds }, returns: v.number(), handler: async (ctx, { ids }) => { + // Fetch all candidates together, as in the application examples. Keep + // patches sequential and in the same order as the direct-patch baseline. + const rows = await Promise.all( + ids.map((id) => ctx.db.get("benchmarkItems", id)), + ); let processed = 0; - // Match the baseline's sequential patch order, adding one point read and - // eligibility check per row. Both paths write exactly the same fields. - for (const id of ids) { - const row = await ctx.db.get("benchmarkItems", id); + for (const row of rows) { if (!row || row.processed) continue; - await ctx.db.patch("benchmarkItems", id, { + await ctx.db.patch("benchmarkItems", row._id, { processed: true, result: row.value + 1, }); From 93b2ab5138561d21112e25ce3fe0babc1e63a15a Mon Sep 17 00:00:00 2001 From: Ian Macartney <366683+ianmacartney@users.noreply.github.com> Date: Fri, 18 Sep 2026 00:21:24 -0700 Subject: [PATCH 3/5] perf: compare parallel patches with parallel reads and patches --- BENCHMARK.md | 65 ++++++++++++++++++++++++++++++------- benchmark.mjs | 10 ++++-- example/README.md | 13 ++++---- example/convex/benchmark.ts | 41 +++++++++++++---------- 4 files changed, 91 insertions(+), 38 deletions(-) diff --git a/BENCHMARK.md b/BENCHMARK.md index a61b923..1267e64 100644 --- a/BENCHMARK.md +++ b/BENCHMARK.md @@ -4,19 +4,61 @@ This benchmark compares two ways of processing a batch when no rows change between selection and processing: - **Direct patch:** a snapshot query returns `{ id, value }`; the mutation - trusts that value and patches `{ processed: true, result: value + 1 }`. + trusts that value and uses `Promise.all` to patch all rows with + `{ processed: true, result: value + 1 }`. - **Re-fetch:** a snapshot query returns IDs; the mutation fetches the rows with `Promise.all(ids.map(id => ctx.db.get(...)))`, then checks eligibility and - applies the same patches in a sequential `for` loop using current values. + applies the same patches with another `Promise.all` using current values. Both variants scan the same index, use `ctx.runQuery` with `useStaleSnapshot: true`, and invoke the worker through `ctx.runMutation` in the same outer mutation, matching the component's query/mutation calling pattern. -Both use sequential patches. Only the gets are parallelized, matching the -application examples. Queries pass only fields needed for processing, so -artificial document padding is not passed in mutation arguments. +Both variants issue patches concurrently. The re-fetch variant awaits all gets +before validating rows and issuing its patches. This gives the direct-patch +baseline the opportunity to fetch its target rows concurrently too, removing the +sequential-write bottleneck in the previous comparison. Queries pass only fields +needed for processing, so artificial document padding is not passed in mutation +arguments. + +## Results: parallel patches in both variants + +Measured on 2026-09-18 on dev `enchanted-cardinal-63`. Both variants were +measured afresh with `Promise.all` for writes. The re-fetch variant first awaits +`Promise.all` gets, checks eligibility, then awaits `Promise.all` patches. All +600 measured executions produced the expected results, with no reported OCC +retries or execution errors. The benchmark table was confirmed empty after +cleanup. Times are milliseconds per complete query-plus-mutation transaction. + +| Rows per batch | Padding per row | Parallel patches mean | Parallel gets + patches mean | Mean difference | Paired 95% interval | +| -------------: | --------------: | --------------------: | ---------------------------: | --------------: | ------------------: | +| 1 | 0 B | 15.96 | 15.31 | -0.65 | -3.77 to +2.46 | +| 25 | 0 B | 69.34 | 25.10 | -44.24 | -47.61 to -40.86 | +| 100 | 0 B | 261.00 | 65.25 | -195.75 | -248.52 to -142.99 | +| 1 | 4096 B | 13.07 | 13.53 | +0.46 | -0.38 to +1.30 | +| 25 | 4096 B | 73.03 | 29.56 | -43.47 | -48.50 to -38.44 | +| 100 | 4096 B | 256.23 | 78.16 | -178.08 | -187.88 to -168.27 | + +Explicit parallel gets reduced mean server time by **60–64% for 25 rows** and +**69–75% for 100 rows** compared with issuing direct patches in parallel. All +four paired intervals exclude zero. Single-row cases showed no clear difference. +Medians also favor re-fetching: for 100 rows they were 225.44 ms versus 62.69 ms +without padding, and 249.20 ms versus 77.82 ms with 4096 bytes of padding. + +Both implementations issue all patch calls together. This result measures the +effect of explicit pre-fetching with that API call pattern; it does not +establish how the backend schedules the patches internally or isolate the +remaining latency difference to a particular cache or database operation. +Database usage is unchanged from the previous runs, as shown below. + +Run ID: `79cc3d5a-a89f-43ef-95e1-9b774fc46beb`. Raw evidence is in +`.context/benchmark-parallel-patches-results/results.json`, `cases.json`, and +`logs.jsonl`. + +## Earlier results: parallel gets, sequential patches -## Results: parallel gets, sequential patches +This run used sequential patches in both variants, as implemented in commit +`97ab116`. Its speedup does not establish the overhead of re-fetching compared +with direct patches that also run in parallel. Measured on 2026-09-18 UTC (2026-09-17 PDT) on dev `enchanted-cardinal-63`. Both variants were measured afresh in this run. All 600 measured executions produced @@ -79,8 +121,8 @@ Run ID: `478bc6c6-a4f9-40f6-bd71-1f6c15fead8e`. Raw evidence is in ## Database usage -The read-document, read-byte, and write-byte measurements were identical in the -parallel-get and sequential-get runs, and deterministic across trials: +The read-document, read-byte, and write-byte measurements were identical in all +three runs, and deterministic across trials: | Rows per batch | Padding per row | Direct patch read docs | Re-fetch read docs | Direct patch read bytes | Re-fetch read bytes | Write bytes, either variant | | -------------: | --------------: | ---------------------: | -----------------: | ----------------------: | ------------------: | --------------------------: | @@ -98,7 +140,7 @@ latency. Byte counts include stored row fields and metadata, not just padding. ## Method -Both runs use Convex JS 1.43.0. For each combination of batch size (1, 25, 100) +All runs use Convex JS 1.43.0. For each combination of batch size (1, 25, 100) and document padding (0, 4096 bytes), discard five warmup pairs, then measure 50 paired trials with alternating variant order. This gives 600 measured executions plus 60 warmup executions. Each pair uses the same rows, reset before each @@ -129,8 +171,9 @@ The harness requires a dev deployment in `.env.local`. Configuration options are documented in [example/README.md](./example/README.md#benchmark-point-reads-before-patching). Machine-readable results, action timings, and raw server logs are written to -`.context/benchmark-parallel-results/`. The original sequential-get run is -retained in `.context/benchmark-results/`. +`.context/benchmark-parallel-patches-results/`. Earlier runs are retained in +`.context/benchmark-parallel-results/` (parallel gets, sequential patches) and +`.context/benchmark-results/` (sequential gets and patches). Each run is a warm, uncontended workload on one deployment. It does not benchmark cold database reads, scheduler throughput, simultaneous writers, or diff --git a/benchmark.mjs b/benchmark.mjs index 6281b25..9cda358 100644 --- a/benchmark.mjs +++ b/benchmark.mjs @@ -1,5 +1,5 @@ #!/usr/bin/env node -// Query values -> sequential patches vs query IDs -> Promise.all gets -> sequential patches. +// Query values -> Promise.all patches vs query IDs -> Promise.all gets -> Promise.all patches. // Usage: node benchmark.mjs [--trials 50] [--batch-sizes 1,25,100] import { spawn, execFile } from "node:child_process"; import { readFileSync, mkdirSync, writeFileSync } from "node:fs"; @@ -12,7 +12,10 @@ const { values } = parseArgs({ warmups: { type: "string", default: "5" }, "batch-sizes": { type: "string", default: "1,25,100" }, "padding-bytes": { type: "string", default: "0,4096" }, - output: { type: "string", default: ".context/benchmark-parallel-results" }, + output: { + type: "string", + default: ".context/benchmark-parallel-patches-results", + }, }, }); const env = parseEnv(readFileSync(".env.local", "utf8")); @@ -211,7 +214,8 @@ try { runId, trials, warmups, - refetchStrategy: "Promise.all gets, sequential patches", + patchStrategy: "Promise.all patches", + refetchStrategy: "Promise.all gets, then Promise.all patches", convexVersion: JSON.parse( readFileSync("node_modules/convex/package.json", "utf8"), ).version, diff --git a/example/README.md b/example/README.md index 3071dee..8eb3d44 100644 --- a/example/README.md +++ b/example/README.md @@ -42,9 +42,10 @@ node benchmark.mjs This compares a snapshot query returning `{ id, value }` followed by direct patches against a snapshot query returning IDs followed by `Promise.all` over the `db.get` calls, eligibility checks, and the same patches. Both paths use -sequential patches in a `for` loop. Only the fields needed for processing are -passed through the batch; optional document padding measures the effect of -larger stored rows. +`Promise.all` for the patches, so the baseline's writes are also concurrent. The +re-fetch variant awaits all gets before starting its parallel patches. Only the +fields needed for processing are passed through the batch; optional document +padding measures the effect of larger stored rows. The default run measures 50 paired trials after five warmup pairs for batches of 1, 25, and 100 rows, with 0 or 4096 padding bytes per row. Trial order @@ -54,9 +55,9 @@ outside the measured transaction. Fixtures live in a separate table and are deleted after each case. All benchmark functions are internal. Results and raw server completion logs are saved to -`.context/benchmark-parallel-results/`. Server execution time includes the -snapshot query and nested worker mutation, but excludes scheduling and network -latency to the CLI. The action's `runMutation` round-trip timing is also +`.context/benchmark-parallel-patches-results/`. Server execution time includes +the snapshot query and nested worker mutation, but excludes scheduling and +network latency to the CLI. The action's `runMutation` round-trip timing is also recorded. This measures the overhead of re-fetching when rows have not changed; it does not simulate stale snapshots or measure conflict retries under load. diff --git a/example/convex/benchmark.ts b/example/convex/benchmark.ts index 0c8aefc..0aaa0b0 100644 --- a/example/convex/benchmark.ts +++ b/example/convex/benchmark.ts @@ -100,12 +100,14 @@ export const patch = internalMutation({ handler: async (ctx, { items }) => { // Intentionally assumes the query's values and eligibility are unchanged. // This is the comparison baseline, not the recommended general pattern. - for (const { id, value } of items) { - await ctx.db.patch("benchmarkItems", id, { - processed: true, - result: value + 1, - }); - } + await Promise.all( + items.map(({ id, value }) => + ctx.db.patch("benchmarkItems", id, { + processed: true, + result: value + 1, + }), + ), + ); return items.length; }, }); @@ -114,21 +116,24 @@ export const refetch = internalMutation({ args: { ids: vIds }, returns: v.number(), handler: async (ctx, { ids }) => { - // Fetch all candidates together, as in the application examples. Keep - // patches sequential and in the same order as the direct-patch baseline. + // Fetch all candidates together, then issue all eligible patches together. + // Both variants use Promise.all for writes, so the direct-patch baseline + // also has the opportunity to fetch its target rows concurrently. const rows = await Promise.all( ids.map((id) => ctx.db.get("benchmarkItems", id)), ); - let processed = 0; - for (const row of rows) { - if (!row || row.processed) continue; - await ctx.db.patch("benchmarkItems", row._id, { - processed: true, - result: row.value + 1, - }); - processed++; - } - return processed; + const eligible = rows.filter( + (row): row is NonNullable => row !== null && !row.processed, + ); + await Promise.all( + eligible.map((row) => + ctx.db.patch("benchmarkItems", row._id, { + processed: true, + result: row.value + 1, + }), + ), + ); + return eligible.length; }, }); From 5fdba092db325b089e20cf931e85f6a40a89e332 Mon Sep 17 00:00:00 2001 From: Ian Macartney <366683+ianmacartney@users.noreply.github.com> Date: Fri, 18 Sep 2026 19:13:06 -0700 Subject: [PATCH 4/5] perf: benchmark per-row get check and patch in one Promise.all --- BENCHMARK.md | 89 ++++++++++++++++++++++++++++++------- benchmark.mjs | 6 +-- example/README.md | 18 ++++---- example/convex/benchmark.ts | 26 +++++------ 4 files changed, 96 insertions(+), 43 deletions(-) diff --git a/BENCHMARK.md b/BENCHMARK.md index 1267e64..b4f8d7d 100644 --- a/BENCHMARK.md +++ b/BENCHMARK.md @@ -6,21 +6,76 @@ between selection and processing: - **Direct patch:** a snapshot query returns `{ id, value }`; the mutation trusts that value and uses `Promise.all` to patch all rows with `{ processed: true, result: value + 1 }`. -- **Re-fetch:** a snapshot query returns IDs; the mutation fetches the rows with - `Promise.all(ids.map(id => ctx.db.get(...)))`, then checks eligibility and - applies the same patches with another `Promise.all` using current values. +- **Re-fetch:** a snapshot query returns IDs; one `Promise.all` wraps async + tasks that each get a row, check eligibility, and apply the same patch using + its current value. + +The re-fetch mutation processes each ID like this: + +```ts +const processed = await Promise.all( + ids.map(async (id) => { + const row = await ctx.db.get("benchmarkItems", id); + if (!row || row.processed) return 0; + await ctx.db.patch("benchmarkItems", id, { + processed: true, + result: row.value + 1, + }); + return 1; + }), +); +return processed.reduce((sum, count) => sum + count, 0); +``` Both variants scan the same index, use `ctx.runQuery` with `useStaleSnapshot: true`, and invoke the worker through `ctx.runMutation` in the same outer mutation, matching the component's query/mutation calling pattern. -Both variants issue patches concurrently. The re-fetch variant awaits all gets -before validating rows and issuing its patches. This gives the direct-patch -baseline the opportunity to fetch its target rows concurrently too, removing the -sequential-write bottleneck in the previous comparison. Queries pass only fields -needed for processing, so artificial document padding is not passed in mutation -arguments. +Both variants issue work concurrently. Each re-fetch task can patch as soon as +its own get and check finish; there is no barrier waiting for all gets to +complete. Queries pass only fields needed for processing, so artificial document +padding is not passed in mutation arguments. + +## Results: one Promise.all over get, check, and patch + +Measured on 2026-09-19 UTC (2026-09-18 PDT) on dev `enchanted-cardinal-63`. Both +variants were measured afresh. All 600 measured executions produced the expected +results, with no reported OCC retries or execution errors. The benchmark table +was confirmed empty after cleanup. Times are milliseconds per complete +query-plus-mutation transaction. + +| Rows per batch | Padding per row | Parallel patches mean | Per-row get/check/patch mean | Mean difference | Paired 95% interval | +| -------------: | --------------: | --------------------: | ---------------------------: | --------------: | ------------------: | +| 1 | 0 B | 13.19 | 13.22 | +0.03 | -0.95 to +1.01 | +| 25 | 0 B | 67.52 | 28.27 | -39.26 | -46.34 to -32.17 | +| 100 | 0 B | 263.39 | 74.57 | -188.82 | -221.30 to -156.34 | +| 1 | 4096 B | 15.06 | 15.58 | +0.52 | -1.44 to +2.49 | +| 25 | 4096 B | 84.73 | 28.79 | -55.94 | -69.30 to -42.57 | +| 100 | 4096 B | 256.46 | 83.98 | -172.48 | -187.61 to -157.35 | + +Re-fetching reduced mean server time by **58–66% for 25 rows** and **67–72% for +100 rows** in this run. All four paired intervals exclude zero. Single-row cases +showed no clear difference. For 100 rows, medians were 241.65 ms versus 66.91 ms +without padding and 244.84 ms versus 80.35 ms with padding. The separately +recorded action round-trip timings also favor re-fetching for all multi-row +cases. + +The lower latency persists when each row's get, check, and patch share a single +async callback. It therefore does not require an explicit barrier waiting for +every get before starting any patches. Re-fetching still adds an explicit get +and one reported document read per row, with identical writes. These +measurements do not explain why the additional work has lower latency: they do +not identify the backend's cache behavior, actual I/O, or internal scheduling. +The earlier two-phase version was measured in a separate run, so the difference +between its timings and these timings is not a paired comparison of the two +re-fetch arrangements. + +Run ID: `9f50beef-4b63-4bea-8290-b0c6561a6a90`. Raw evidence is in +`.context/benchmark-interleaved-results/results.json`, `cases.json`, and +`logs.jsonl`. + +## Earlier results: all parallel gets, then all parallel patches -## Results: parallel patches in both variants +This run used the two-phase implementation in commit `93b2ab5`. Measured on 2026-09-18 on dev `enchanted-cardinal-63`. Both variants were measured afresh with `Promise.all` for writes. The re-fetch variant first awaits @@ -57,7 +112,7 @@ Run ID: `79cc3d5a-a89f-43ef-95e1-9b774fc46beb`. Raw evidence is in ## Earlier results: parallel gets, sequential patches This run used sequential patches in both variants, as implemented in commit -`97ab116`. Its speedup does not establish the overhead of re-fetching compared +`a3d37cc`. Its speedup does not establish the overhead of re-fetching compared with direct patches that also run in parallel. Measured on 2026-09-18 UTC (2026-09-17 PDT) on dev `enchanted-cardinal-63`. Both @@ -91,7 +146,7 @@ Run ID: `5b565bfd-2962-44b6-a408-856f1ef7b942`. Raw evidence is in The results below used the original implementation on dev `first-lobster-65`, which awaited one get and one patch at a time. They do not measure `Promise.all` -gets. That implementation is available in commit `066836b`. Since the parallel +gets. That implementation is available in commit `c8a3d5c`. Since the parallel run uses a different deployment, compare each re-fetch variant with its own direct-patch baseline; these runs do not isolate the speedup from changing sequential gets to parallel gets. @@ -122,7 +177,7 @@ Run ID: `478bc6c6-a4f9-40f6-bd71-1f6c15fead8e`. Raw evidence is in ## Database usage The read-document, read-byte, and write-byte measurements were identical in all -three runs, and deterministic across trials: +four runs, and deterministic across trials: | Rows per batch | Padding per row | Direct patch read docs | Re-fetch read docs | Direct patch read bytes | Re-fetch read bytes | Write bytes, either variant | | -------------: | --------------: | ---------------------: | -----------------: | ----------------------: | ------------------: | --------------------------: | @@ -171,9 +226,11 @@ The harness requires a dev deployment in `.env.local`. Configuration options are documented in [example/README.md](./example/README.md#benchmark-point-reads-before-patching). Machine-readable results, action timings, and raw server logs are written to -`.context/benchmark-parallel-patches-results/`. Earlier runs are retained in -`.context/benchmark-parallel-results/` (parallel gets, sequential patches) and -`.context/benchmark-results/` (sequential gets and patches). +`.context/benchmark-interleaved-results/`. Earlier runs are retained in +`.context/benchmark-parallel-patches-results/` (all parallel gets, then all +parallel patches), `.context/benchmark-parallel-results/` (parallel gets, +sequential patches), and `.context/benchmark-results/` (sequential gets and +patches). Each run is a warm, uncontended workload on one deployment. It does not benchmark cold database reads, scheduler throughput, simultaneous writers, or diff --git a/benchmark.mjs b/benchmark.mjs index 9cda358..960def4 100644 --- a/benchmark.mjs +++ b/benchmark.mjs @@ -1,5 +1,5 @@ #!/usr/bin/env node -// Query values -> Promise.all patches vs query IDs -> Promise.all gets -> Promise.all patches. +// Query values -> Promise.all patches vs query IDs -> Promise.all(get -> check -> patch). // Usage: node benchmark.mjs [--trials 50] [--batch-sizes 1,25,100] import { spawn, execFile } from "node:child_process"; import { readFileSync, mkdirSync, writeFileSync } from "node:fs"; @@ -14,7 +14,7 @@ const { values } = parseArgs({ "padding-bytes": { type: "string", default: "0,4096" }, output: { type: "string", - default: ".context/benchmark-parallel-patches-results", + default: ".context/benchmark-interleaved-results", }, }, }); @@ -215,7 +215,7 @@ try { trials, warmups, patchStrategy: "Promise.all patches", - refetchStrategy: "Promise.all gets, then Promise.all patches", + refetchStrategy: "Promise.all of per-row get, check, then patch", convexVersion: JSON.parse( readFileSync("node_modules/convex/package.json", "utf8"), ).version, diff --git a/example/README.md b/example/README.md index 8eb3d44..ffb61f6 100644 --- a/example/README.md +++ b/example/README.md @@ -40,12 +40,12 @@ node benchmark.mjs ``` This compares a snapshot query returning `{ id, value }` followed by direct -patches against a snapshot query returning IDs followed by `Promise.all` over -the `db.get` calls, eligibility checks, and the same patches. Both paths use -`Promise.all` for the patches, so the baseline's writes are also concurrent. The -re-fetch variant awaits all gets before starting its parallel patches. Only the -fields needed for processing are passed through the batch; optional document -padding measures the effect of larger stored rows. +patches against a snapshot query returning IDs followed by one `Promise.all` +over async tasks that each get a row, check eligibility, and patch it. Each task +can patch its row as soon as its get finishes; there is no wait for all gets to +complete before patches start. The baseline also uses `Promise.all` for its +patches. Only the fields needed for processing are passed through the batch; +optional document padding measures the effect of larger stored rows. The default run measures 50 paired trials after five warmup pairs for batches of 1, 25, and 100 rows, with 0 or 4096 padding bytes per row. Trial order @@ -55,9 +55,9 @@ outside the measured transaction. Fixtures live in a separate table and are deleted after each case. All benchmark functions are internal. Results and raw server completion logs are saved to -`.context/benchmark-parallel-patches-results/`. Server execution time includes -the snapshot query and nested worker mutation, but excludes scheduling and -network latency to the CLI. The action's `runMutation` round-trip timing is also +`.context/benchmark-interleaved-results/`. Server execution time includes the +snapshot query and nested worker mutation, but excludes scheduling and network +latency to the CLI. The action's `runMutation` round-trip timing is also recorded. This measures the overhead of re-fetching when rows have not changed; it does not simulate stale snapshots or measure conflict retries under load. diff --git a/example/convex/benchmark.ts b/example/convex/benchmark.ts index 0aaa0b0..2cf9c8a 100644 --- a/example/convex/benchmark.ts +++ b/example/convex/benchmark.ts @@ -116,24 +116,20 @@ export const refetch = internalMutation({ args: { ids: vIds }, returns: v.number(), handler: async (ctx, { ids }) => { - // Fetch all candidates together, then issue all eligible patches together. - // Both variants use Promise.all for writes, so the direct-patch baseline - // also has the opportunity to fetch its target rows concurrently. - const rows = await Promise.all( - ids.map((id) => ctx.db.get("benchmarkItems", id)), - ); - const eligible = rows.filter( - (row): row is NonNullable => row !== null && !row.processed, - ); - await Promise.all( - eligible.map((row) => - ctx.db.patch("benchmarkItems", row._id, { + // Each concurrent task gets, checks, and patches one row. A patch can start + // as soon as that row is ready, without waiting for every get to finish. + const processed = await Promise.all( + ids.map(async (id) => { + const row = await ctx.db.get("benchmarkItems", id); + if (!row || row.processed) return 0; + await ctx.db.patch("benchmarkItems", id, { processed: true, result: row.value + 1, - }), - ), + }); + return 1; + }), ); - return eligible.length; + return processed.reduce((sum, count) => sum + count, 0); }, }); From c56ad687f48ce82f54a4c06bd8b0f81eeb52333a Mon Sep 17 00:00:00 2001 From: Ian Macartney <366683+ianmacartney@users.noreply.github.com> Date: Fri, 18 Sep 2026 20:10:18 -0700 Subject: [PATCH 5/5] perf: control query payload in refetch benchmark --- BENCHMARK.md | 94 ++++++++++++++++++++++++++++++++----- benchmark.mjs | 68 +++++++++++++++++---------- example/README.md | 36 +++++++------- example/convex/benchmark.ts | 81 ++++++++++++++++++++++---------- 4 files changed, 199 insertions(+), 80 deletions(-) diff --git a/BENCHMARK.md b/BENCHMARK.md index b4f8d7d..5103bcd 100644 --- a/BENCHMARK.md +++ b/BENCHMARK.md @@ -1,6 +1,6 @@ # Re-fetching before patching -This benchmark compares two ways of processing a batch when no rows change +This benchmark compares three ways of processing a batch when no rows change between selection and processing: - **Direct patch:** a snapshot query returns `{ id, value }`; the mutation @@ -9,6 +9,13 @@ between selection and processing: - **Re-fetch:** a snapshot query returns IDs; one `Promise.all` wraps async tasks that each get a row, check eligibility, and apply the same patch using its current value. +- **Re-fetch with values:** the same query and `{ items: [{ id, value }] }` + mutation arguments as direct patch, but supplied values are ignored. It uses + the same re-fetch helper as the IDs-only variant. This control keeps the query + return values and mutation arguments identical to direct patch. + +None of these variants passes full documents. Even the original direct-patch +baseline passed only `{ id, value }`; document padding stays in the database. The re-fetch mutation processes each ID like this: @@ -27,15 +34,68 @@ const processed = await Promise.all( return processed.reduce((sum, count) => sum + count, 0); ``` -Both variants scan the same index, use `ctx.runQuery` with +All variants scan the same index, use `ctx.runQuery` with `useStaleSnapshot: true`, and invoke the worker through `ctx.runMutation` in the same outer mutation, matching the component's query/mutation calling pattern. -Both variants issue work concurrently. Each re-fetch task can patch as soon as +All variants issue work concurrently. Each re-fetch task can patch as soon as its own get and check finish; there is no barrier waiting for all gets to complete. Queries pass only fields needed for processing, so artificial document padding is not passed in mutation arguments. -## Results: one Promise.all over get, check, and patch +## Results: control for the query and mutation payload + +Measured on 2026-09-19 UTC (2026-09-18 PDT) on dev `enchanted-cardinal-63`. All +three variants were measured together in 50 matched trials per case, with all +six execution orders cycled. All 900 measured executions produced the expected +results, with no reported OCC retries or execution errors. The benchmark table +was confirmed empty after cleanup. Means below are milliseconds per complete +query-plus-mutation transaction. + +| Rows per batch | Padding per row | Values → patch | IDs → re-fetch/patch | Values → re-fetch/patch | +| -------------: | --------------: | -------------: | -------------------: | ----------------------: | +| 1 | 0 B | 18.83 | 16.63 | 26.91 | +| 25 | 0 B | 107.17 | 40.31 | 41.98 | +| 100 | 0 B | 342.30 | 98.92 | 123.54 | +| 1 | 4096 B | 16.89 | 16.46 | 16.72 | +| 25 | 4096 B | 103.29 | 39.61 | 40.67 | +| 100 | 4096 B | 388.43 | 106.36 | 114.60 | + +Two paired comparisons test the payload hypothesis. The first holds the payload +constant and adds re-fetching. The second holds re-fetching constant and changes +the payload from IDs to `{ id, value }`. That comparison includes constructing +and validating the payload and extracting IDs in the control mutation; it is not +an isolated serialization measurement: + +| Rows | Padding | Re-fetch minus patch, identical payload (mean) | Paired 95% interval | Values minus IDs, both re-fetch (mean) | Paired 95% interval | +| ---: | ------: | ---------------------------------------------: | ------------------: | -------------------------------------: | ------------------: | +| 1 | 0 B | +8.08 | -3.38 to +19.54 | +10.28 | -1.82 to +22.38 | +| 25 | 0 B | -65.19 | -77.92 to -52.47 | +1.67 | -4.80 to +8.15 | +| 100 | 0 B | -218.76 | -275.85 to -161.67 | +24.61 | -18.80 to +68.03 | +| 1 | 4096 B | -0.17 | -2.70 to +2.36 | +0.26 | -2.20 to +2.73 | +| 25 | 4096 B | -62.62 | -77.95 to -47.29 | +1.07 | -5.94 to +8.07 | +| 100 | 4096 B | -273.83 | -311.09 to -236.57 | +8.24 | -7.17 to +23.65 | + +With identical query results and mutation arguments, re-fetching still reduced +mean server time by **about 61% for 25 rows** and **64–71% for 100 rows**. All +four paired intervals exclude zero. The payload difference therefore does not +explain the large speedup in this workload. + +Every interval for the payload-only comparison includes zero; this run does not +establish a consistent latency penalty for passing values. It does not prove +zero overhead either. Means are noisy: at 100 rows without padding, re-fetching +with values had a mean 24.61 ms above IDs-only, but their medians were 91.55 ms +and 89.02 ms respectively. The direct-patch median was 307.50 ms. The +measurements still do not identify the backend cache, I/O, or scheduling reason +for the remaining difference. Full-document payloads are not tested here. + +Run ID: `b5b845ca-810a-422e-a541-f86d8b17ac03`. Raw evidence is in +`.context/benchmark-payload-control-results/results.json`, `cases.json`, and +`logs.jsonl`. The JSON includes all three paired comparisons and action timings. + +## Earlier results: one Promise.all over get, check, and patch + +This run compared only direct patch with IDs-only re-fetching, as implemented in +commit `5fdba09`. Measured on 2026-09-19 UTC (2026-09-18 PDT) on dev `enchanted-cardinal-63`. Both variants were measured afresh. All 600 measured executions produced the expected @@ -177,7 +237,8 @@ Run ID: `478bc6c6-a4f9-40f6-bd71-1f6c15fead8e`. Raw evidence is in ## Database usage The read-document, read-byte, and write-byte measurements were identical in all -four runs, and deterministic across trials: +five runs, and deterministic across trials. In the payload-control run, both +re-fetch variants have the same usage shown in the re-fetch columns: | Rows per batch | Padding per row | Direct patch read docs | Re-fetch read docs | Direct patch read bytes | Re-fetch read bytes | Write bytes, either variant | | -------------: | --------------: | ---------------------: | -----------------: | ----------------------: | ------------------: | --------------------------: | @@ -196,10 +257,13 @@ latency. Byte counts include stored row fields and metadata, not just padding. ## Method All runs use Convex JS 1.43.0. For each combination of batch size (1, 25, 100) -and document padding (0, 4096 bytes), discard five warmup pairs, then measure 50 -paired trials with alternating variant order. This gives 600 measured executions -plus 60 warmup executions. Each pair uses the same rows, reset before each -variant, with no concurrent writers to those rows. All results are verified +and document padding (0, 4096 bytes), discard five warmup trials, then measure +50 matched trials. The current payload-control run executes all three variants +in each trial, cycling through all six orders to balance position and +preceding-variant effects. This gives 900 measured executions plus 90 warmup +executions. Earlier runs used two variants in alternating order, giving 600 +measured executions plus 60 warmups. Each trial uses the same rows, reset before +each variant, with no concurrent writers to those rows. All results are verified after every execution. Fixtures are scoped to each case and removed in a `finally` block. @@ -210,9 +274,12 @@ and network latency to the CLI. The harness separately records the action's `runMutation` round-trip time. No timers run inside mutations, where `Date.now()` is fixed. -Paired differences are re-fetch minus direct patch. The reported 95% intervals -use a normal approximation from the standard error of the paired differences; -they describe variation within this run, not variation across deployments. +Paired differences are computed within each trial. Comparisons are re-fetch +minus direct patch, re-fetch with values minus direct patch (identical +payloads), and re-fetch with values minus IDs-only re-fetch (identical worker +logic). The reported 95% intervals use a normal approximation from the standard +error of the paired differences; they describe variation within this run, not +variation across deployments. ## Reproduce @@ -226,7 +293,8 @@ The harness requires a dev deployment in `.env.local`. Configuration options are documented in [example/README.md](./example/README.md#benchmark-point-reads-before-patching). Machine-readable results, action timings, and raw server logs are written to -`.context/benchmark-interleaved-results/`. Earlier runs are retained in +`.context/benchmark-payload-control-results/`. Earlier runs are retained in +`.context/benchmark-interleaved-results/` (per-row get/check/patch with IDs), `.context/benchmark-parallel-patches-results/` (all parallel gets, then all parallel patches), `.context/benchmark-parallel-results/` (parallel gets, sequential patches), and `.context/benchmark-results/` (sequential gets and diff --git a/benchmark.mjs b/benchmark.mjs index 960def4..7ee3312 100644 --- a/benchmark.mjs +++ b/benchmark.mjs @@ -1,5 +1,5 @@ #!/usr/bin/env node -// Query values -> Promise.all patches vs query IDs -> Promise.all(get -> check -> patch). +// Compare parallel patches with per-row get/check/patch, controlling query payload. // Usage: node benchmark.mjs [--trials 50] [--batch-sizes 1,25,100] import { spawn, execFile } from "node:child_process"; import { readFileSync, mkdirSync, writeFileSync } from "node:fs"; @@ -14,7 +14,7 @@ const { values } = parseArgs({ "padding-bytes": { type: "string", default: "0,4096" }, output: { type: "string", - default: ".context/benchmark-interleaved-results", + default: ".context/benchmark-payload-control-results", }, }, }); @@ -47,8 +47,9 @@ const exec = promisify(execFile); const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); const events = []; const cases = []; +const modes = ["patch", "refetch", "refetchValues"]; const tagPattern = - /batch-read-bench\|([\w-]+)\|(\d+)\|(\d+)\|(-?\d+)\|(patch|refetch)\|(true|false)/; + /batch-read-bench\|([\w-]+)\|(\d+)\|(\d+)\|(-?\d+)\|(patch|refetch|refetchValues)\|(true|false)/; function completions() { const unique = new Map(); @@ -96,8 +97,25 @@ function summarize(samples) { }; } +function pairedDifference(a, b) { + const diffs = a.map((e, i) => (e.executionTime - b[i].executionTime) * 1000); + const mean = stats(diffs).mean; + const se = + trials > 1 + ? Math.sqrt( + diffs.reduce((sum, d) => sum + (d - mean) ** 2, 0) / + (trials - 1) / + trials, + ) + : null; + return { + mean, + normalApprox95CI: se === null ? null : [mean - 1.96 * se, mean + 1.96 * se], + }; +} + console.log( - `Target: dev (${deployment}); ${trials} paired trials, ${warmups} warmup pairs per case.`, + `Target: dev (${deployment}); ${trials} matched trials, ${warmups} warmup trials per case; ${modes.length} variants per trial.`, ); const logProcess = spawn( process.execPath, @@ -147,7 +165,7 @@ try { ); } } - const expected = cases.length * trials * 2; + const expected = cases.length * trials * modes.length; const deadline = Date.now() + 30000; while ( completions().filter((e) => !e.warmup && !e.error && !e.willRetry).length < @@ -170,34 +188,26 @@ try { const refetch = samples .filter((e) => e.mode === "refetch") .sort((a, b) => a.trial - b.trial); - for (const rows of [patch, refetch]) { + const refetchValues = samples + .filter((e) => e.mode === "refetchValues") + .sort((a, b) => a.trial - b.trial); + for (const rows of [patch, refetch, refetchValues]) { if (rows.length !== trials || rows.some((e, i) => e.trial !== i)) throw new Error("Missing or duplicated trial logs"); } - const diffs = refetch.map( - (e, i) => (e.executionTime - patch[i].executionTime) * 1000, - ); - const mean = stats(diffs).mean; - const se = - trials > 1 - ? Math.sqrt( - diffs.reduce((sum, d) => sum + (d - mean) ** 2, 0) / - (trials - 1) / - trials, - ) - : null; return { batchSize: spec.batchSize, paddingBytes: spec.paddingBytes, patch: summarize(patch), refetch: summarize(refetch), - pairedDifferenceMs: { - mean, - normalApprox95CI: - se === null ? null : [mean - 1.96 * se, mean + 1.96 * se], + refetchValues: summarize(refetchValues), + pairedDifferencesMs: { + refetchMinusPatch: pairedDifference(refetch, patch), + refetchValuesMinusPatch: pairedDifference(refetchValues, patch), + refetchValuesMinusRefetch: pairedDifference(refetchValues, refetch), }, actionRoundTripMs: Object.fromEntries( - ["patch", "refetch"].map((mode) => [ + modes.map((mode) => [ mode, stats( spec.actionSamples @@ -214,8 +224,16 @@ try { runId, trials, warmups, + modes, + trialOrder: "Cycle all six permutations of the three variants", patchStrategy: "Promise.all patches", refetchStrategy: "Promise.all of per-row get, check, then patch", + payloads: { + patch: "Query { id, value }[]; mutation { items }", + refetch: "Query Id[]; mutation { ids }", + refetchValues: + "Query { id, value }[]; mutation { items }, ignore supplied values and re-fetch", + }, convexVersion: JSON.parse( readFileSync("node_modules/convex/package.json", "utf8"), ).version, @@ -230,8 +248,8 @@ try { batch: r.batchSize, padding: r.paddingBytes, "patch mean ms": r.patch.executionMs.mean.toFixed(3), - "refetch mean ms": r.refetch.executionMs.mean.toFixed(3), - "difference ms": r.pairedDifferenceMs.mean.toFixed(3), + "IDs refetch mean ms": r.refetch.executionMs.mean.toFixed(3), + "values refetch mean ms": r.refetchValues.executionMs.mean.toFixed(3), "patch reads": r.patch.readDocuments.mean, "refetch reads": r.refetch.readDocuments.mean, })), diff --git a/example/README.md b/example/README.md index ffb61f6..8c39277 100644 --- a/example/README.md +++ b/example/README.md @@ -39,25 +39,27 @@ npx convex dev --once node benchmark.mjs ``` -This compares a snapshot query returning `{ id, value }` followed by direct -patches against a snapshot query returning IDs followed by one `Promise.all` -over async tasks that each get a row, check eligibility, and patch it. Each task -can patch its row as soon as its get finishes; there is no wait for all gets to -complete before patches start. The baseline also uses `Promise.all` for its -patches. Only the fields needed for processing are passed through the batch; -optional document padding measures the effect of larger stored rows. - -The default run measures 50 paired trials after five warmup pairs for batches of -1, 25, and 100 rows, with 0 or 4096 padding bytes per row. Trial order -alternates. There are no concurrent writers, and every trial verifies that all -rows received identical results. Setup, reset, verification, and cleanup are -outside the measured transaction. Fixtures live in a separate table and are -deleted after each case. All benchmark functions are internal. +This compares three variants: `{ id, value }` followed by direct patches, IDs +followed by re-fetch/check/patch, and `{ id, value }` followed by the same +re-fetch/check/patch logic. The third variant controls for query return values +and mutation arguments: it receives the same payload as direct patch but ignores +the supplied values. Re-fetching uses one `Promise.all` over async tasks that +each get a row, check eligibility, and patch it. The baseline also patches with +`Promise.all`. Full documents and artificial padding are never passed through +the batch; padding only changes the size of stored rows. + +The default run measures 50 matched trials after five warmup trials for batches +of 1, 25, and 100 rows, with 0 or 4096 padding bytes per row. Each trial runs +all three variants, cycling through all six orders, for 900 measured executions. +There are no concurrent writers, and every trial verifies that all rows received +identical results. Setup, reset, verification, and cleanup are outside the +measured transaction. Fixtures live in a separate table and are deleted after +each case. All benchmark functions are internal. Results and raw server completion logs are saved to -`.context/benchmark-interleaved-results/`. Server execution time includes the -snapshot query and nested worker mutation, but excludes scheduling and network -latency to the CLI. The action's `runMutation` round-trip timing is also +`.context/benchmark-payload-control-results/`. Server execution time includes +the snapshot query and nested worker mutation, but excludes scheduling and +network latency to the CLI. The action's `runMutation` round-trip timing is also recorded. This measures the overhead of re-fetching when rows have not changed; it does not simulate stale snapshots or measure conflict retries under load. diff --git a/example/convex/benchmark.ts b/example/convex/benchmark.ts index 2cf9c8a..2567a8b 100644 --- a/example/convex/benchmark.ts +++ b/example/convex/benchmark.ts @@ -1,9 +1,10 @@ -import { v } from "convex/values"; +import { v, type Infer } from "convex/values"; import { internal } from "./_generated/api.js"; import { internalAction, internalMutation, internalQuery, + type MutationCtx, } from "./_generated/server.js"; import type { Id } from "./_generated/dataModel.js"; @@ -13,7 +14,12 @@ const vIds = v.array(v.id("benchmarkItems")); const vItems = v.array( v.object({ id: v.id("benchmarkItems"), value: v.number() }), ); -const vMode = v.union(v.literal("patch"), v.literal("refetch")); +const vMode = v.union( + v.literal("patch"), + v.literal("refetch"), + v.literal("refetchValues"), +); +type Mode = Infer; const vCase = { runId: v.string(), batchSize: v.number(), @@ -112,25 +118,39 @@ export const patch = internalMutation({ }, }); +async function refetchRows(ctx: MutationCtx, ids: Id<"benchmarkItems">[]) { + // Each concurrent task gets, checks, and patches one row. A patch can start + // as soon as that row is ready, without waiting for every get to finish. + const processed = await Promise.all( + ids.map(async (id) => { + const row = await ctx.db.get("benchmarkItems", id); + if (!row || row.processed) return 0; + await ctx.db.patch("benchmarkItems", id, { + processed: true, + result: row.value + 1, + }); + return 1; + }), + ); + return processed.reduce((sum, count) => sum + count, 0); +} + export const refetch = internalMutation({ args: { ids: vIds }, returns: v.number(), - handler: async (ctx, { ids }) => { - // Each concurrent task gets, checks, and patches one row. A patch can start - // as soon as that row is ready, without waiting for every get to finish. - const processed = await Promise.all( - ids.map(async (id) => { - const row = await ctx.db.get("benchmarkItems", id); - if (!row || row.processed) return 0; - await ctx.db.patch("benchmarkItems", id, { - processed: true, - result: row.value + 1, - }); - return 1; - }), - ); - return processed.reduce((sum, count) => sum + count, 0); - }, + handler: async (ctx, { ids }) => await refetchRows(ctx, ids), +}); + +export const refetchValues = internalMutation({ + // Same validator and payload as patch; supplied values are intentionally + // ignored so the re-fetch logic is identical to the IDs-only variant. + args: { items: vItems }, + returns: v.number(), + handler: async (ctx, { items }) => + await refetchRows( + ctx, + items.map(({ id }) => id), + ), }); export const iteration = internalMutation({ @@ -146,11 +166,16 @@ export const iteration = internalMutation({ `batch-read-bench|${runId}|${batchSize}|${paddingBytes}|${trial}|${mode}|${warmup}`, ); const args = { runId, batchSize }; - if (mode === "patch") { + if (mode === "patch" || mode === "refetchValues") { const items = await ctx.runQuery(internal.benchmark.batchValues, args, { useStaleSnapshot: true, }); - return await ctx.runMutation(internal.benchmark.patch, { items }); + return await ctx.runMutation( + mode === "patch" + ? internal.benchmark.patch + : internal.benchmark.refetchValues, + { items }, + ); } const ids = await ctx.runQuery(internal.benchmark.batchIds, args, { useStaleSnapshot: true, @@ -195,18 +220,24 @@ export const run = internalAction({ boundedInteger(trials, 1, 100); boundedInteger(warmups, 0, 20); const samples: { - mode: "patch" | "refetch"; + mode: Mode; trial: number; elapsedMs: number; }[] = []; try { const ids = await ctx.runMutation(internal.benchmark.seed, spec); + // Cycle all six orders to balance position and preceding-variant effects. + const orders: Mode[][] = [ + ["patch", "refetch", "refetchValues"], + ["refetch", "refetchValues", "patch"], + ["refetchValues", "patch", "refetch"], + ["refetchValues", "refetch", "patch"], + ["refetch", "patch", "refetchValues"], + ["patch", "refetchValues", "refetch"], + ]; for (let trial = -warmups; trial < trials; trial++) { - // Alternate which variant runs first to reduce drift/order bias. const modes = - trial % 2 === 0 - ? (["patch", "refetch"] as const) - : (["refetch", "patch"] as const); + orders[((trial % orders.length) + orders.length) % orders.length]; for (const mode of modes) { await ctx.runMutation(internal.benchmark.reset, { ids }); // Date.now advances in actions; it is fixed inside mutations. Server