diff --git a/scripts/ops/r2-weekly-archive.mjs b/scripts/ops/r2-weekly-archive.mjs index ce3566b4..d8f42fdb 100644 --- a/scripts/ops/r2-weekly-archive.mjs +++ b/scripts/ops/r2-weekly-archive.mjs @@ -23,8 +23,10 @@ * Safety contract — nothing is deleted until a replacement is PROVEN good: * 1. Consistent snapshot via SQLite's Online Backup API (no writer downtime). * 2. Local PRAGMA integrity_check. - * 3. gzip, then upload with a real signed payload hash (a truncated body is - * rejected by R2 rather than silently stored). + * 3. gzip, then upload as an S3 multipart upload — every part signed with a + * real payload hash (a truncated part is rejected by R2 rather than + * silently stored) and retried on its own, so one dropped connection + * costs one ~64 MB part, not the whole ~1.5 GB transfer. * 4. Download the object back, re-hash it, decompress it, and run * integrity_check on the DECOMPRESSED copy. That is a genuine weekly * restore drill, not a "the PUT returned 200" check. @@ -41,7 +43,6 @@ import { createWriteStream, existsSync, mkdirSync, - readFileSync, readdirSync, renameSync, rmSync, @@ -258,6 +259,201 @@ function writeStatus(statusPath, payload) { } } +/** + * Multipart upload. The single-PUT upload this replaces died mid-transfer + * twice on 2026-11-07 with `S3 PUT transport failed (UND_ERR_SOCKET)` after + * several minutes: one dropped connection voided a ~1.5 GB transfer that had + * to restart from byte zero. Parts bound the blast radius — each is signed + * with its own payload hash and retried up to 3 times, and only an + * unrecoverable part aborts the upload (AbortMultipartUpload, so no orphan + * parts linger against the bucket's storage). The completed object is + * indistinguishable from a single-PUT one: the metadata rides on + * CreateMultipartUpload, and the download-back verification in runArchive + * still proves the stored bytes match the local hash before anything prunes. + */ +const MULTIPART_PART_BYTES = 64 * 1024 * 1024; +const MULTIPART_MAX_PART_ATTEMPTS = 3; + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function xmlEscape(value) { + return String(value).replace(/&/g, "&").replace(//g, ">"); +} + +/** Streaming SHA-256 + byte count of a file, without loading it into memory. */ +async function hashFile(path) { + const hash = createHash("sha256"); + let bytes = 0; + for await (const chunk of createReadStream(path)) { + hash.update(chunk); + bytes += chunk.length; + } + return { bytes, sha256: hash.digest("hex") }; +} + +/** Read exactly `length` bytes starting at `start` — one part's worth. */ +async function readFileRange(path, start, length) { + if (length === 0) return Buffer.alloc(0); + const chunks = []; + let total = 0; + for await (const chunk of createReadStream(path, { start, end: start + length - 1 })) { + chunks.push(chunk); + total += chunk.length; + } + if (total !== length) { + throw new Error(`short read on ${path}: wanted ${length} bytes at offset ${start}, got ${total}`); + } + return Buffer.concat(chunks, total); +} + +/** + * Every failure of the multipart sequence must classify as `upload_failed`. + * Part failures already do — the signer reports them as `S3 PUT ...`, which + * `classifyFailure` maps. Create/complete failures arrive as `S3 POST ...`; + * re-prefix those so the status file keeps the reason vocabulary the + * single-PUT upload produced. + */ +function asUploadError(error, phase) { + if (typeof error?.message === "string" && error.message.startsWith("S3 PUT")) return error; + return new Error(`S3 PUT multipart ${phase} failed: ${error?.message ?? String(error)}`); +} + +async function uploadPartWithRetry({ creds, bucket, key, uploadId, partNumber, body, fetchImpl }) { + let lastError; + for (let attempt = 1; attempt <= MULTIPART_MAX_PART_ATTEMPTS; attempt += 1) { + try { + const response = await s3Request({ + method: "PUT", + creds, + bucket, + key, + query: { partNumber: String(partNumber), uploadId }, + body, + fetchImpl, + }); + const etag = response.headers?.get?.("etag"); + if (!etag) throw new Error(`S3 PUT part ${partNumber} response carried no ETag`); + return etag; + } catch (error) { + lastError = error; + if (attempt < MULTIPART_MAX_PART_ATTEMPTS) { + log(`part ${partNumber} attempt ${attempt} failed (${error.message}); retrying`); + await sleep(500 * attempt); + } + } + } + throw lastError; +} + +async function abortMultipartUpload({ creds, bucket, key, uploadId, fetchImpl }) { + try { + await s3Request({ + method: "DELETE", + creds, + bucket, + key, + query: { uploadId }, + expectStatuses: [200, 204, 404], + fetchImpl, + }); + } catch (error) { + // Best effort: a stranded upload costs a little storage until the bucket + // lifecycle reaps it, but masking the real upload failure would cost the + // diagnosis, so the original error is the one that propagates. + log(`WARNING: could not abort multipart upload ${uploadId}: ${error.message}`); + } +} + +export async function uploadMultipart({ + creds, + bucket, + key, + filePath, + fileBytes, + headers = {}, + fetchImpl, + partBytes = MULTIPART_PART_BYTES, +}) { + // Fail fast on nonsense geometry: partBytes 0 makes partCount Infinity and + // the part loop below would issue empty PUTs forever instead of failing. + if (!Number.isSafeInteger(fileBytes) || fileBytes < 0) { + throw new Error(`uploadMultipart fileBytes must be a non-negative integer, got ${fileBytes}`); + } + if (!Number.isSafeInteger(partBytes) || partBytes <= 0) { + throw new Error(`uploadMultipart partBytes must be a positive integer, got ${partBytes}`); + } + let uploadId; + try { + const createResponse = await s3Request({ + method: "POST", + creds, + bucket, + key, + query: { uploads: "" }, + headers, + fetchImpl, + }); + const createXml = await createResponse.text(); + uploadId = /([\s\S]*?)<\/UploadId>/.exec(createXml)?.[1]; + if (!uploadId) throw new Error("CreateMultipartUpload response carried no UploadId"); + } catch (error) { + throw asUploadError(error, "create"); + } + + try { + const partCount = Math.max(1, Math.ceil(fileBytes / partBytes)); + log(`multipart upload ${uploadId}: ${partCount} part(s) of up to ${partBytes} bytes`); + const parts = []; + for (let partNumber = 1; partNumber <= partCount; partNumber += 1) { + const start = (partNumber - 1) * partBytes; + const body = await readFileRange(filePath, start, Math.min(partBytes, fileBytes - start)); + const etag = await uploadPartWithRetry({ + creds, + bucket, + key, + uploadId, + partNumber, + body, + fetchImpl, + }); + parts.push({ partNumber, etag }); + } + const completeBody = + `\n` + + parts + .map( + (part) => + `${part.partNumber}${xmlEscape(part.etag)}` + ) + .join("") + + ``; + const completeResponse = await s3Request({ + method: "POST", + creds, + bucket, + key, + query: { uploadId }, + body: completeBody, + headers: { "content-type": "application/xml" }, + fetchImpl, + }); + const completeXml = await completeResponse.text(); + // S3 answers a failed completion with HTTP 200 and an body, and + // R2 inherits the quirk. Without this check a failed completion reads as + // success and the verification GET 404s with a misleading reason. + if (completeXml.includes("")) { + const code = /([\s\S]*?)<\/Code>/.exec(completeXml)?.[1] ?? "unknown"; + throw new Error(`CompleteMultipartUpload returned an error body (code ${code})`); + } + return { uploadId, partCount }; + } catch (error) { + await abortMultipartUpload({ creds, bucket, key, uploadId, fetchImpl }); + throw asUploadError(error, "upload"); + } +} + export async function runArchive({ env = process.env, argv = [], fetchImpl } = {}) { const options = parseArgs(argv); const config = resolveArchiveConfig(env, { keep: options.keep }); @@ -313,15 +509,15 @@ export async function runArchive({ env = process.env, argv = [], fetchImpl } = { // read after this. Drop it rather than carry a second full-size copy // through the rest of the run on a volume with a free-space floor. rmSync(snapshotPath, { force: true }); - // Read once, then derive size, hash, and the upload body from those exact - // bytes. A stat-then-hash-then-read sequence is a genuine TOCTOU race: the - // file could change between steps and we would advertise a hash that does - // not describe what we actually uploaded. - const compressedBuffer = readFileSync(compressedPath); - // The upload body now lives in memory; the on-disk copy is redundant. - rmSync(compressedPath, { force: true }); - const compressedBytes = compressedBuffer.length; - const localHash = createHash("sha256").update(compressedBuffer).digest("hex"); + // Hash the gzip in one streaming pass, then read upload parts back from + // the same file. The old design read the whole gzip into memory so the + // advertised hash provably described the uploaded bytes; that guarantee + // now comes from where the file lives (this run's private work dir — + // nothing else can write to it) plus the download-back verification + // below, which still refuses to prune anything unless the stored object + // re-hashes to exactly this value. What changes is only the memory: the + // gzip is ~1.5 GB and no longer has to fit in a Buffer to be uploaded. + const { bytes: compressedBytes, sha256: localHash } = await hashFile(compressedPath); log(`compressed to ${compressedBytes} bytes (sha256 ${localHash.slice(0, 12)}...)`); const key = archiveKeyFor(config.prefix, startedAt); @@ -332,12 +528,12 @@ export async function runArchive({ env = process.env, argv = [], fetchImpl } = { } log(`uploading ${key}`); - await s3Request({ - method: "PUT", + await uploadMultipart({ creds: config.creds, bucket: config.bucket, key, - body: compressedBuffer, + filePath: compressedPath, + fileBytes: compressedBytes, headers: { "content-type": "application/gzip", // Survives independently of this script so a human can verify by hand. @@ -347,6 +543,9 @@ export async function runArchive({ env = process.env, argv = [], fetchImpl } = { }, fetchImpl, }); + // The upload is complete; the on-disk gzip is redundant from here, and + // the verification download below needs the room more. + rmSync(compressedPath, { force: true }); // ---- Verification: prove the remote object restores, not just that it exists. log("verifying uploaded object (download, re-hash, decompress, integrity_check)"); diff --git a/scripts/test-r2-weekly-archive.mjs b/scripts/test-r2-weekly-archive.mjs index b93ea29d..60df2523 100644 --- a/scripts/test-r2-weekly-archive.mjs +++ b/scripts/test-r2-weekly-archive.mjs @@ -8,7 +8,7 @@ */ import assert from "node:assert/strict"; -import { mkdtempSync, rmSync, existsSync, readFileSync } from "node:fs"; +import { mkdtempSync, rmSync, existsSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { gunzipSync } from "node:zlib"; @@ -23,6 +23,7 @@ import { resolveArchiveConfig, runArchive, selectPruneTargets, + uploadMultipart, } from "./ops/r2-weekly-archive.mjs"; const DAY_MS = 24 * 60 * 60 * 1000; @@ -303,6 +304,143 @@ await test("S3 transport diagnostics exclude arbitrary messages, codes and crede }); }); +// ------------------------------------------------------------ multipart ---- + +const STUB_CREDS = { + endpoint: "https://acct.r2.cloudflarestorage.com", + region: "auto", + accessKeyId: "test-key", + secretAccessKey: "test-secret", +}; + +await test("uploadMultipart assembles sequential parts into one object", async () => { + const dir = mkdtempSync(join(tmpdir(), "r2-archive-test-")); + try { + const filePath = join(dir, "payload.bin"); + const payload = Buffer.alloc(2500); + for (let i = 0; i < payload.length; i += 1) payload[i] = i % 251; + writeFileSync(filePath, payload); + const s3 = makeS3Stub(); + + const { partCount } = await uploadMultipart({ + creds: STUB_CREDS, + bucket: "usage-monitor-prod-v3", + key: "weekly/test.db.gz", + filePath, + fileBytes: payload.length, + fetchImpl: s3.fetchImpl, + partBytes: 1000, + }); + + assert.equal(partCount, 3, "2500 bytes at 1000-byte parts is 3 parts"); + assert.deepEqual(s3.store.get("weekly/test.db.gz"), payload); + assert.deepEqual( + s3.calls.map((c) => c.method), + ["POST", "PUT", "PUT", "PUT", "POST"], + "create, three parts, complete" + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +await test("uploadMultipart retries a part whose connection drops, then completes", async () => { + const dir = mkdtempSync(join(tmpdir(), "r2-archive-test-")); + try { + const filePath = join(dir, "payload.bin"); + const payload = Buffer.from("one part, dropped once"); + writeFileSync(filePath, payload); + const s3 = makeS3Stub(); + let putAttempts = 0; + + const result = await uploadMultipart({ + creds: STUB_CREDS, + bucket: "usage-monitor-prod-v3", + key: "weekly/test.db.gz", + filePath, + fileBytes: payload.length, + fetchImpl: async (url, init) => { + if (init.method === "PUT") { + putAttempts += 1; + if (putAttempts === 1) { + // The 2026-11-07 live failure: the socket dies mid-transfer. + throw new TypeError("fetch failed", { cause: { code: "UND_ERR_SOCKET" } }); + } + } + return s3.fetchImpl(url, init); + }, + }); + + assert.equal(result.partCount, 1); + assert.equal(putAttempts, 2, "the dropped part is retried, not the whole upload"); + assert.deepEqual(s3.store.get("weekly/test.db.gz"), payload); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +await test("uploadMultipart aborts and preserves the signer error when a part never lands", async () => { + const dir = mkdtempSync(join(tmpdir(), "r2-archive-test-")); + try { + const filePath = join(dir, "payload.bin"); + writeFileSync(filePath, Buffer.from("never lands")); + const s3 = makeS3Stub(); + + await assert.rejects( + uploadMultipart({ + creds: STUB_CREDS, + bucket: "usage-monitor-prod-v3", + key: "weekly/test.db.gz", + filePath, + fileBytes: 11, + fetchImpl: async (url, init) => { + if (init.method === "PUT") { + throw new TypeError("fetch failed", { cause: { code: "ECONNRESET" } }); + } + return s3.fetchImpl(url, init); + }, + }), + (error) => { + assert.equal(error.message, "S3 PUT transport failed (ECONNRESET)"); + assert.equal(classifyFailure(error), "upload_failed"); + return true; + } + ); + assert.equal(s3.uploads.size, 0, "abort removes the in-flight upload"); + assert.equal(s3.store.size, 0, "no object is created from a failed upload"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +await test("uploadMultipart rejects non-positive part geometry before any S3 call", async () => { + const s3 = makeS3Stub(); + await assert.rejects( + uploadMultipart({ + creds: STUB_CREDS, + bucket: "usage-monitor-prod-v3", + key: "weekly/test.db.gz", + filePath: "/nonexistent", + fileBytes: 10, + fetchImpl: s3.fetchImpl, + partBytes: 0, + }), + /partBytes must be a positive integer/ + ); + await assert.rejects( + uploadMultipart({ + creds: STUB_CREDS, + bucket: "usage-monitor-prod-v3", + key: "weekly/test.db.gz", + filePath: "/nonexistent", + fileBytes: -1, + fetchImpl: s3.fetchImpl, + }), + /fileBytes must be a non-negative integer/ + ); + assert.equal(s3.calls.length, 0, "validation happens before CreateMultipartUpload"); +}); + // --------------------------------------------------------------- config ---- await test("resolveArchiveConfig refuses to fall back to the B2 litestream keys", () => { @@ -367,7 +505,15 @@ function makeDatabase(path) { /** Tiny in-memory S3 so the full flow runs with no network and no credential. */ function makeS3Stub({ corruptOnGet = false } = {}) { const store = new Map(); + const uploads = new Map(); const calls = []; + let uploadSeq = 0; + const respond = (status, textBody, headers = {}) => ({ + ok: status >= 200 && status < 300, + status, + headers: { get: (name) => headers[String(name).toLowerCase()] ?? null }, + async text() { return textBody; }, + }); const fetchImpl = async (url, init) => { const method = init.method; const parsed = new URL(url); @@ -375,9 +521,42 @@ function makeS3Stub({ corruptOnGet = false } = {}) { const key = keyParts.join("/"); calls.push({ method, key: key || "(list)" }); + // CreateMultipartUpload — POST with ?uploads and no uploadId yet. + if (method === "POST" && parsed.searchParams.has("uploads")) { + uploadSeq += 1; + const uploadId = `stub-upload-${uploadSeq}`; + uploads.set(uploadId, { key, parts: new Map() }); + return respond( + 200, + `stub` + + `${key}${uploadId}` + ); + } + // CompleteMultipartUpload — assemble the parts in part-number order. + if (method === "POST" && parsed.searchParams.get("uploadId")) { + const uploadId = parsed.searchParams.get("uploadId"); + const upload = uploads.get(uploadId); + if (!upload) return respond(404, "NoSuchUpload"); + const assembled = Buffer.concat( + [...upload.parts.entries()].sort((a, b) => a[0] - b[0]).map(([, body]) => body) + ); + store.set(upload.key, assembled); + uploads.delete(uploadId); + return respond( + 200, + `${upload.key}` + ); + } if (method === "PUT") { + const partNumber = parsed.searchParams.get("partNumber"); + if (partNumber !== null) { + const upload = uploads.get(parsed.searchParams.get("uploadId")); + if (!upload) return respond(404, "NoSuchUpload"); + upload.parts.set(Number(partNumber), Buffer.from(init.body)); + return respond(200, "", { etag: `"stub-etag-${partNumber}"` }); + } store.set(key, Buffer.from(init.body)); - return { ok: true, status: 200, async text() { return ""; } }; + return respond(200, "", { etag: '"stub-etag"' }); } if (method === "GET" && parsed.searchParams.get("list-type") === "2") { const prefix = parsed.searchParams.get("prefix") ?? ""; @@ -390,15 +569,16 @@ function makeS3Stub({ corruptOnGet = false } = {}) { ) .join(""); const xml = `${contents}false`; - return { ok: true, status: 200, async text() { return xml; } }; + return respond(200, xml); } if (method === "GET") { const stored = store.get(key); - if (!stored) return { ok: false, status: 404, async text() { return ""; } }; + if (!stored) return respond(404, ""); const body = corruptOnGet ? Buffer.concat([stored, Buffer.from("x")]) : stored; return { ok: true, status: 200, + headers: { get: () => null }, // Real fetch exposes a web ReadableStream; the archiver streams it. body: new Response(body).body, async arrayBuffer() { return body.buffer.slice(body.byteOffset, body.byteOffset + body.length); }, @@ -406,12 +586,18 @@ function makeS3Stub({ corruptOnGet = false } = {}) { }; } if (method === "DELETE") { + const uploadId = parsed.searchParams.get("uploadId"); + if (uploadId) { + // AbortMultipartUpload — the parts vanish, no object is created. + uploads.delete(uploadId); + return respond(204, ""); + } store.delete(key); - return { ok: true, status: 204, async text() { return ""; } }; + return respond(204, ""); } throw new Error(`unexpected ${method}`); }; - return { fetchImpl, store, calls }; + return { fetchImpl, store, calls, uploads }; } function baseEnv(dir, dbPath) { @@ -440,7 +626,6 @@ await test("runArchive uploads a verifiable, restorable snapshot", async () => { // The stored bytes really are a gzipped, openable SQLite database. const restored = join(dir, "restored.db"); - const { writeFileSync } = await import("node:fs"); writeFileSync(restored, gunzipSync(s3.store.get(result.key))); const db = new DatabaseSync(restored, { readOnly: true }); const row = db.prepare("SELECT COUNT(*) AS n FROM budget").get(); @@ -547,7 +732,10 @@ await test("an upload transport rejection reports its phase and never verifies o assert.equal(classifyFailure(error), "upload_failed"); return true; }); - assert.deepEqual(methods, ["PUT"]); + // Create, the part's three attempts (the per-part retry budget), then the + // abort — and nothing else: a failed upload never verifies or prunes. + assert.deepEqual(methods, ["POST", "PUT", "PUT", "PUT", "DELETE"]); + assert.equal(s3.uploads.size, 0, "the aborted upload leaves nothing behind"); assert.deepEqual([...s3.store.keys()], [oldKey]); } finally { rmSync(dir, { recursive: true, force: true });