From c2ad3d45da5ac587a370c759796781e463d694f0 Mon Sep 17 00:00:00 2001 From: Enderson Maia Date: Wed, 3 Jun 2026 17:39:27 +0000 Subject: [PATCH 1/2] feat(cli): add --cache-from and --cache-to options to cartesi build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Expose Docker Buildx cache backend specs via two new repeatable CLI flags: cartesi build --cache-from type=local,src=/tmp/cache \ --cache-to type=local,dest=/tmp/cache The same options are also available per-drive in cartesi.toml: [drives.root] cache_from = ["type=gha"] cache_to = ["type=gha,mode=max"] CLI flags override the TOML values when non-empty. Changes: - config.ts: add cacheFrom/cacheTo fields to DockerDriveConfig and defaultRootDriveConfig; parse cache_from/cache_to from TOML - builder/docker.ts: thread cacheFrom/cacheTo through ImageBuildOptions and append --cache-from / --cache-to args to the buildx invocation - commands/build.ts: add --cache-from and --cache-to options (accumulator style); merge into BuildContext and apply over per-drive config - tests/unit/config.test.ts: add assertion for cache_from/cache_to parsing - tests/integration/builder/docker.test.ts: add two integration tests — one verifying the local cache directory is written, one verifying that layers are recovered from a local cache after the daemon build cache is pruned (attested by comparing root.ext2 SHA-256 between the two builds) - tests/integration/builder/fixtures/Dockerfile.cache: new fixture using a non-deterministic RUN date layer to make cache-hit vs miss detectable Co-Authored-By: Claude Sonnet 4.6 --- .changeset/cache-options-for-build.md | 10 ++ apps/cli/src/builder/docker.ts | 14 +- apps/cli/src/commands/build.ts | 20 ++- apps/cli/src/config.ts | 8 + .../tests/integration/builder/docker.test.ts | 137 ++++++++++++++++++ .../builder/fixtures/Dockerfile.cache | 3 + apps/cli/tests/unit/config.test.ts | 15 ++ 7 files changed, 205 insertions(+), 2 deletions(-) create mode 100644 .changeset/cache-options-for-build.md create mode 100644 apps/cli/tests/integration/builder/fixtures/Dockerfile.cache diff --git a/.changeset/cache-options-for-build.md b/.changeset/cache-options-for-build.md new file mode 100644 index 000000000..1c99a3242 --- /dev/null +++ b/.changeset/cache-options-for-build.md @@ -0,0 +1,10 @@ +--- +"@cartesi/cli": minor +--- + +add --cache-from and --cache-to options to cartesi build + +Expose Docker Buildx cache backend options via `--cache-from ` and +`--cache-to ` CLI flags (repeatable). The same options can also be +set per-drive in `cartesi.toml` via `cache_from` and `cache_to` arrays. +CLI flags override the TOML values when provided. diff --git a/apps/cli/src/builder/docker.ts b/apps/cli/src/builder/docker.ts index cf734bebc..9e48ae9f8 100644 --- a/apps/cli/src/builder/docker.ts +++ b/apps/cli/src/builder/docker.ts @@ -9,7 +9,13 @@ import type { BuildxMetadata } from "../types/docker.js"; type ImageBuildOptions = Pick< DockerDriveConfig, - "buildArgs" | "context" | "dockerfile" | "tags" | "target" + | "buildArgs" + | "cacheFrom" + | "cacheTo" + | "context" + | "dockerfile" + | "tags" + | "target" > & { destination: string; dockerfileContent?: string; reporter?: Reporter }; type ImageInfo = { @@ -25,6 +31,8 @@ type ImageInfo = { const buildImage = async (options: ImageBuildOptions): Promise => { const { buildArgs, + cacheFrom, + cacheTo, context, destination, dockerfile, @@ -62,6 +70,10 @@ const buildImage = async (options: ImageBuildOptions): Promise => { const metadataFile = path.join(tmpDir, "metadata.json"); args.push("--metadata-file", metadataFile); + // set cache options + args.push(...cacheFrom.flatMap((spec) => ["--cache-from", spec])); + args.push(...cacheTo.flatMap((spec) => ["--cache-to", spec])); + if (target) { args.push("--target", target); } diff --git a/apps/cli/src/commands/build.ts b/apps/cli/src/commands/build.ts index 9aa69ef5f..00119a120 100755 --- a/apps/cli/src/commands/build.ts +++ b/apps/cli/src/commands/build.ts @@ -18,6 +18,8 @@ import { describeWithdrawalConfig, resolveWithdrawal } from "../withdrawal.js"; // context for Listr build tasks interface BuildContext { + cacheFrom: string[]; + cacheTo: string[]; config: Config; debug: boolean; destination: string; @@ -49,6 +51,8 @@ const buildDriveTask = ( break; } case "docker": { + if (ctx.cacheFrom.length > 0) drive.cacheFrom = ctx.cacheFrom; + if (ctx.cacheTo.length > 0) drive.cacheTo = ctx.cacheTo; const imageInfo = await buildDocker( name, drive, @@ -110,10 +114,22 @@ export const createBuildCommand = () => { .default(false) .hideHelp(), ) + .option( + "--cache-from ", + "cache source for docker buildx build (can be repeated)", + (value: string, prev: string[]) => prev.concat([value]), + [] as string[], + ) + .option( + "--cache-to ", + "cache destination for docker buildx build (can be repeated)", + (value: string, prev: string[]) => prev.concat([value]), + [] as string[], + ) .option("-d, --drives-only", "only build drives, do not boot machine") .option("-v, --verbose", "verbose output", false) .action(async (options) => { - const { debug, drivesOnly, verbose } = options; + const { cacheFrom, cacheTo, debug, drivesOnly, verbose } = options; // get application configuration from 'cartesi.toml' const config = getApplicationConfig(options.config); @@ -126,6 +142,8 @@ export const createBuildCommand = () => { // build context const ctx = { + cacheFrom, + cacheTo, config, debug, destination, diff --git a/apps/cli/src/config.ts b/apps/cli/src/config.ts index bc2cb3531..873ec793d 100644 --- a/apps/cli/src/config.ts +++ b/apps/cli/src/config.ts @@ -183,6 +183,8 @@ export type DirectoryDriveConfig = { export type DockerDriveConfig = { builder: "docker"; buildArgs: string[]; // default is empty array + cacheFrom: string[]; // default is empty array + cacheTo: string[]; // default is empty array context: string; dockerfile: string; extraSize: number; // default is 0 (no extra size) @@ -424,6 +426,8 @@ type TomlTable = { [key: string]: TomlPrimitive }; export const defaultRootDriveConfig = (): DriveConfig => ({ builder: "docker", buildArgs: [], + cacheFrom: [], + cacheTo: [], context: ".", dockerfile: "Dockerfile", // file on current working directory extraSize: 0, @@ -824,6 +828,8 @@ const parseBuilderDrive = (drive: TomlPrimitive): DriveConfig => { case "docker": { const { build_args, + cache_from, + cache_to, context, dockerfile, extra_size, @@ -838,6 +844,8 @@ const parseBuilderDrive = (drive: TomlPrimitive): DriveConfig => { return { builder: "docker", buildArgs: parseStringArray(build_args), + cacheFrom: parseStringArray(cache_from), + cacheTo: parseStringArray(cache_to), image: parseOptionalString(image), context: parseString(context, "."), dockerfile: parseString(dockerfile, "Dockerfile"), diff --git a/apps/cli/tests/integration/builder/docker.test.ts b/apps/cli/tests/integration/builder/docker.test.ts index 0e4c3cbca..e2e78c231 100644 --- a/apps/cli/tests/integration/builder/docker.test.ts +++ b/apps/cli/tests/integration/builder/docker.test.ts @@ -6,8 +6,10 @@ import { expect, it, } from "bun:test"; +import crypto from "node:crypto"; import fs from "fs-extra"; import path from "node:path"; +import { execa } from "execa"; import { build } from "../../../src/builder/docker.js"; import type { DockerDriveConfig } from "../../../src/config.js"; import { setupIntegrationTests, TEST_SDK } from "../config.js"; @@ -36,6 +38,8 @@ describe("when building with the docker builder", () => { const drive: DockerDriveConfig = { buildArgs: [], builder: "docker", + cacheFrom: [], + cacheTo: [], context: ".", dockerfile: "Dockerfile", extraSize: 0, @@ -53,6 +57,8 @@ describe("when building with the docker builder", () => { const drive: DockerDriveConfig = { buildArgs: [], builder: "docker", + cacheFrom: [], + cacheTo: [], context: path.join(__dirname, "data"), dockerfile: "Dockerfile", extraSize: 0, @@ -70,6 +76,8 @@ describe("when building with the docker builder", () => { const drive: DockerDriveConfig = { buildArgs: [], builder: "docker", + cacheFrom: [], + cacheTo: [], context: path.join(__dirname, "fixtures"), dockerfile: path.join(__dirname, "fixtures", "Dockerfile"), extraSize: 0, @@ -88,6 +96,8 @@ describe("when building with the docker builder", () => { const drive: DockerDriveConfig = { buildArgs: [], builder: "docker", + cacheFrom: [], + cacheTo: [], context: path.join(__dirname, "fixtures"), dockerfile: path.join(__dirname, "fixtures", "Dockerfile"), extraSize: 0, @@ -106,6 +116,8 @@ describe("when building with the docker builder", () => { const drive: DockerDriveConfig = { buildArgs: [], builder: "docker", + cacheFrom: [], + cacheTo: [], context: path.join(__dirname, "fixtures"), dockerfile: path.join(__dirname, "fixtures", "Dockerfile"), extraSize: 0, @@ -120,3 +132,128 @@ describe("when building with the docker builder", () => { expect(stat.size).toEqual(29327360); }); }); + +describe("when using cache options", () => { + const image = TEST_SDK; + const CACHE_TEST_TAG = "cartesi-cli-integration-test-cache:latest"; + let destination: string; + + beforeEach(async () => { + destination = await createTempDir(); + }); + + afterEach(async () => { + await cleanupTempDir(destination); + }); + + it( + "should populate local cache when --cache-to is specified", + async () => { + const cacheDir = path.join(destination, "cache"); + const drive: DockerDriveConfig = { + buildArgs: [], + builder: "docker", + cacheFrom: [], + cacheTo: [`type=local,dest=${cacheDir}`], + context: path.join(__dirname, "fixtures"), + dockerfile: path.join( + __dirname, + "fixtures", + "Dockerfile.cache", + ), + extraSize: 0, + format: "ext2", + image: undefined, + tags: [CACHE_TEST_TAG], + target: undefined, + }; + + await build("root", drive, image, destination, false); + + // Clean up the tagged image + await execa("docker", ["image", "rm", CACHE_TEST_TAG], { + reject: false, + }); + + // Cache directory must have been written + expect(fs.existsSync(path.join(cacheDir, "index.json"))).toBe(true); + }, + { timeout: 120000 }, + ); + + it( + "should recover layers from local cache when --cache-from is specified", + async () => { + const cacheDir = path.join(destination, "cache"); + + // ── Build 1: populate local cache ──────────────────────────────── + const driveWithCacheTo: DockerDriveConfig = { + buildArgs: [], + builder: "docker", + cacheFrom: [], + cacheTo: [`type=local,dest=${cacheDir}`], + context: path.join(__dirname, "fixtures"), + dockerfile: path.join( + __dirname, + "fixtures", + "Dockerfile.cache", + ), + extraSize: 0, + format: "ext2", + image: undefined, + tags: [CACHE_TEST_TAG], + target: undefined, + }; + await build("root", driveWithCacheTo, image, destination, false); + + const hash1 = crypto + .createHash("sha256") + .update(fs.readFileSync(path.join(destination, "root.ext2"))) + .digest("hex"); + + // ── Teardown: remove image from daemon and build cache ──────────── + // Remove tagged image so BuildKit cannot reuse its layers implicitly + await execa("docker", ["image", "rm", CACHE_TEST_TAG], { + reject: false, + }); + // Remove all build-cache records (including non-dangling ones) from the + // currently selected buildx builder, which may not be the daemon's own + // (e.g. a docker-container builder created by docker/setup-buildx-action) + await execa("docker", ["buildx", "prune", "--all", "--force"]); + + // Remove build artefacts so the second build starts clean + await fs.remove(path.join(destination, "root.ext2")); + + // ── Build 2: rebuild using only the local cache ────────────────── + const driveWithCacheFrom: DockerDriveConfig = { + buildArgs: [], + builder: "docker", + cacheFrom: [`type=local,src=${cacheDir}`], + cacheTo: [], + context: path.join(__dirname, "fixtures"), + dockerfile: path.join( + __dirname, + "fixtures", + "Dockerfile.cache", + ), + extraSize: 0, + format: "ext2", + image: undefined, + tags: [], + target: undefined, + }; + await build("root", driveWithCacheFrom, image, destination, false); + + const hash2 = crypto + .createHash("sha256") + .update(fs.readFileSync(path.join(destination, "root.ext2"))) + .digest("hex"); + + // ── Attest: cache was recovered ────────────────────────────────── + // Identical hashes mean the RUN date layer was reused (not re-executed). + // A fresh build would produce a different timestamp → different content → different hash. + expect(hash2).toEqual(hash1); + }, + { timeout: 180000 }, + ); +}); diff --git a/apps/cli/tests/integration/builder/fixtures/Dockerfile.cache b/apps/cli/tests/integration/builder/fixtures/Dockerfile.cache new file mode 100644 index 000000000..09b14628c --- /dev/null +++ b/apps/cli/tests/integration/builder/fixtures/Dockerfile.cache @@ -0,0 +1,3 @@ +FROM --platform=linux/riscv64 ubuntu:24.04@sha256:3f83fb03282ef4e453bdf0060e0d83833bb3cf6e6f36f54d9b8517d311d78e03 +# Non-deterministic RUN: output changes each build unless the layer is reused from cache +RUN date +%s%N > /build-id.txt diff --git a/apps/cli/tests/unit/config.test.ts b/apps/cli/tests/unit/config.test.ts index 1d04adb5f..d0e7657a9 100644 --- a/apps/cli/tests/unit/config.test.ts +++ b/apps/cli/tests/unit/config.test.ts @@ -97,6 +97,8 @@ shared = true`, root: { buildArgs: [], builder: "docker", + cacheFrom: [], + cacheTo: [], dockerfile: "backend/Dockerfile", context: ".", extraSize: 0, @@ -112,6 +114,19 @@ shared = true`, }); }); + it("should parse cache_from and cache_to for docker drive", () => { + const config = parse([ + `[drives.root] +builder = "docker" +cache_from = ["type=gha"] +cache_to = ["type=gha,mode=max"]`, + ]); + expect(config.drives.root).toMatchObject({ + cacheFrom: ["type=gha"], + cacheTo: ["type=gha,mode=max"], + }); + }); + /** * [machine] */ From 09335096377b572ef920e318a92d7d6d069ff131 Mon Sep 17 00:00:00 2001 From: Enderson Maia Date: Tue, 20 Oct 2026 10:45:43 -0300 Subject: [PATCH 2/2] fixup! feat(cli): add --cache-from and --cache-to options to cartesi build --- .../tests/integration/builder/docker.test.ts | 40 +++++++++++++++++-- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/apps/cli/tests/integration/builder/docker.test.ts b/apps/cli/tests/integration/builder/docker.test.ts index e2e78c231..405f0c1c5 100644 --- a/apps/cli/tests/integration/builder/docker.test.ts +++ b/apps/cli/tests/integration/builder/docker.test.ts @@ -1,4 +1,5 @@ import { + afterAll, afterEach, beforeAll, beforeEach, @@ -6,10 +7,10 @@ import { expect, it, } from "bun:test"; -import crypto from "node:crypto"; +import { execa } from "execa"; import fs from "fs-extra"; +import crypto from "node:crypto"; import path from "node:path"; -import { execa } from "execa"; import { build } from "../../../src/builder/docker.js"; import type { DockerDriveConfig } from "../../../src/config.js"; import { setupIntegrationTests, TEST_SDK } from "../config.js"; @@ -136,8 +137,40 @@ describe("when building with the docker builder", () => { describe("when using cache options", () => { const image = TEST_SDK; const CACHE_TEST_TAG = "cartesi-cli-integration-test-cache:latest"; + const CACHE_TEST_BUILDER = `cartesi-cli-test-${process.pid}`; + let previousBuilder: string | undefined; let destination: string; + // Use a disposable, test-owned builder so pruning its cache never touches + // the developer's own builders. BUILDX_BUILDER is process-scoped, so the + // builder selected by `docker buildx use` is left unchanged. + beforeAll( + async () => { + await execa("docker", [ + "buildx", + "create", + "--name", + CACHE_TEST_BUILDER, + "--driver", + "docker-container", + ]); + previousBuilder = process.env.BUILDX_BUILDER; + process.env.BUILDX_BUILDER = CACHE_TEST_BUILDER; + }, + { timeout: 60000 }, + ); + + afterAll(async () => { + if (previousBuilder === undefined) { + delete process.env.BUILDX_BUILDER; + } else { + process.env.BUILDX_BUILDER = previousBuilder; + } + await execa("docker", ["buildx", "rm", "--force", CACHE_TEST_BUILDER], { + reject: false, + }); + }); + beforeEach(async () => { destination = await createTempDir(); }); @@ -217,8 +250,7 @@ describe("when using cache options", () => { reject: false, }); // Remove all build-cache records (including non-dangling ones) from the - // currently selected buildx builder, which may not be the daemon's own - // (e.g. a docker-container builder created by docker/setup-buildx-action) + // test-owned builder selected through BUILDX_BUILDER await execa("docker", ["buildx", "prune", "--all", "--force"]); // Remove build artefacts so the second build starts clean