From 51eb98ea6a0651b37189cdf2169f4f7f99cdb31d Mon Sep 17 00:00:00 2001 From: Kim Romero Date: Wed, 5 Aug 2026 13:09:42 +0200 Subject: [PATCH] feat(k8s): pod-unique port names and registry pull mode Two fixes needed to render a decker stack into a real cluster rather than a laptop docker daemon. k8s validates containerPort names pod-wide and Services reference them by name, but decker containers reuse "http"/"metrics" across containers in one pod, so the apply was rejected. Assign pod-unique <=15-char port names and dedupe Service ports on the numeric port instead of erroring on duplicate names. Image handling assumed a local daemon: ImageBuildSpec tags have no registry host, so kubelet tried docker.io and got ImagePullBackOff. Add DECKER_IMAGE_REGISTRY to prefix built tags and DECKER_IMAGE_MODE=pull to make ensureImages a no-op so an external builder can supply the images, plus imagePullPolicy: IfNotPresent for registry-less tags. `build --pods ` lets a recipe be rendered for k8s without editing the recipe, and DECKER_RELAY_HOST lets relay-warmup reach the relay by Service name instead of localhost. --- commands/build.ts | 3 +- renderers/k8s.ts | 86 +++++++++++++++++++++++++++++++++-------- scripts/relay-warmup.ts | 6 ++- utils/build.ts | 8 +++- utils/image-build.ts | 23 ++++++++++- 5 files changed, 106 insertions(+), 20 deletions(-) diff --git a/commands/build.ts b/commands/build.ts index e75c807..fd103ae 100644 --- a/commands/build.ts +++ b/commands/build.ts @@ -16,6 +16,7 @@ async function listRecipes(): Promise { export const command = new Command() .description("Build recipes into manifests/") .option("--opt ", "Pass an option to a factory recipe: key=value (repeatable)", { collect: true }) + .option("--pods ", "Render for a specific pods renderer (e.g. k8s) instead of the recipe default") .arguments("[recipes...:string]") .action(async (opts, ...recipes: string[]) => { const options = parseOpts(opts.opt); @@ -32,7 +33,7 @@ export const command = new Command() } done(sArt, artifactsLabel(recipe)); const sp = step(`rendering ${r}`); - const { name, binaries, binaryBuilds } = await buildOne(r, options); + const { name, binaries, binaryBuilds } = await buildOne(r, options, { pods: opts.pods }); done(sp, `manifests/${name}/`); // Binaries built from source land at `up` time; don't flag them missing here. const managed = new Set(binaryBuilds); diff --git a/renderers/k8s.ts b/renderers/k8s.ts index e2b7c59..c889ed9 100644 --- a/renderers/k8s.ts +++ b/renderers/k8s.ts @@ -1,4 +1,5 @@ import { stringify } from "jsr:@std/yaml@^1.0.5"; +import { imageTag } from "../utils/image-build.ts"; import { lookup, makeCtx } from "../utils/resolve.ts"; import { portInService, portNum, portProtocol } from "../utils/types.ts"; import type { @@ -6,6 +7,7 @@ import type { ContainerDef, ContainerResult, Ctx, + ImageBuildSpec, Pod, Ports, Recipe, @@ -41,12 +43,16 @@ function build(recipe: Recipe, _ctx: RenderCtx): RenderResult { }, yamlOpts), }]; + // ImageBuildSpec images are reported back so `up` can build (or, in pull + // mode, an external builder can supply) them - previously the k8s renderer + // referenced images nothing built. + const imageBuilds = new Map(); for (const pod of recipe.pods) { - const docs = podDocs(pod, ctx); + const docs = podDocs(pod, ctx, imageBuilds); const content = docs.map((d) => stringify(d, yamlOpts)).join("---\n"); files.push({ relPath: `deploy/${pod.name}.yaml`, content }); } - return { files }; + return { files, imageBuilds }; } function summary(paths: RendererPaths): Array<[string, string]> { @@ -65,7 +71,15 @@ type ContainerBuild = { built: ContainerResult; }; -function podDocs(pod: Pod, ctx: Ctx): unknown[] { +// hasRegistryHost: does the image reference start with a registry host +// (contains "." or ":" in its first path segment, or is "localhost")? +// Local-only tags cannot be pulled by kubelet and must be IfNotPresent. +function hasRegistryHost(image: string): boolean { + const first = image.split("/")[0]; + return first.includes(".") || first.includes(":") || first === "localhost"; +} + +function podDocs(pod: Pod, ctx: Ctx, imageBuilds: Map): unknown[] { const labels = { "app.kubernetes.io/name": pod.name, "app.kubernetes.io/part-of": "decker-l1", @@ -98,6 +112,8 @@ function podDocs(pod: Pod, ctx: Ctx): unknown[] { const allMounts: VolumeMount[] = builds.flatMap((b) => b.built.container.volumeMounts ?? []); + const portNames = podPortNames(builds); + const configMapDocs: unknown[] = []; const configVolumes: unknown[] = []; const containers = builds.map(({ def, built }) => { @@ -120,11 +136,25 @@ function podDocs(pod: Pod, ctx: Ctx): unknown[] { configMap: { name: configMountName }, }); } - const ports = expandDeployPorts(c.ports); + const ports = expandDeployPorts(def.name, c.ports, portNames); const env = c.env ? Object.entries(c.env).map(([name, value]) => ({ name, value })) : []; + // built-from-source images: reference by the canonical build tag (the + // same one `up`/an external builder produces), registry-prefixed when + // DECKER_IMAGE_REGISTRY is set. Local-only tags get IfNotPresent so + // kubelet uses the loaded image instead of trying to pull. + let image: string; + let pullPolicy: string | undefined; + if (typeof c.image === "string") { + image = c.image; + } else { + image = imageTag(c.image); + imageBuilds.set(image, c.image); + if (!hasRegistryHost(image)) pullPolicy = "IfNotPresent"; + } return { name: def.name, - image: typeof c.image === "string" ? c.image : `decker-${def.name}`, + image, + ...(pullPolicy ? { imagePullPolicy: pullPolicy } : {}), ...(c.command ? { command: c.command } : {}), ...(c.args ? { args: c.args } : {}), ...(env.length > 0 ? { env } : {}), @@ -155,7 +185,7 @@ function podDocs(pod: Pod, ctx: Ctx): unknown[] { }, ]; - const servicePorts = collectServicePorts(builds); + const servicePorts = collectServicePorts(builds, portNames); if (servicePorts.length > 0) { docs.push({ apiVersion: "v1", @@ -170,31 +200,55 @@ function podDocs(pod: Pod, ctx: Ctx): unknown[] { return docs; } -function expandDeployPorts(ports: Ports | undefined) { +// Port names must be unique across the whole pod (k8s validates containerPort +// names pod-wide, and Service targetPort refers to them by name), but decker +// containers freely reuse names like "http" and "metrics". Assign each +// (container, port) a pod-unique name <= 15 chars: the raw name if free, else +// prefixed with as much of the container name as fits. +function podPortNames(builds: ContainerBuild[]): Map { + const used = new Set(); + const out = new Map(); + for (const { def, built } of builds) { + for (const name of Object.keys(built.container.ports ?? {})) { + let candidate = name; + if (used.has(candidate)) { + const room = Math.max(15 - name.length - 1, 1); + candidate = `${def.name.slice(0, room)}-${name}`.slice(0, 15); + } + let i = 0; + while (used.has(candidate)) candidate = `${candidate.slice(0, 14)}${i++}`; + used.add(candidate); + out.set(`${def.name}/${name}`, candidate); + } + } + return out; +} + +function expandDeployPorts(defName: string, ports: Ports | undefined, portNames: Map) { if (!ports) return []; return Object.entries(ports).map(([name, spec]) => { const protocol = portProtocol(spec); return { - name, + name: portNames.get(`${defName}/${name}`) ?? name, containerPort: portNum(spec), ...(protocol ? { protocol } : {}), }; }); } -function collectServicePorts(builds: ContainerBuild[]) { - const seen = new Set(); +function collectServicePorts(builds: ContainerBuild[], portNames: Map) { + const seen = new Set(); const out: { name: string; port: number; targetPort: string }[] = []; - for (const { built } of builds) { + for (const { def, built } of builds) { const ports = built.container.ports; if (!ports) continue; for (const [name, spec] of Object.entries(ports)) { if (!portInService(spec)) continue; - if (seen.has(name)) { - throw new Error(`duplicate service port name ${name}`); - } - seen.add(name); - out.push({ name, port: portNum(spec), targetPort: name }); + const num = portNum(spec); + if (seen.has(num)) continue; // same numeric port twice in one pod: first wins + seen.add(num); + const unique = portNames.get(`${def.name}/${name}`) ?? name; + out.push({ name: unique, port: num, targetPort: unique }); } } return out; diff --git a/scripts/relay-warmup.ts b/scripts/relay-warmup.ts index 9de87eb..358a641 100644 --- a/scripts/relay-warmup.ts +++ b/scripts/relay-warmup.ts @@ -110,7 +110,11 @@ function resolveRelayUrl(recipe: Recipe, relay: WarmupRelay): string { throw new Error(`relay-warmup: relay ${relay.container} has no port ${DEFAULT_PORT_NAME}`); } const port = portNum(portSpec as Parameters[0]); - return `http://localhost:${port}`; + // DECKER_RELAY_HOST: reach the relay at a non-local host (e.g. its k8s + // Service name when this runs in-cluster). Default preserves the local + // port-forward / docker flow. + const host = Deno.env.get("DECKER_RELAY_HOST") ?? "localhost"; + return `http://${host}:${port}`; } async function waitForRelay(url: string, deadlineMs: number): Promise { diff --git a/utils/build.ts b/utils/build.ts index bb9886c..0d6c102 100644 --- a/utils/build.ts +++ b/utils/build.ts @@ -110,8 +110,14 @@ export async function generateArtifacts(recipe: Recipe): Promise { export async function buildOne( target: string, options: RecipeOptions = {}, + override: { pods?: string } = {}, ): Promise<{ name: string; binaries: string[]; binaryBuilds: string[] }> { - const { name, recipe } = await loadRecipe(target, options); + let { name, recipe } = await loadRecipe(target, options); + // renderer override (build --pods k8s): render for a specific pods + // renderer without editing the recipe, mirroring `up --pods`. + if (override.pods) { + recipe = { ...recipe, target: { ...recipe.target, pods: override.pods } }; + } const { binaries, binaryBuilds } = await emit(name, recipe); return { name, binaries, binaryBuilds: [...binaryBuilds.keys()] }; } diff --git a/utils/image-build.ts b/utils/image-build.ts index 5c175b8..cfde9ca 100644 --- a/utils/image-build.ts +++ b/utils/image-build.ts @@ -3,8 +3,26 @@ import type { ImageBuildSpec, ImageEngine } from "./types.ts"; import { DECKER_ROOT } from "./root.ts"; const CACHE_DIR = `${DECKER_ROOT}/cache/images`; +// DECKER_IMAGE_REGISTRY: when set, images built from ImageBuildSpecs are +// tagged (and referenced by renderers) under this registry prefix, e.g. +// registry.example.svc:5000 -> registry.example.svc:5000/decker-foo:main. +// This is what lets rendered manifests run on a real (non-local) cluster: +// nodes pull from the registry instead of a local image store / kind load. +// DECKER_IMAGE_MODE: "build" (default) builds missing images locally; +// "pull" skips building entirely - an external builder (CI or an in-cluster +// build system) supplies the images, decker only references them. +export function imageRegistry(): string { + return (Deno.env.get("DECKER_IMAGE_REGISTRY") ?? "").replace(/\/+$/, ""); +} + +export function imagePullMode(): boolean { + return Deno.env.get("DECKER_IMAGE_MODE") === "pull"; +} + export function imageTag(spec: ImageBuildSpec): string { - return `decker-${repoBasename(spec.repo)}:${slug(spec.ref)}`; + const base = `decker-${repoBasename(spec.repo)}:${slug(spec.ref)}`; + const reg = imageRegistry(); + return reg ? `${reg}/${base}` : base; } function repoBasename(repo: string): string { @@ -75,6 +93,9 @@ export async function ensureImages( specs: Map, engine: ImageEngine, ): Promise { + // pull mode: the registry already holds the images (external builder); + // nothing to do locally and no image engine is required. + if (imagePullMode()) return []; const built: string[] = []; for (const [tag, spec] of specs) { if (await imageExists(tag, engine)) continue;