diff --git a/docs/services.md b/docs/services.md index 1ffc82ccb..2b521ba7e 100644 --- a/docs/services.md +++ b/docs/services.md @@ -153,6 +153,36 @@ redeploy) cannot run conflicting operations on the same service. Leases are hear every 30 s while the operation runs; a lease not refreshed for 2 minutes belongs to a crashed process and is stolen automatically, so no manual cleanup is ever needed. +## Readiness and startup progress + +`Running` only means the container started. For workloads it recognizes, the node also reports +whether the service can serve requests and how far its startup downloads have got, on the job +returned by `SERVICE_GET_STATUS`: + +- `readiness` — `waiting`, `ready` or `failing`, from a request the node makes to the workload + itself: vLLM `GET /v1/models` on 8000, llama.cpp `GET /health` on 8080, ComfyUI + `GET /system_stats` on 8188. A profile applies only when the service publishes that port, so an + app that runs an engine behind its own port reports no readiness and `Running` stays the signal. + A service still `waiting` 15 minutes after its container started is logged once as a node warning + with the last probe error — usually a node that cannot reach its containers. +- `imagePull` — byte progress of the image pull. +- `modelDownload` — bytes downloaded against the total, and `filesComplete` / `filesTotal` when the + download is listed up front. + +`modelDownload` comes from the launch script's download manifest when it writes one: a +tab-separated `/tmp/ocean/downloads.tsv` in the container, one ` []` +line per item, written before the first download. A `file` entry is complete once `` exists +and in flight while `.part` does; a `dir` entry is a folder filled under names of its own +(huggingface_hub's snapshot cache), measured with `du -sb`. A `bytes` of `0` means unknown. Lines may +be appended while the script runs; at most 64 entries are read. Without a manifest, the node falls +back to the engine's own Hugging Face cache (vLLM, llama.cpp). + +A service with no engine profile (an app that runs its engine behind its own port) still reports +`modelDownload` from its manifest, with no `readiness`. With nothing to say when such a service is +ready, the node checks it only within 30 minutes of the container starting unless a manifest has +appeared, and stops once everything listed is complete. All of it is best-effort and owner-only: a +missing or wrong manifest only changes what that service's own progress report says. + ## Service outputs A service writes its results to `/data/outputs`. With an `outputBucketId` that folder is the diff --git a/src/@types/C2D/ServiceOnDemand.ts b/src/@types/C2D/ServiceOnDemand.ts index aa3b4a681..a038c4e31 100644 --- a/src/@types/C2D/ServiceOnDemand.ts +++ b/src/@types/C2D/ServiceOnDemand.ts @@ -193,6 +193,7 @@ export interface ServiceModelDownload { percent?: number filesComplete: number filesInFlight: number // the hub fetches files in parallel, so only the aggregate is meaningful + filesTotal?: number // from the download manifest updatedAt: number // Unix ms } diff --git a/src/components/c2d/compute_engine_docker.ts b/src/components/c2d/compute_engine_docker.ts index bec9f76a3..b3612c398 100755 --- a/src/components/c2d/compute_engine_docker.ts +++ b/src/components/c2d/compute_engine_docker.ts @@ -90,7 +90,8 @@ import { runReadinessProbe, PROBE_INITIAL_DELAY_SECONDS, PROBE_PERIOD_SECONDS, - READY_PROBE_PERIOD_SECONDS + READY_PROBE_PERIOD_SECONDS, + READINESS_WARN_AFTER_SECONDS } from './serviceReadiness.js' import { resolveServiceEngine, type ServiceEngineProfile } from './serviceEngines.js' import { @@ -99,6 +100,11 @@ import { isModelDownloadComplete, ModelDownloadSampler } from './modelDownload.js' +import { + MANIFEST_DISCOVERY_MS, + MANIFEST_SAMPLING_MS, + sampleDownloadManifest +} from './downloadManifest.js' import type { DockerMountObject } from '../../@types/PersistentStorage.js' import { resolveServiceImage } from './serviceResourceMatching.js' import { @@ -145,6 +151,9 @@ const trivyImage = 'aquasec/trivy:0.69.3' // Use pinned versions for safety // doesn't get a live operation's resources torn down under it. const SERVICE_LOCK_HEARTBEAT_MS = 30_000 const SERVICE_LOCK_STALE_MS = 120_000 +// `du` on a manifest folder runs in a user-controlled container, so it must not hold the probe. +const MANIFEST_DU_TIMEOUT_MS = 15_000 +const MANIFEST_DIR_MEASURE_MS = 15_000 // Identifies one engine instance as a service-lock holder across processes. pid alone is // not enough: pids are reused, and one process can host several engine instances. @@ -215,6 +224,15 @@ export class C2DEngineDocker extends C2DEngine { private serviceProbeUrls: Map = new Map() // Per-service model-download sampling state (which cache files to stat, when to re-list them). private modelDownloadSampler = new ModelDownloadSampler() + // serviceId -> last manifest check, for services the node cannot probe. + private manifestCheckedAt: Map = new Map() + // serviceId -> containerId the "still not ready" warning was logged for. + private readinessWarned: Map = new Map() + // serviceId -> manifest folder path -> last `du`. + private manifestDirSizes: Map< + string, + Map + > = new Map() // serviceId -> its readiness probe (+ model-download sample) still running in the background. // The probe is launched fire-and-forget so a slow engine, Docker daemon or Hub never holds up // InternalLoop; this keeps one probe per service at a time and lets stop() drain them. @@ -3066,7 +3084,8 @@ export class C2DEngineDocker extends C2DEngine { // never report an unmeasurable container as "0 bytes used". private async getContainerDiskUsage( containerName: string, - path: string = '/data' + path: string = '/data', + timeoutMs?: number ): Promise { try { const container = this.docker.getContainer(containerName) @@ -3078,22 +3097,47 @@ export class C2DEngineDocker extends C2DEngine { return null } + // Without a TTY, Docker's 8-byte stream headers can read as digits prepended to the size. const exec = await container.exec({ Cmd: ['du', '-sb', path], AttachStdout: true, - AttachStderr: true + AttachStderr: true, + Tty: true }) - const stream = await exec.start({ Detach: false, Tty: false }) + const stream = await exec.start({ Detach: false, Tty: true }) + let timedOut = false + const timer = + timeoutMs !== undefined + ? setTimeout(() => { + timedOut = true + stream.destroy() + }, timeoutMs) + : undefined const chunks: Buffer[] = [] - for await (const chunk of stream) { - chunks.push(chunk as Buffer) + try { + for await (const chunk of stream) { + chunks.push(chunk as Buffer) + } + } catch (error) { + if (!timedOut) { + throw error + } + } finally { + clearTimeout(timer) + } + if (timedOut) { + CORE_LOGGER.warn( + `du ${path} in ${containerName} did not finish within ${timeoutMs / 1000}s` + ) + return null } const output = Buffer.concat(chunks).toString() - const match = output.match(/(\d+)\s/) + // The size line follows any "du: cannot access" lines, which the TTY merges in. + const match = output.match(/^(\d+)\t/m) return match ? parseInt(match[1], 10) : null } catch (error) { CORE_LOGGER.error( @@ -4639,9 +4683,16 @@ export class C2DEngineDocker extends C2DEngine { // expired), whichever path ended them. const running = new Set(services.map((svc) => svc.serviceId)) this.modelDownloadSampler.retain(running) - for (const serviceId of this.serviceProbeUrls.keys()) { - if (!running.has(serviceId)) { - this.serviceProbeUrls.delete(serviceId) + for (const state of [ + this.serviceProbeUrls, + this.manifestCheckedAt, + this.readinessWarned, + this.manifestDirSizes + ]) { + for (const serviceId of state.keys()) { + if (!running.has(serviceId)) { + state.delete(serviceId) + } } } return runningOnly @@ -4716,7 +4767,10 @@ export class C2DEngineDocker extends C2DEngine { ): Promise { try { const engine = resolveServiceEngine(job) - if (!engine) return + if (!engine) { + await this.sampleUnprobedServiceDownload(job, details) + return + } if (this.serviceOpsInFlight.has(job.serviceId)) return const now = Date.now() @@ -4819,24 +4873,24 @@ export class C2DEngineDocker extends C2DEngine { ? undefined : await this.sampleModelDownload(job, engine) - // Same cross-process guard as the metrics write: a lifecycle transition must win. + // Usually a node that cannot reach its container: tell the operator once per container. if ( - this.serviceOpsInFlight.has(job.serviceId) || - (await this.db.isServiceLocked(job.serviceId, SERVICE_LOCK_STALE_MS)) + readiness.state === 'waiting' && + Number.isFinite(startedAt) && + now - startedAt >= READINESS_WARN_AFTER_SECONDS * 1000 && + this.readinessWarned.get(job.serviceId) !== job.containerId ) { - return + this.readinessWarned.set(job.serviceId, job.containerId) + const download = modelDownload ?? job.modelDownload + CORE_LOGGER.warn( + `[readiness] service ${job.serviceId} (${engine.id}) not ready ` + + `${Math.round((now - startedAt) / 60_000)} min after its container started: ` + + `${readiness.lastError ?? `HTTP ${readiness.httpStatus}`} via ${readiness.probedUrl}` + + (download ? `; model download at ${download.percent ?? '?'}%` : '') + ) } - await this.db.updateServiceJobReadiness( - job.serviceId, - { - owner: job.owner, - clusterHash: job.clusterHash, - status: ServiceStatusNumber.Running, - containerId: job.containerId - }, - readiness, - modelDownload - ) + + await this.persistServiceReadiness(job, readiness, modelDownload) } catch (e: any) { CORE_LOGGER.debug( `[readiness] service ${job.serviceId}: probe failed: ${e?.message}` @@ -4844,8 +4898,99 @@ export class C2DEngineDocker extends C2DEngine { } } + // Same cross-process guard as the metrics write: a lifecycle transition must win. + private async persistServiceReadiness( + job: ServiceJob, + readiness: ServiceReadiness | undefined, + modelDownload: ServiceModelDownload | undefined + ): Promise { + if ( + this.serviceOpsInFlight.has(job.serviceId) || + (await this.db.isServiceLocked(job.serviceId, SERVICE_LOCK_STALE_MS)) + ) { + return + } + await this.db.updateServiceJobReadiness( + job.serviceId, + { + owner: job.owner, + clusterHash: job.clusterHash, + status: ServiceStatusNumber.Running, + containerId: job.containerId + }, + readiness, + modelDownload + ) + } + /** - * How much of its model the container has pulled down, read from the engine's own cache. + * Manifest download progress for a service the node cannot probe; no readiness, so Running stays + * the signal. With no readiness to end it, sampling is bounded by MANIFEST_DISCOVERY_MS / + * MANIFEST_SAMPLING_MS and stops once the listed download is complete. + */ + private async sampleUnprobedServiceDownload( + job: ServiceJob, + details: Dockerode.ContainerInspectInfo + ): Promise { + if ( + this.serviceOpsInFlight.has(job.serviceId) || + isModelDownloadComplete(job.modelDownload) + ) { + return + } + const now = Date.now() + const startedAt = Date.parse(details.State?.StartedAt ?? '') + if ( + Number.isFinite(startedAt) && + now - startedAt > (job.modelDownload ? MANIFEST_SAMPLING_MS : MANIFEST_DISCOVERY_MS) + ) { + return + } + if ( + now - (this.manifestCheckedAt.get(job.serviceId) ?? 0) < + PROBE_PERIOD_SECONDS * 1000 + ) { + return + } + this.manifestCheckedAt.set(job.serviceId, now) + const modelDownload = await sampleDownloadManifest( + this.docker.getContainer(job.containerId), + (path) => this.manifestDirectoryBytes(job, path) + ) + if (modelDownload) { + await this.persistServiceReadiness(job, undefined, modelDownload) + } + } + + private async manifestDirectoryBytes( + job: ServiceJob, + path: string + ): Promise { + let sizes = this.manifestDirSizes.get(job.serviceId) + if (!sizes) { + sizes = new Map() + this.manifestDirSizes.set(job.serviceId, sizes) + } + const now = Date.now() + const last = sizes.get(path) + if ( + last?.containerId === job.containerId && + now - last.at < MANIFEST_DIR_MEASURE_MS + ) { + return last.bytes + } + const bytes = await this.getContainerDiskUsage( + job.containerId, + path, + MANIFEST_DU_TIMEOUT_MS + ) + sizes.set(path, { containerId: job.containerId, bytes, at: now }) + return bytes + } + + /** + * How much of its model the container has pulled down: from the launch script's download + * manifest when it writes one, otherwise from the engine's own cache. * * This is the wait the image pull does not cover: the image is fetched once per node and cached * forever after, while the weights are fetched by the engine on EVERY container start, after it @@ -4859,14 +5004,23 @@ export class C2DEngineDocker extends C2DEngine { job: ServiceJob, engine: ServiceEngineProfile ): Promise { - if (!engine.modelCachePath) return undefined - // Downloaded: what remains is the engine loading weights, which the cache says nothing about. - // Returning nothing keeps the stored 100% record as it is. - if (isModelDownloadComplete(job.modelDownload)) { - return undefined - } try { const container = this.docker.getContainer(job.containerId) + // Before the completion check: a script can append to its manifest after earlier downloads. + const fromManifest = await sampleDownloadManifest(container, (path) => + this.manifestDirectoryBytes(job, path) + ) + if (fromManifest) { + return fromManifest + } + if (!engine.modelCachePath) { + return undefined + } + // Downloaded: what remains is the engine loading weights, which the cache says nothing about. + // Returning nothing keeps the stored 100% record as it is. + if (isModelDownloadComplete(job.modelDownload)) { + return undefined + } const downloaded = await this.modelDownloadSampler.sample(job.serviceId, container) if (!downloaded) return undefined // Only a Hugging Face repo has a size the node can look up; a local path or an object-store diff --git a/src/components/c2d/downloadManifest.ts b/src/components/c2d/downloadManifest.ts new file mode 100644 index 000000000..b25c06648 --- /dev/null +++ b/src/components/c2d/downloadManifest.ts @@ -0,0 +1,144 @@ +import type Dockerode from 'dockerode' +import { pipeline } from 'stream' +import * as tarStream from 'tar-stream' +import type { ServiceModelDownload } from '../../@types/C2D/ServiceOnDemand.js' +import { buildModelDownload, fileSize } from './modelDownload.js' + +/** + * Download progress from a manifest the service's launch script writes before its first download, + * one `\t\t[\t]` line per item (`bytes` 0 = unknown): + * + * file written to `.part`, renamed to `` once complete + * dir a folder filled under names of its own, measured with `du` + * + * Re-read on every sample, since the script may append to it. Untrusted: it only affects this + * service's own progress report. + */ +export const MANIFEST_PATH = '/tmp/ocean/downloads.tsv' +const MAX_DOWNLOAD_ENTRIES = 64 +// Each `dir` entry costs a `du` inside the container. +const MAX_DIR_ENTRIES = 8 +// For a service the node cannot probe: how long after start to look for a manifest, and the +// longest it is sampled, so a download that never completes is not sampled forever. +export const MANIFEST_DISCOVERY_MS = 30 * 60 * 1000 +export const MANIFEST_SAMPLING_MS = 6 * 60 * 60 * 1000 +const MAX_READ_BYTES = 64 * 1024 +const BYTES = /^\d{1,13}$/ + +export interface DownloadEntry { + kind: 'file' | 'dir' + path: string + bytes: number | null +} + +export type DirectoryBytes = (path: string) => Promise + +async function readContainerFile( + container: Dockerode.Container, + path: string +): Promise { + try { + const extract = tarStream.extract() + pipeline(await container.getArchive({ path }), extract, () => {}) + for await (const entry of extract) { + const { type, size } = entry.header + if (type !== 'file' || (size ?? 0) > MAX_READ_BYTES) { + return null + } + const chunks: Buffer[] = [] + for await (const chunk of entry) { + chunks.push(chunk) + } + return Buffer.concat(chunks).toString('utf8') + } + return null + } catch { + return null + } +} + +/** `truncated`: the manifest names more than the caps allow, so it must never read as complete. */ +export function parseDownloadManifest(text: string): { + entries: DownloadEntry[] + truncated: boolean +} { + const entries = new Map() + let dirs = 0 + for (const line of text.split('\n')) { + const [kind, bytes, path] = line.trim().split('\t') + if ( + (kind !== 'file' && kind !== 'dir') || + !BYTES.test(bytes ?? '') || + !path?.startsWith('/') || + path.includes('\0') || + entries.has(path) + ) { + continue + } + if ( + entries.size >= MAX_DOWNLOAD_ENTRIES || + (kind === 'dir' && dirs >= MAX_DIR_ENTRIES) + ) { + return { entries: [...entries.values()], truncated: true } + } + if (kind === 'dir') { + dirs++ + } + entries.set(path, { kind, path, bytes: Number(bytes) || null }) + } + return { entries: [...entries.values()], truncated: false } +} + +/** Null when the script writes no manifest. */ +export async function sampleDownloadManifest( + container: Dockerode.Container, + directoryBytes: DirectoryBytes +): Promise { + const manifest = await readContainerFile(container, MANIFEST_PATH) + if (!manifest) { + return null + } + const { entries, truncated } = parseDownloadManifest(manifest) + if (entries.length === 0) { + return null + } + let downloadedBytes = 0 + let files = 0 + let inFlight = 0 + for (const { kind, path, bytes } of entries) { + if (kind === 'dir') { + const held = await directoryBytes(path) + if (held === null) { + continue + } + // A reused cache can hold more than this launch asked for. + downloadedBytes += bytes ? Math.min(held, bytes) : held + if (bytes && held >= bytes) { + files++ + } else { + inFlight++ + } + continue + } + // Finished first, so a stale .part beside it cannot keep the file in flight. + const finished = await fileSize(container, path) + if (finished !== null) { + downloadedBytes += finished + files++ + continue + } + const partial = await fileSize(container, `${path}.part`) + if (partial !== null) { + downloadedBytes += partial + inFlight++ + } + } + const totalBytes = + !truncated && entries.every(({ bytes }) => bytes !== null) + ? entries.reduce((sum, { bytes }) => sum + bytes, 0) + : null + return { + ...buildModelDownload({ downloadedBytes, files, inFlight }, totalBytes, null), + ...(truncated ? {} : { filesTotal: entries.length }) + } +} diff --git a/src/components/c2d/modelDownload.ts b/src/components/c2d/modelDownload.ts index 073f1cc23..eba6801a1 100644 --- a/src/components/c2d/modelDownload.ts +++ b/src/components/c2d/modelDownload.ts @@ -73,7 +73,7 @@ async function statContainerPath( * `huggingface_hub` can link a blob into a shared store — the link's own size would count a few * bytes for a multi-gigabyte file. */ -async function fileSize( +export async function fileSize( container: Dockerode.Container, path: string ): Promise { @@ -470,11 +470,15 @@ async function fetchGgufFileBytes( * Whether a recorded download has finished: the total reached AND nothing still arriving. The total * is an estimate (dtype × parameters), so on its own it can read 100% while the last shard is still * being written; no partial files left is what makes it final. A record with no total never - * completes here, and sampling simply carries on until the service is ready. + * completes here, and sampling simply carries on until the service is ready. A manifest record is + * complete once every listed item is. */ export function isModelDownloadComplete( download: ServiceModelDownload | undefined ): boolean { + if (download?.filesTotal !== undefined) { + return download.filesComplete >= download.filesTotal + } return download?.percent === 100 && download.filesInFlight === 0 } diff --git a/src/components/c2d/serviceEngines.ts b/src/components/c2d/serviceEngines.ts index 621e6e000..7cd7cfd0b 100644 --- a/src/components/c2d/serviceEngines.ts +++ b/src/components/c2d/serviceEngines.ts @@ -149,6 +149,12 @@ export const SERVICE_ENGINE_PROFILES: ServiceEngineProfile[] = [ modelCachePath: '/root/.cache/huggingface/hub', modelIdFromCommand: hfRepoIdFromLlamaCppCommand, modelQuantFromCommand: quantFromLlamaCppCommand + }, + { + id: 'comfyui', + matchesImage: (image) => /(^|\/)yanwk\/comfyui-boot$/.test(image), + // Binds its port only after models are fetched and custom nodes loaded. + probe: { path: '/system_stats', port: 8188, expectStatus: [200] } } ] @@ -165,11 +171,26 @@ function stripImageRef(image: string): string { return colon === -1 ? withoutDigest : withoutDigest.slice(0, colon) } -/** The engine profile for a service, or null when the node does not recognize the image. */ +/** + * Null for an unrecognized image, or when the engine's port is not published (an app running the + * engine behind its own port has nothing to probe). + */ export function resolveServiceEngine(job: ServiceJob): ServiceEngineProfile | null { const image = stripImageRef((job.image || job.containerImage || '').trim()) if (!image) { return null } - return SERVICE_ENGINE_PROFILES.find((profile) => profile.matchesImage(image)) ?? null + const profile = SERVICE_ENGINE_PROFILES.find((candidate) => + candidate.matchesImage(image) + ) + if (!profile) { + return null + } + if ( + profile.probe.port !== undefined && + !job.exposedPorts?.includes(profile.probe.port) + ) { + return null + } + return profile } diff --git a/src/components/c2d/serviceReadiness.ts b/src/components/c2d/serviceReadiness.ts index ac370e57b..e775c3a0e 100644 --- a/src/components/c2d/serviceReadiness.ts +++ b/src/components/c2d/serviceReadiness.ts @@ -111,6 +111,8 @@ export const PROBE_PERIOD_SECONDS = 5 // Once ready the check keeps running — to catch an engine that dies without its container exiting — // but far more slowly: it is a liveness check at that point, not a wait. export const READY_PROBE_PERIOD_SECONDS = 30 +// Not ready this long after start: warn once per container. +export const READINESS_WARN_AFTER_SECONDS = 15 * 60 const PROBE_TIMEOUT_MS = 2000 export interface ReadinessProbeResult { diff --git a/src/components/database/C2DDatabase.ts b/src/components/database/C2DDatabase.ts index 9fbf0379d..72a43c049 100755 --- a/src/components/database/C2DDatabase.ts +++ b/src/components/database/C2DDatabase.ts @@ -120,7 +120,7 @@ export class C2DDatabase extends AbstractDatabase { status: number containerId: string }, - readiness: ServiceReadiness, + readiness: ServiceReadiness | undefined, modelDownload?: ServiceModelDownload ): Promise { return await this.provider.updateServiceJobReadiness( diff --git a/src/components/database/sqliteCompute.ts b/src/components/database/sqliteCompute.ts index 90890cb36..314623f43 100644 --- a/src/components/database/sqliteCompute.ts +++ b/src/components/database/sqliteCompute.ts @@ -409,6 +409,7 @@ export class SQLiteCompute implements ComputeDatabaseProvider { // Same guarantees as updateServiceJobMetrics, for the readiness probe result: it is sampled from // the same lease-free background loop, so it must never overwrite a lifecycle transition either. + // `readiness` is undefined for a service the node cannot probe. // eslint-disable-next-line require-await async updateServiceJobReadiness( serviceId: string, @@ -418,11 +419,13 @@ export class SQLiteCompute implements ComputeDatabaseProvider { status: number containerId: string }, - readiness: ServiceReadiness, + readiness: ServiceReadiness | undefined, modelDownload?: ServiceModelDownload ): Promise { return this.patchServiceJobBody(serviceId, expected, 'readiness', (body) => { - body.readiness = readiness + if (readiness) { + body.readiness = readiness + } // Only overwritten when a fresh sample was taken: once the engine is ready the walk stops, // and the last figures stay as the record of what was downloaded. if (modelDownload) body.modelDownload = modelDownload diff --git a/src/test/unit/service/downloadManifest.test.ts b/src/test/unit/service/downloadManifest.test.ts new file mode 100644 index 000000000..7372252ba --- /dev/null +++ b/src/test/unit/service/downloadManifest.test.ts @@ -0,0 +1,152 @@ +import { expect } from 'chai' +import * as tarStream from 'tar-stream' +import { isModelDownloadComplete } from '../../../components/c2d/modelDownload.js' +import { + MANIFEST_PATH, + parseDownloadManifest, + sampleDownloadManifest +} from '../../../components/c2d/downloadManifest.js' + +const MODELS = '/data/outputs/comfy/models' +const GEMMA = '/data/outputs/comfy/.cache/huggingface/hub/models--google--gemma-4-31B-it' + +describe('parseDownloadManifest', () => { + it('reads file and dir entries, treating an unknown size as null', () => { + const text = [ + `file\t1000\t${MODELS}/vae/a.safetensors\thttps://huggingface.co/org/repo/resolve/main/a.safetensors`, + `file\t0\t${MODELS}/loras/b.safetensors`, + `dir\t5000\t${GEMMA}`, + '' + ].join('\n') + expect(parseDownloadManifest(text)).to.deep.equal({ + entries: [ + { kind: 'file', path: `${MODELS}/vae/a.safetensors`, bytes: 1000 }, + { kind: 'file', path: `${MODELS}/loras/b.safetensors`, bytes: null }, + { kind: 'dir', path: GEMMA, bytes: 5000 } + ], + truncated: false + }) + }) + + it('drops malformed lines and repeated paths', () => { + const text = [ + `blob\t1\t${MODELS}/a`, + `file\t-1\t${MODELS}/a`, + `file\tlots\t${MODELS}/a`, + 'file\t1\trelative/path', + `file\t1\t${MODELS}/a\u0000b`, + `file\t1\t${MODELS}/a`, + `file\t2\t${MODELS}/a`, + 'file\t1' + ].join('\n') + expect(parseDownloadManifest(text).entries).to.deep.equal([ + { kind: 'file', path: `${MODELS}/a`, bytes: 1 } + ]) + }) + + it('measures at most 8 folders and flags a manifest that names more', () => { + const dir = (i: number) => `dir\t1\t/data/hub/models--org--m${i}` + const eight = Array.from({ length: 8 }, (_, i) => dir(i)) + expect(parseDownloadManifest(eight.join('\n')).truncated).to.equal(false) + const over = parseDownloadManifest([...eight, dir(8)].join('\n')) + expect(over.entries).to.have.length(8) + expect(over.truncated).to.equal(true) + }) + + it('caps the entries and flags a manifest that names more', () => { + const line = (i: number) => `file\t1\t${MODELS}/f${i}` + const capped = Array.from({ length: 64 }, (_, i) => line(i)) + expect(parseDownloadManifest(capped.join('\n')).truncated).to.equal(false) + const over = parseDownloadManifest([...capped, line(64)].join('\n')) + expect(over.entries).to.have.length(64) + expect(over.truncated).to.equal(true) + }) +}) + +describe('sampleDownloadManifest', () => { + // A container double: `getArchive()` serves the manifest as a tar, `infoArchive()` answers stats. + function fakeContainer(manifest: string | null, files: Record) { + return { + getArchive: ({ path }: { path: string }) => { + if (manifest === null || path !== MANIFEST_PATH) { + return Promise.reject(new Error('no such file')) + } + const pack = tarStream.pack() + pack.entry({ name: 'downloads.tsv' }, manifest) + pack.finalize() + return Promise.resolve(pack) + }, + infoArchive: ({ path }: { path: string }) => { + if (files[path] === undefined) { + return Promise.reject(new Error('not found')) + } + const header = Buffer.from(JSON.stringify({ size: files[path] })).toString( + 'base64' + ) + return Promise.resolve({ headers: { 'x-docker-container-path-stat': header } }) + } + } as any + } + + const manifest = [ + `file\t1000\t${MODELS}/vae/a.safetensors`, + `file\t2000\t${MODELS}/loras/b.safetensors`, + `dir\t7000\t${GEMMA}` + ].join('\n') + + it('measures files and folders against the sizes the script declared', async () => { + const container = fakeContainer(manifest, { + [`${MODELS}/vae/a.safetensors`]: 1000, + [`${MODELS}/loras/b.safetensors.part`]: 500 + }) + const download = await sampleDownloadManifest(container, (path) => + Promise.resolve(path === GEMMA ? 3000 : null) + ) + expect(download).to.include({ + downloadedBytes: 4500, + totalBytes: 10000, + percent: 45, + filesComplete: 1, + filesInFlight: 2, + filesTotal: 3 + }) + expect(isModelDownloadComplete(download)).to.equal(false) + }) + + it('is complete once every file is in place and every folder holds its size', async () => { + const container = fakeContainer(manifest, { + [`${MODELS}/vae/a.safetensors`]: 1000, + [`${MODELS}/loras/b.safetensors`]: 2000 + }) + // A reused cache can hold more than declared; it counts only up to the declared size. + const download = await sampleDownloadManifest(container, () => Promise.resolve(9000)) + expect(download).to.include({ + downloadedBytes: 10000, + percent: 100, + filesComplete: 3 + }) + expect(isModelDownloadComplete(download)).to.equal(true) + }) + + it('counts a finished file as done even with a stale .part beside it', async () => { + const container = fakeContainer(`file\t1000\t${MODELS}/vae/a.safetensors`, { + [`${MODELS}/vae/a.safetensors`]: 1000, + [`${MODELS}/vae/a.safetensors.part`]: 10 + }) + const download = await sampleDownloadManifest(container, () => Promise.resolve(null)) + expect(download).to.include({ filesComplete: 1, filesInFlight: 0, percent: 100 }) + }) + + it('reports no total when a size is unknown', async () => { + const container = fakeContainer(`file\t0\t${MODELS}/vae/a.safetensors`, {}) + const download = await sampleDownloadManifest(container, () => Promise.resolve(null)) + expect(download?.totalBytes).to.equal(undefined) + expect(download?.filesTotal).to.equal(1) + }) + + it('returns null when the script writes no manifest, or an empty one', async () => { + const none = () => Promise.resolve(null) + expect(await sampleDownloadManifest(fakeContainer(null, {}), none)).to.equal(null) + expect(await sampleDownloadManifest(fakeContainer('', {}), none)).to.equal(null) + }) +}) diff --git a/src/test/unit/service/readinessWarning.test.ts b/src/test/unit/service/readinessWarning.test.ts new file mode 100644 index 000000000..c6641b0ad --- /dev/null +++ b/src/test/unit/service/readinessWarning.test.ts @@ -0,0 +1,73 @@ +import { expect } from 'chai' +import sinon from 'sinon' +import { C2DEngineDocker } from '../../../components/c2d/compute_engine_docker.js' +import { ServiceStatusNumber } from '../../../@types/C2D/ServiceOnDemand.js' +import { CORE_LOGGER } from '../../../utils/logging/common.js' + +// A ComfyUI service whose port the node cannot reach (port 1 refuses at once), so every probe +// reads "waiting" — the case the warning exists for. +describe('warning for a service that stays not ready', () => { + let warn: sinon.SinonStub + beforeEach(() => { + warn = sinon.stub(CORE_LOGGER, 'warn') + }) + afterEach(() => { + warn.restore() + }) + + function engine() { + const e: any = Object.create(C2DEngineDocker.prototype) + e.serviceOpsInFlight = new Set() + e.serviceProbeUrls = new Map() + e.manifestCheckedAt = new Map() + e.readinessWarned = new Map() + e.docker = { + getContainer: () => ({ + getArchive: () => Promise.reject(new Error('no such file')) + }) + } + e.db = { + isServiceLocked: sinon.stub().resolves(false), + updateServiceJobReadiness: sinon.stub().resolves(true) + } + return e + } + + const job = (containerId = 'container-1') => ({ + serviceId: 'svc-1', + owner: '0x0000000000000000000000000000000000000001', + clusterHash: '0xcluster', + containerId, + image: 'yanwk/comfyui-boot', + exposedPorts: [8188], + endpoints: [{ containerPort: 8188, hostPort: 1, url: 'http://127.0.0.1:1' }], + status: ServiceStatusNumber.Running + }) + const startedAgo = (minutes: number) => ({ + State: { StartedAt: new Date(Date.now() - minutes * 60_000).toISOString() }, + NetworkSettings: { Networks: {} } + }) + const readinessWarnings = () => + warn.getCalls().filter((call) => String(call.args[0]).startsWith('[readiness]')) + + it('warns once per container after 15 minutes', async () => { + const e = engine() + await e.probeServiceReadiness(job(), startedAgo(16)) + await e.probeServiceReadiness(job(), startedAgo(16)) + expect(readinessWarnings()).to.have.length(1) + expect(readinessWarnings()[0].args[0]).to.include('svc-1 (comfyui) not ready 16 min') + }) + + it('stays quiet before then', async () => { + const e = engine() + await e.probeServiceReadiness(job(), startedAgo(5)) + expect(readinessWarnings()).to.have.length(0) + }) + + it('warns again for the new container after a restart', async () => { + const e = engine() + await e.probeServiceReadiness(job('container-1'), startedAgo(16)) + await e.probeServiceReadiness(job('container-2'), startedAgo(16)) + expect(readinessWarnings()).to.have.length(2) + }) +}) diff --git a/src/test/unit/service/serviceReadiness.test.ts b/src/test/unit/service/serviceReadiness.test.ts index cf55eabe6..92fd1a35c 100644 --- a/src/test/unit/service/serviceReadiness.test.ts +++ b/src/test/unit/service/serviceReadiness.test.ts @@ -118,7 +118,7 @@ describe('ImagePullTracker', () => { }) describe('resolveServiceEngine', () => { - const job = (image: string) => ({ image }) as ServiceJob + const job = (image: string) => ({ image, exposedPorts: [8000, 8080] }) as ServiceJob it('recognizes the vLLM image, including a registry mirror of it', () => { expect(resolveServiceEngine(job('vllm/vllm-openai'))?.id).to.equal('vllm') @@ -140,7 +140,10 @@ describe('resolveServiceEngine', () => { }) it('falls back to containerImage when image is missing', () => { - const noImage = { containerImage: 'vllm/vllm-openai:v0.28.0' } as ServiceJob + const noImage = { + containerImage: 'vllm/vllm-openai:v0.28.0', + exposedPorts: [8000] + } as ServiceJob expect(resolveServiceEngine(noImage)?.id).to.equal('vllm') }) @@ -150,6 +153,15 @@ describe('resolveServiceEngine', () => { expect(resolveServiceEngine(job(''))).to.equal(null) }) + it('leaves an app that runs the engine behind its own port unrecognized', () => { + // A bundle that publishes only its app (vLLM on loopback inside) has nothing to probe. + const app = { image: 'vllm/vllm-openai', exposedPorts: [9119] } as ServiceJob + expect(resolveServiceEngine(app)).to.equal(null) + expect(resolveServiceEngine({ image: 'vllm/vllm-openai' } as ServiceJob)).to.equal( + null + ) + }) + it('reads the Hugging Face repo id out of the vLLM command', () => { const vllm = resolveServiceEngine(job('vllm/vllm-openai'))! expect( @@ -168,8 +180,21 @@ describe('resolveServiceEngine', () => { }) }) +describe('resolveServiceEngine — ComfyUI', () => { + const job = (image: string) => ({ image, exposedPorts: [8188] }) as ServiceJob + + it('recognizes the comfyui-boot image under any tag', () => { + expect(resolveServiceEngine(job('yanwk/comfyui-boot'))?.id).to.equal('comfyui') + expect( + resolveServiceEngine(job('yanwk/comfyui-boot:cu130-megapak-pt211'))?.id + ).to.equal('comfyui') + expect(resolveServiceEngine(job('someone/comfyui-boot'))).to.equal(null) + }) +}) + describe('resolveServiceEngine — llama.cpp', () => { - const job = (image: string, cmd?: string[]) => ({ image, dockerCmd: cmd }) as ServiceJob + const job = (image: string, cmd?: string[]) => + ({ image, dockerCmd: cmd, exposedPorts: [8000, 8080] }) as ServiceJob it('recognizes the llama.cpp image and probes /health', () => { const engine = resolveServiceEngine(job('ghcr.io/ggml-org/llama.cpp')) diff --git a/src/test/unit/service/unprobedServiceDownload.test.ts b/src/test/unit/service/unprobedServiceDownload.test.ts new file mode 100644 index 000000000..67bd9e3db --- /dev/null +++ b/src/test/unit/service/unprobedServiceDownload.test.ts @@ -0,0 +1,192 @@ +import { expect } from 'chai' +import sinon from 'sinon' +import { PassThrough, Readable } from 'stream' +import * as tarStream from 'tar-stream' +import { C2DEngineDocker } from '../../../components/c2d/compute_engine_docker.js' +import { ServiceStatusNumber } from '../../../@types/C2D/ServiceOnDemand.js' + +// A vLLM-based app bundle: the engine image, but only the app's own port published, so the node +// has no engine to probe and reports the launch script's download manifest alone. +describe('download progress for a service the node cannot probe', () => { + const CACHE = '/data/outputs/hf/hub/models--Qwen--Qwen3.8-27B-FP8' + + function engineWith(manifest: string | null, folderBytes: number | null) { + const engine: any = Object.create(C2DEngineDocker.prototype) + engine.serviceOpsInFlight = new Set() + engine.manifestCheckedAt = new Map() + engine.manifestDirSizes = new Map() + engine.getArchive = sinon.stub().callsFake(() => { + if (manifest === null) { + return Promise.reject(new Error('no such file')) + } + const pack = tarStream.pack() + pack.entry({ name: 'downloads.tsv' }, manifest) + pack.finalize() + return Promise.resolve(pack) + }) + engine.docker = { getContainer: () => ({ getArchive: engine.getArchive }) } + engine.getContainerDiskUsage = sinon.stub().resolves(folderBytes) + engine.db = { + isServiceLocked: sinon.stub().resolves(false), + updateServiceJobReadiness: sinon.stub().resolves(true) + } + return engine + } + + const job = (extra: object = {}) => ({ + serviceId: 'svc-1', + owner: '0x0000000000000000000000000000000000000001', + clusterHash: '0xcluster', + containerId: 'container-1', + image: 'vllm/vllm-openai', + exposedPorts: [9119], + status: ServiceStatusNumber.Running, + ...extra + }) + const startedAgo = (ms: number) => ({ + State: { StartedAt: new Date(Date.now() - ms).toISOString() } + }) + + it('records the manifest progress with no readiness', async () => { + const engine = engineWith(`dir\t1000\t${CACHE}\n`, 400) + await engine.probeServiceReadiness(job(), startedAgo(60_000)) + sinon.assert.calledOnce(engine.db.updateServiceJobReadiness) + const [serviceId, expected, readiness, download] = + engine.db.updateServiceJobReadiness.firstCall.args + expect(serviceId).to.equal('svc-1') + expect(expected.status).to.equal(ServiceStatusNumber.Running) + expect(readiness).to.equal(undefined) + expect(download).to.include({ + downloadedBytes: 400, + totalBytes: 1000, + percent: 40, + filesTotal: 1 + }) + sinon.assert.calledWith(engine.getContainerDiskUsage, 'container-1', CACHE) + }) + + it('checks at most once per probe period', async () => { + const engine = engineWith(`dir\t1000\t${CACHE}\n`, 400) + await engine.probeServiceReadiness(job(), startedAgo(60_000)) + await engine.probeServiceReadiness(job(), startedAgo(60_000)) + sinon.assert.calledOnce(engine.getArchive) + }) + + it('stops once the listed download is complete', async () => { + const engine = engineWith(`dir\t1000\t${CACHE}\n`, 1000) + const complete = { + downloadedBytes: 1000, + totalBytes: 1000, + percent: 100, + filesComplete: 1, + filesInFlight: 0, + filesTotal: 1, + updatedAt: Date.now() + } + await engine.probeServiceReadiness( + job({ modelDownload: complete }), + startedAgo(60_000) + ) + sinon.assert.notCalled(engine.getArchive) + sinon.assert.notCalled(engine.db.updateServiceJobReadiness) + }) + + it('stops looking once the discovery window has passed with no manifest', async () => { + const engine = engineWith(null, null) + await engine.probeServiceReadiness(job(), startedAgo(31 * 60 * 1000)) + sinon.assert.notCalled(engine.getArchive) + }) + + it('measures a manifest folder at most once per period, and again in a new container', async () => { + const engine = engineWith(null, 400) + await engine.manifestDirectoryBytes(job(), CACHE) + await engine.manifestDirectoryBytes(job(), CACHE) + sinon.assert.calledOnce(engine.getContainerDiskUsage) + await engine.manifestDirectoryBytes(job({ containerId: 'container-2' }), CACHE) + sinon.assert.calledTwice(engine.getContainerDiskUsage) + }) + + it('stops sampling six hours after the container started, complete or not', async () => { + const engine = engineWith(`dir\t0\t${CACHE}\n`, 400) + const unfinished = { + downloadedBytes: 400, + filesComplete: 0, + filesInFlight: 1, + filesTotal: 1, + updatedAt: Date.now() + } + await engine.probeServiceReadiness( + job({ modelDownload: unfinished }), + startedAgo(6 * 60 * 60 * 1000 + 1) + ) + sinon.assert.notCalled(engine.getArchive) + }) + + it('writes nothing for a service that has no manifest', async () => { + const engine = engineWith(null, null) + await engine.probeServiceReadiness(job(), startedAgo(60_000)) + sinon.assert.calledOnce(engine.getArchive) + sinon.assert.notCalled(engine.db.updateServiceJobReadiness) + }) +}) + +describe('getContainerDiskUsage', () => { + function engineWithOutput(output: string) { + const engine: any = Object.create(C2DEngineDocker.prototype) + const exec = { start: sinon.stub().resolves(Readable.from([Buffer.from(output)])) } + engine.docker = { + getContainer: () => ({ + inspect: sinon.stub().resolves({ State: { Running: true } }), + exec: sinon.stub().resolves(exec) + }) + } + return engine + } + + it('reads the size du prints for a long path', async () => { + const path = '/data/outputs/hf/hub/models--Qwen--Qwen3.8-test' + const engine = engineWithOutput(`5004096\t${path}\r\n`) + expect(await engine.getContainerDiskUsage('container-1', path)).to.equal(5004096) + }) + + it('finds the size after the error lines du prints for /proc', async () => { + const engine = engineWithOutput( + "du: cannot access '/proc/7/task/7/fd/3': No such file or directory\r\n" + + "du: cannot access '/proc/7/fd/4': No such file or directory\r\n" + + '204542432\t/\r\n' + ) + expect(await engine.getContainerDiskUsage('container-1', '/')).to.equal(204542432) + }) + + it('never reads an error message as a size', async () => { + const engine = engineWithOutput( + "du: cannot access '/data/models--org--model-2 1': No such file or directory\r\n" + ) + expect(await engine.getContainerDiskUsage('container-1', '/data/x')).to.equal(null) + }) +}) + +describe('getContainerDiskUsage timeout', () => { + let clock: sinon.SinonFakeTimers + beforeEach(() => { + clock = sinon.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) + }) + afterEach(() => { + clock.restore() + }) + + it('gives up on a du that never finishes, instead of hanging', async () => { + const stream = new PassThrough() + const engine: any = Object.create(C2DEngineDocker.prototype) + engine.docker = { + getContainer: () => ({ + inspect: sinon.stub().resolves({ State: { Running: true } }), + exec: sinon.stub().resolves({ start: sinon.stub().resolves(stream) }) + }) + } + const result = engine.getContainerDiskUsage('container-1', '/data/hung', 15_000) + await clock.tickAsync(15_000) + expect(await result).to.equal(null) + expect(stream.destroyed).to.equal(true) + }) +})