From 4133967030b99d36f13a9a26cef9e6b5df8dc6b4 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Mon, 31 Aug 2026 18:37:18 +0000 Subject: [PATCH 01/41] Materialize per-layer artifacts with explicit whiteout handling --- lib/images/layer_artifact.go | 838 ++++++++++++++++++++++++++++++ lib/images/layer_artifact_test.go | 390 ++++++++++++++ lib/paths/paths.go | 22 + 3 files changed, 1250 insertions(+) create mode 100644 lib/images/layer_artifact.go create mode 100644 lib/images/layer_artifact_test.go diff --git a/lib/images/layer_artifact.go b/lib/images/layer_artifact.go new file mode 100644 index 000000000..8ca313ea4 --- /dev/null +++ b/lib/images/layer_artifact.go @@ -0,0 +1,838 @@ +package images + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "syscall" + "time" + + "github.com/kernel/hypeman/lib/paths" + "github.com/klauspost/compress/zstd" + "golang.org/x/sys/unix" +) + +// OCI whiteout marker files. A ".wh." entry in directory D removes +// "" from D as inherited from lower layers; a ".wh..wh..opq" entry marks +// its directory opaque, hiding everything below it from lower layers. These +// are tar-level conventions: they do not compose on overlayfs by themselves, +// so composition must interpret them explicitly (see applyLayerTree). +const ( + whiteoutPrefix = ".wh." + opaqueWhiteout = ".wh..wh..opq" +) + +const layerRecordSchemaVersion = 1 + +// layerArtifact is the persisted record for one materialized layer artifact. +// The key is the compressed layer blob digest plus the artifact format, so the +// same layer can coexist in several materializations. +type layerArtifact struct { + SchemaVersion int `json:"schema_version"` + Digest string `json:"digest"` // compressed layer blob digest, sha256:... + DiffID string `json:"diff_id,omitempty"` + Format string `json:"format"` + SizeBytes int64 `json:"size_bytes"` // artifact bytes on disk + UnpackedBytes int64 `json:"unpacked_bytes"` + Entries int `json:"entries"` + Whiteouts []whiteoutRecord `json:"whiteouts,omitempty"` + CreatedAt time.Time `json:"created_at"` +} + +// validate checks a record read back from disk. The format fully determines +// the artifact options (erofs is always lz4-compressed, ext4 uncompressed), +// so only the format is stored. +func (a *layerArtifact) validate() error { + if a.SchemaVersion != layerRecordSchemaVersion { + return fmt.Errorf("unsupported schema version: %d", a.SchemaVersion) + } + if a.Digest == "" { + return fmt.Errorf("missing digest") + } + if a.Format != layerFormatErofs && a.Format != layerFormatExt4 { + return fmt.Errorf("invalid format: %s", a.Format) + } + if a.SizeBytes < 0 || a.UnpackedBytes < 0 || a.Entries < 0 { + return fmt.Errorf("invalid size or entry counts") + } + return nil +} + +// whiteoutRecord describes one whiteout marker found in a layer. Dir is the +// directory containing the marker relative to the layer root ("" for root). +// For opaque markers, Target is empty and the whole directory is hidden. +type whiteoutRecord struct { + Dir string `json:"dir"` + Target string `json:"target,omitempty"` + Opaque bool `json:"opaque,omitempty"` +} + +func (a *layerArtifact) matches(desc layerDescriptor) bool { + if a.Digest != desc.Digest || a.Format != layerArtifactFormat() { + return false + } + return desc.DiffID == "" || a.DiffID == desc.DiffID +} + +const ( + layerFormatErofs = "erofs" + layerFormatExt4 = "ext4" +) + +func layerArtifactFormat() string { + switch DefaultImageFormat { + case FormatErofs: + return layerFormatErofs + case FormatExt4: + return layerFormatExt4 + default: + return layerFormatErofs + } +} + +func layerArtifactPath(p *paths.Paths, layerHex string) string { + return p.ImageLayerArtifactForFormat(layerHex, layerArtifactFormat()) +} + +// readLayerRecord loads the artifact record for a layer digest, if present. +// A missing record returns (nil, nil): the layer simply was never +// materialized. +func readLayerRecord(p *paths.Paths, layerHex string) (*layerArtifact, error) { + data, err := os.ReadFile(p.ImageLayerRecord(layerHex)) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("read layer record: %w", err) + } + var record layerArtifact + if err := json.Unmarshal(data, &record); err != nil { + return nil, fmt.Errorf("unmarshal layer record: %w", err) + } + if err := record.validate(); err != nil { + return nil, fmt.Errorf("invalid layer record: %w", err) + } + return &record, nil +} + +// materializeLayerArtifact ensures a layer has a materialized artifact keyed +// by its blob digest, building it from the shared OCI cache blob when absent. +// The layer is unpacked into an isolated temp directory, converted to the +// default image format, and installed atomically; an interrupted build leaves +// only temp files that the next attempt replaces. No production caller yet: +// pull integration and composition land in later changes. +func (m *manager) materializeLayerArtifact(desc layerDescriptor) (*layerArtifact, error) { + layerHex := strings.TrimPrefix(desc.Digest, "sha256:") + if err := paths.ValidatePathComponent(layerHex); err != nil { + return nil, fmt.Errorf("invalid layer digest: %s", desc.Digest) + } + + if record, err := readLayerRecord(m.paths, layerHex); err != nil { + return nil, err + } else if record != nil && record.matches(desc) { + if _, statErr := os.Stat(layerArtifactPath(m.paths, layerHex)); statErr == nil { + return record, nil + } + // Record without artifact: rebuild below. + } + + blobPath := m.paths.OCICacheBlob(layerHex) + if _, err := os.Stat(blobPath); err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("layer blob missing from oci cache: %s", desc.Digest) + } + return nil, fmt.Errorf("stat layer blob: %w", err) + } + + layerDir := m.paths.ImageLayerDir(layerHex) + if err := os.MkdirAll(layerDir, 0755); err != nil { + return nil, fmt.Errorf("create layer directory: %w", err) + } + unpackDir, err := os.MkdirTemp(layerDir, ".unpack-*") + if err != nil { + return nil, fmt.Errorf("create unpack directory: %w", err) + } + defer os.RemoveAll(unpackDir) + + stats, err := unpackLayerBlob(blobPath, desc.MediaType, unpackDir) + if err != nil { + return nil, fmt.Errorf("unpack layer %s: %w", desc.Digest, err) + } + if desc.DiffID != "" && stats.diffID != desc.DiffID { + return nil, fmt.Errorf("layer %s diff id mismatch: got %s, want %s", desc.Digest, stats.diffID, desc.DiffID) + } + + return m.installLayerArtifact(desc, layerHex, unpackDir, stats) +} + +func (m *manager) installLayerArtifact(desc layerDescriptor, layerHex, unpackDir string, stats *unpackStats) (*layerArtifact, error) { + record := &layerArtifact{ + SchemaVersion: layerRecordSchemaVersion, + Digest: desc.Digest, + DiffID: desc.DiffID, + Format: layerArtifactFormat(), + UnpackedBytes: stats.unpackedBytes, + Entries: stats.entries, + // Nothing reads Whiteouts yet; composition re-derives whiteouts + // from the unpacked tree once it lands. + Whiteouts: stats.whiteouts, + CreatedAt: time.Now(), + } + + if err := installAtomically(layerArtifactPath(m.paths, layerHex), func(path string) error { + // The artifact intentionally retains .wh. marker files so + // composition can re-derive whiteouts from the tree itself. + size, convErr := ExportRootfs(unpackDir, path, DefaultImageFormat) + if convErr != nil { + return convErr + } + record.SizeBytes = size + return nil + }); err != nil { + return nil, fmt.Errorf("install layer artifact %s: %w", desc.Digest, err) + } + + data, err := json.MarshalIndent(record, "", " ") + if err != nil { + return nil, fmt.Errorf("marshal layer record: %w", err) + } + if err := writeJSONAtomic(m.paths.ImageLayerRecord(layerHex), data); err != nil { + _ = os.Remove(layerArtifactPath(m.paths, layerHex)) + return nil, fmt.Errorf("write layer record: %w", err) + } + return record, nil +} + +type unpackStats struct { + entries int + unpackedBytes int64 + diffID string + whiteouts []whiteoutRecord +} + +// unpackLayerBlob extracts one compressed layer blob into dest, preserving +// whiteout marker files and recording them. Paths are confined to dest. +func unpackLayerBlob(blobPath, mediaType, dest string) (*unpackStats, error) { + blob, err := os.Open(blobPath) + if err != nil { + return nil, fmt.Errorf("open blob: %w", err) + } + defer blob.Close() + + reader, closer, err := decompressLayer(blob, mediaType) + if err != nil { + return nil, err + } + defer closer.Close() + + hash := sha256.New() + stats := &unpackStats{whiteouts: make([]whiteoutRecord, 0)} + // Directory metadata is re-applied after extraction, once children + // exist, so tar directory mtimes are not overwritten by later writes. + var pendingDirs []pendingDir + hashedReader := io.TeeReader(reader, hash) + tr := tar.NewReader(hashedReader) + for { + header, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, fmt.Errorf("read tar entry: %w", err) + } + + target, err := safeJoin(dest, header.Name) + if err != nil { + return nil, err + } + stats.entries++ + + dir, base := filepath.Dir(header.Name), filepath.Base(header.Name) + if dir == "." { + dir = "" + } + if base == opaqueWhiteout { + stats.whiteouts = append(stats.whiteouts, whiteoutRecord{Dir: dir, Opaque: true}) + } else if strings.HasPrefix(base, whiteoutPrefix) { + targetName := strings.TrimPrefix(base, whiteoutPrefix) + if targetName == "" || targetName == "." || targetName == ".." { + return nil, fmt.Errorf("invalid whiteout entry: %s", header.Name) + } + stats.whiteouts = append(stats.whiteouts, whiteoutRecord{Dir: dir, Target: targetName}) + } + + if header.Typeflag == tar.TypeDir { + pendingDirs = append(pendingDirs, pendingDir{target: target, header: header}) + } + if err := extractTarEntry(tr, header, dest, target); err != nil { + return nil, fmt.Errorf("extract %s: %w", header.Name, err) + } + if header.Typeflag == tar.TypeReg { + stats.unpackedBytes += header.Size + } + } + if _, err := io.Copy(io.Discard, hashedReader); err != nil { + return nil, fmt.Errorf("drain layer: %w", err) + } + for _, dir := range pendingDirs { + if err := applyTarMetadata(dir.target, dir.header); err != nil { + return nil, fmt.Errorf("restore dir metadata %s: %w", dir.target, err) + } + } + stats.diffID = fmt.Sprintf("sha256:%x", hash.Sum(nil)) + return stats, nil +} + +type pendingDir struct { + target string + header *tar.Header +} + +// decompressLayer wraps the blob in the reader for its layer media type. Both +// OCI-style suffixes (+gzip, +zstd) and docker-style media types (tar.gzip, +// tar.zstd) are matched so neither encoding falls through to the raw path. +func decompressLayer(blob *os.File, mediaType string) (io.Reader, io.Closer, error) { + switch { + case strings.HasSuffix(mediaType, "+zstd"), strings.Contains(mediaType, "tar.zstd"): + decoder, err := zstd.NewReader(blob) + if err != nil { + return nil, nil, fmt.Errorf("zstd reader: %w", err) + } + return decoder, multiCloser{decoder.IOReadCloser(), blob}, nil + case strings.HasSuffix(mediaType, "+gzip"), strings.Contains(mediaType, "tar.gzip"): + gz, err := gzip.NewReader(blob) + if err != nil { + return nil, nil, fmt.Errorf("gzip reader: %w", err) + } + return gz, multiCloser{gz, blob}, nil + default: + return blob, blob, nil + } +} + +type multiCloser []io.Closer + +func (c multiCloser) Close() error { + var firstErr error + for _, closer := range c { + if err := closer.Close(); err != nil && firstErr == nil { + firstErr = err + } + } + return firstErr +} + +// safeJoin resolves a tar entry name inside root and rejects symlinked +// parents: extraction must never create an entry through a symlink an earlier +// tar entry planted, so existing parents are Lstat-walked and rejected rather +// than resolved. Symlink entries themselves may legitimately name paths that +// do not exist yet, so their targets are checked by resolve in +// validateSymlinkTarget instead. +func safeJoin(root, name string) (string, error) { + if filepath.IsAbs(name) { + return "", fmt.Errorf("tar entry escapes root: %s", name) + } + clean := filepath.Clean(name) + if clean == "." { + return filepath.Clean(root), nil + } + if clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("tar entry escapes root: %s", name) + } + root = filepath.Clean(root) + target := filepath.Join(root, clean) + if target == root { + return target, nil + } + if !strings.HasPrefix(target, root+string(filepath.Separator)) { + return "", fmt.Errorf("tar entry escapes root: %s", name) + } + for parent := filepath.Dir(target); parent != root; parent = filepath.Dir(parent) { + info, err := os.Lstat(parent) + if err != nil { + if os.IsNotExist(err) { + continue + } + return "", fmt.Errorf("inspect tar entry parent: %w", err) + } + if info.Mode()&os.ModeSymlink != 0 { + return "", fmt.Errorf("tar entry traverses symlink: %s", name) + } + if !info.IsDir() { + return "", fmt.Errorf("tar entry parent is not a directory: %s", parent) + } + } + return target, nil +} + +func validateSymlinkTarget(root, target, linkname string) error { + if filepath.IsAbs(linkname) { + return nil + } + resolved := filepath.Clean(filepath.Join(filepath.Dir(target), linkname)) + root = filepath.Clean(root) + if resolved != root && !strings.HasPrefix(resolved, root+string(filepath.Separator)) { + return fmt.Errorf("symlink target escapes root: %s", linkname) + } + return nil +} + +func extractTarEntry(tr *tar.Reader, header *tar.Header, root, target string) error { + switch header.Typeflag { + case tar.TypeDir: + return extractTarDir(target) + case tar.TypeReg: + return extractTarFile(tr, target, header) + case tar.TypeSymlink: + return extractTarSymlink(root, target, header) + case tar.TypeLink: + return extractTarHardlink(root, target, header) + case tar.TypeChar, tar.TypeBlock: + return extractTarDevice(target, header) + case tar.TypeFifo: + return extractTarFIFO(target, header) + default: + return nil + } +} + +func prepareTarTarget(target string) error { + if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { + return err + } + return removePath(target) +} + +func extractTarDir(target string) error { + if info, err := os.Lstat(target); err == nil && !info.IsDir() { + if err := removePath(target); err != nil { + return err + } + } + return os.MkdirAll(target, 0755) +} + +func extractTarFile(tr *tar.Reader, target string, header *tar.Header) error { + if err := prepareTarTarget(target); err != nil { + return err + } + file, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC|syscall.O_NOFOLLOW, 0644) + if err != nil { + return err + } + if _, err := io.Copy(file, tr); err != nil { + _ = file.Close() + return err + } + if err := file.Close(); err != nil { + return err + } + return applyTarMetadata(target, header) +} + +func extractTarSymlink(root, target string, header *tar.Header) error { + if err := validateSymlinkTarget(root, target, header.Linkname); err != nil { + return err + } + if err := prepareTarTarget(target); err != nil { + return err + } + if err := os.Symlink(header.Linkname, target); err != nil { + return err + } + return applyTarMetadata(target, header) +} + +func extractTarHardlink(root, target string, header *tar.Header) error { + linkTarget, err := safeJoin(root, header.Linkname) + if err != nil { + return err + } + if err := prepareTarTarget(target); err != nil { + return err + } + return os.Link(linkTarget, target) +} + +func extractTarDevice(target string, header *tar.Header) error { + if err := prepareTarTarget(target); err != nil { + return err + } + mode := uint32(syscall.S_IFCHR) + if header.Typeflag == tar.TypeBlock { + mode = uint32(syscall.S_IFBLK) + } + dev := int(unix.Mkdev(uint32(header.Devmajor), uint32(header.Devminor))) + if err := unix.Mknod(target, mode|uint32(header.FileInfo().Mode().Perm()), dev); err != nil { + if !errors.Is(err, unix.EPERM) { + return fmt.Errorf("mknod: %w", err) + } + file, openErr := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|syscall.O_NOFOLLOW, 0644) + if openErr != nil { + return fmt.Errorf("create rootless device placeholder: %w", openErr) + } + if closeErr := file.Close(); closeErr != nil { + return closeErr + } + } + return applyTarMetadata(target, header) +} + +func extractTarFIFO(target string, header *tar.Header) error { + if err := prepareTarTarget(target); err != nil { + return err + } + if err := syscall.Mkfifo(target, uint32(header.FileInfo().Mode().Perm())); err != nil { + return err + } + return applyTarMetadata(target, header) +} + +func applyTarMetadata(path string, header *tar.Header) error { + if err := os.Lchown(path, header.Uid, header.Gid); err != nil && !errors.Is(err, os.ErrPermission) && !errors.Is(err, unix.EPERM) { + return fmt.Errorf("restore ownership: %w", err) + } + if header.Typeflag != tar.TypeSymlink { + if err := os.Chmod(path, os.FileMode(header.Mode)); err != nil { + return fmt.Errorf("restore mode: %w", err) + } + mtime := header.ModTime + if mtime.IsZero() { + mtime = time.Now() + } + atime := header.AccessTime + if atime.IsZero() { + atime = mtime + } + if err := os.Chtimes(path, atime, mtime); err != nil { + return fmt.Errorf("restore timestamps: %w", err) + } + } + for name, value := range header.Xattrs { + if err := unix.Lsetxattr(path, name, []byte(value), 0); err != nil && !errors.Is(err, unix.ENOTSUP) && !errors.Is(err, unix.EPERM) { + return fmt.Errorf("restore xattr %s: %w", name, err) + } + } + return nil +} + +// removePath removes whatever entry occupies path, including non-empty +// directories, and tolerates a missing path. +func removePath(path string) error { + if err := os.RemoveAll(path); err != nil && !os.IsNotExist(err) { + return err + } + return nil +} + +// applyLayerTree merges one unpacked layer directory into targetDir following +// OCI whiteout semantics: whiteouts and opaque markers remove what lower layers +// contributed, then the layer's own entries are copied on top. Raw tar +// whiteout files are interpreted here rather than passed through, because +// overlayfs does not understand them. No production caller yet; composition +// lands in a later change. +func applyLayerTree(layerDir, targetDir string) error { + if err := os.MkdirAll(targetDir, 0755); err != nil { + return fmt.Errorf("create target directory: %w", err) + } + + // Phase 1: apply whiteouts against what is already in the target. + err := filepath.WalkDir(layerDir, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + base := entry.Name() + if path == layerDir || !strings.HasPrefix(base, whiteoutPrefix) { + return nil + } + rel, err := filepath.Rel(layerDir, path) + if err != nil { + return err + } + targetParent, err := safeJoin(targetDir, filepath.Dir(rel)) + if err != nil { + return err + } + if base == opaqueWhiteout { + return clearDirContents(targetParent) + } + hidden := strings.TrimPrefix(base, whiteoutPrefix) + target, err := safeJoin(targetDir, filepath.Join(filepath.Dir(rel), hidden)) + if err != nil { + return err + } + return removePath(target) + }) + if err != nil { + return fmt.Errorf("apply whiteouts: %w", err) + } + + // Phase 2: copy the layer's own entries, skipping whiteout markers. + // Directory metadata is deferred until all children are copied, so tar + // directory mtimes survive the merge. + var pendingDirs []dirMeta + hardlinks := make(map[hardlinkIdentity]string) + err = filepath.WalkDir(layerDir, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if path == layerDir { + return nil + } + if strings.HasPrefix(entry.Name(), whiteoutPrefix) { + if entry.IsDir() { + return fs.SkipDir + } + return nil + } + rel, err := filepath.Rel(layerDir, path) + if err != nil { + return err + } + target, err := safeJoin(targetDir, rel) + if err != nil { + return err + } + if entry.IsDir() { + info, err := entry.Info() + if err != nil { + return err + } + pendingDirs = append(pendingDirs, dirMeta{src: path, dst: target, info: info}) + } + return copyEntryInto(path, target, hardlinks) + }) + if err != nil { + return fmt.Errorf("copy layer tree: %w", err) + } + for _, dir := range pendingDirs { + if err := copyEntryMetadata(dir.src, dir.dst, dir.info); err != nil { + return fmt.Errorf("restore dir metadata %s: %w", dir.dst, err) + } + } + return nil +} + +type dirMeta struct { + src, dst string + info os.FileInfo +} + +// clearDirContents removes everything inside dir without removing dir itself, +// and without following symlinks. +func clearDirContents(dir string) error { + info, err := os.Lstat(dir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + if !info.IsDir() { + return removePath(dir) + } + entries, err := os.ReadDir(dir) + if err != nil { + return err + } + for _, entry := range entries { + if err := os.RemoveAll(filepath.Join(dir, entry.Name())); err != nil { + return err + } + } + return nil +} + +type hardlinkIdentity struct { + dev uint64 + ino uint64 +} + +// copyEntryInto copies one filesystem entry from src to dst, replacing any +// conflicting entry and preserving hardlinks within the layer. +func copyEntryInto(src, dst string, hardlinks map[hardlinkIdentity]string) error { + info, err := os.Lstat(src) + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { + return err + } + + switch info.Mode() & fs.ModeType { + case 0: + return copyRegularEntry(src, dst, info, hardlinks) + case fs.ModeDir: + return copyDirectoryEntry(src, dst, info) + case fs.ModeSymlink: + return copySymlinkEntry(src, dst) + default: + return copySpecialEntry(src, dst, info) + } +} + +func copyRegularEntry(src, dst string, info os.FileInfo, hardlinks map[hardlinkIdentity]string) error { + if err := removePath(dst); err != nil { + return err + } + if stat, ok := info.Sys().(*syscall.Stat_t); ok && stat.Nlink > 1 { + identity := hardlinkIdentity{dev: uint64(stat.Dev), ino: uint64(stat.Ino)} + if first, seen := hardlinks[identity]; seen { + return os.Link(first, dst) + } + hardlinks[identity] = dst + } + if err := copyFileContents(src, dst); err != nil { + return err + } + return copyEntryMetadata(src, dst, info) +} + +func copyDirectoryEntry(src, dst string, info os.FileInfo) error { + if existing, err := os.Lstat(dst); err == nil && !existing.IsDir() { + if err := removePath(dst); err != nil { + return err + } + } + return os.MkdirAll(dst, info.Mode().Perm()) +} + +func copySymlinkEntry(src, dst string) error { + linkTarget, err := os.Readlink(src) + if err != nil { + return err + } + if err := removePath(dst); err != nil { + return err + } + if err := os.Symlink(linkTarget, dst); err != nil { + return err + } + return nil +} + +func copySpecialEntry(src, dst string, info os.FileInfo) error { + if err := removePath(dst); err != nil { + return err + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return fmt.Errorf("unsupported entry type for %s", src) + } + mode, err := specialFileMode(info.Mode() & fs.ModeType) + if err != nil { + return fmt.Errorf("unsupported entry type for %s: %w", src, err) + } + if err := unix.Mknod(dst, mode|uint32(info.Mode().Perm()), int(stat.Rdev)); err != nil { + if !errors.Is(err, unix.EPERM) { + return fmt.Errorf("mknod: %w", err) + } + file, openErr := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|syscall.O_NOFOLLOW, 0644) + if openErr != nil { + return fmt.Errorf("create rootless device placeholder: %w", openErr) + } + if closeErr := file.Close(); closeErr != nil { + return closeErr + } + } + return copyEntryMetadata(src, dst, info) +} + +func specialFileMode(mode fs.FileMode) (uint32, error) { + switch mode { + case fs.ModeCharDevice: + return syscall.S_IFCHR, nil + case fs.ModeDevice: + return syscall.S_IFBLK, nil + case fs.ModeNamedPipe: + return syscall.S_IFIFO, nil + default: + return 0, fmt.Errorf("unsupported file mode") + } +} + +func copyEntryMetadata(src, dst string, info os.FileInfo) error { + if stat, ok := info.Sys().(*syscall.Stat_t); ok { + if err := os.Lchown(dst, int(stat.Uid), int(stat.Gid)); err != nil && !errors.Is(err, os.ErrPermission) && !errors.Is(err, unix.EPERM) { + return err + } + } + if info.Mode()&os.ModeSymlink == 0 { + mode := info.Mode().Perm() | info.Mode()&(os.ModeSetuid|os.ModeSetgid|os.ModeSticky) + if err := os.Chmod(dst, mode); err != nil { + return err + } + if err := os.Chtimes(dst, info.ModTime(), info.ModTime()); err != nil { + return err + } + if err := copyXattrs(src, dst); err != nil { + return err + } + } + return nil +} + +func copyXattrs(src, dst string) error { + size, err := unix.Llistxattr(src, nil) + if err != nil { + if errors.Is(err, unix.ENOTSUP) || errors.Is(err, unix.EPERM) { + return nil + } + return err + } + names := make([]byte, size) + if size > 0 { + n, err := unix.Llistxattr(src, names) + if err != nil { + return err + } + names = names[:n] + } + for _, attr := range bytes.Split(bytes.TrimSuffix(names, []byte{0}), []byte{0}) { + if len(attr) == 0 { + continue + } + name := string(attr) + size, err := unix.Lgetxattr(src, name, nil) + if err != nil { + if errors.Is(err, unix.ENOTSUP) || errors.Is(err, unix.EPERM) || errors.Is(err, unix.ENODATA) { + continue + } + return err + } + value := make([]byte, size) + if _, err := unix.Lgetxattr(src, name, value); err != nil { + return err + } + if err := unix.Lsetxattr(dst, name, value, 0); err != nil && !errors.Is(err, unix.ENOTSUP) && !errors.Is(err, unix.EPERM) { + return err + } + } + return nil +} + +func copyFileContents(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC|syscall.O_NOFOLLOW, 0644) + if err != nil { + return err + } + if _, err := io.Copy(out, in); err != nil { + out.Close() + return err + } + return out.Close() +} diff --git a/lib/images/layer_artifact_test.go b/lib/images/layer_artifact_test.go new file mode 100644 index 000000000..d92940b7a --- /dev/null +++ b/lib/images/layer_artifact_test.go @@ -0,0 +1,390 @@ +package images + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha256" + "errors" + "fmt" + "io" + "io/fs" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "golang.org/x/sys/unix" + + gcr "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/layout" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +// writeLayerTestLayout writes img into the shared OCI cache of p tagged with +// the image's digest, mirroring pullToOCILayout. +func writeLayerTestLayout(t *testing.T, p *paths.Paths, img gcr.Image) { + t.Helper() + digest, err := img.Digest() + require.NoError(t, err) + layoutPath, err := layout.Write(p.SystemOCICache(), empty.Index) + require.NoError(t, err) + require.NoError(t, layoutPath.AppendImage(img, layout.WithAnnotations(map[string]string{ + "org.opencontainers.image.ref.name": digestToLayoutTag(digest.String()), + }))) +} + +func layerDescFromImage(t *testing.T, img gcr.Image, index int) layerDescriptor { + t.Helper() + manifest, err := img.Manifest() + require.NoError(t, err) + configFile, err := img.ConfigFile() + require.NoError(t, err) + layer := manifest.Layers[index] + return layerDescriptor{ + Digest: layer.Digest.String(), + Size: layer.Size, + MediaType: string(layer.MediaType), + DiffID: configFile.RootFS.DiffIDs[index].String(), + } +} + +func TestMaterializeLayerArtifact(t *testing.T) { + if _, err := exec.LookPath("mkfs.erofs"); err != nil { + t.Skip("mkfs.erofs not available") + } + + p := paths.New(t.TempDir()) + img, err := mutate.AppendLayers(empty.Image, syntheticLayer(t, "base.txt", "base layer content")) + require.NoError(t, err) + writeLayerTestLayout(t, p, img) + + desc := layerDescFromImage(t, img, 0) + m := &manager{paths: p} + + record, err := m.materializeLayerArtifact(desc) + require.NoError(t, err) + require.Equal(t, desc.Digest, record.Digest) + require.Equal(t, desc.DiffID, record.DiffID) + require.Equal(t, layerArtifactFormat(), record.Format) + require.Greater(t, record.SizeBytes, int64(0)) + require.Greater(t, record.UnpackedBytes, int64(0)) + require.Greater(t, record.Entries, 0) + + layerHex := desc.Digest[len("sha256:"):] + _, err = os.Stat(p.ImageLayerArtifactForFormat(layerHex, layerArtifactFormat())) + require.NoError(t, err, "layer.erofs must be installed") + + // A second materialization reuses the existing artifact. + artifactInfo, err := os.Stat(p.ImageLayerArtifactForFormat(layerHex, layerArtifactFormat())) + require.NoError(t, err) + reused, err := m.materializeLayerArtifact(desc) + require.NoError(t, err) + require.True(t, record.CreatedAt.Equal(reused.CreatedAt), "reuse must return the stored record") + artifactInfoAfter, err := os.Stat(p.ImageLayerArtifactForFormat(layerHex, layerArtifactFormat())) + require.NoError(t, err) + require.Equal(t, artifactInfo.ModTime(), artifactInfoAfter.ModTime(), "reuse must not rebuild") +} + +func TestMaterializeLayerArtifactMissingBlob(t *testing.T) { + p := paths.New(t.TempDir()) + m := &manager{paths: p} + + _, err := m.materializeLayerArtifact(layerDescriptor{ + Digest: "sha256:abababababababababababababababababababababababababababababababab", + MediaType: "application/vnd.oci.image.layer.v1.tar+gzip", + }) + require.ErrorContains(t, err, "missing from oci cache") +} + +// whiteoutLayer builds a gzipped tar layer exercising whiteouts: a plain file, +// a whiteout marker, an opaque directory marker, and a whiteout+recreate pair. +func whiteoutLayer(t *testing.T) gcr.Layer { + t.Helper() + + var buf bytes.Buffer + gzw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gzw) + + writeEntry := func(header *tar.Header, content string) { + require.NoError(t, tw.WriteHeader(header)) + if content != "" { + _, err := tw.Write([]byte(content)) + require.NoError(t, err) + } + } + writeEntry(&tar.Header{Name: "keep.txt", Typeflag: tar.TypeReg, Mode: 0644, Size: 4}, "keep") + writeEntry(&tar.Header{Name: "gone/", Typeflag: tar.TypeDir, Mode: 0755}, "") + writeEntry(&tar.Header{Name: "gone/.wh.deleted.txt", Typeflag: tar.TypeReg, Mode: 0644, Size: 0}, "") + writeEntry(&tar.Header{Name: "opq/", Typeflag: tar.TypeDir, Mode: 0755}, "") + writeEntry(&tar.Header{Name: "opq/.wh..wh..opq", Typeflag: tar.TypeReg, Mode: 0644, Size: 0}, "") + writeEntry(&tar.Header{Name: "opq/fresh.txt", Typeflag: tar.TypeReg, Mode: 0644, Size: 5}, "fresh") + writeEntry(&tar.Header{Name: "added/", Typeflag: tar.TypeDir, Mode: 0755}, "") + writeEntry(&tar.Header{Name: "added/.wh.foo", Typeflag: tar.TypeReg, Mode: 0644, Size: 0}, "") + writeEntry(&tar.Header{Name: "added/foo", Typeflag: tar.TypeReg, Mode: 0644, Size: 3}, "new") + + require.NoError(t, tw.Close()) + require.NoError(t, gzw.Close()) + + data := buf.Bytes() + layer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(data)), nil + }) + require.NoError(t, err) + return layer +} + +func TestMaterializeLayerRecordsWhiteouts(t *testing.T) { + if _, err := exec.LookPath("mkfs.erofs"); err != nil { + t.Skip("mkfs.erofs not available") + } + + p := paths.New(t.TempDir()) + img, err := mutate.AppendLayers(empty.Image, whiteoutLayer(t)) + require.NoError(t, err) + writeLayerTestLayout(t, p, img) + + desc := layerDescFromImage(t, img, 0) + m := &manager{paths: p} + + record, err := m.materializeLayerArtifact(desc) + require.NoError(t, err) + + require.Contains(t, record.Whiteouts, whiteoutRecord{Dir: "gone", Target: "deleted.txt"}) + require.Contains(t, record.Whiteouts, whiteoutRecord{Dir: "opq", Opaque: true}) + require.Contains(t, record.Whiteouts, whiteoutRecord{Dir: "added", Target: "foo"}) + + // Opaque and whiteout markers are recorded distinctly. + opaqueCount := 0 + for _, whiteout := range record.Whiteouts { + if whiteout.Opaque { + opaqueCount++ + require.Empty(t, whiteout.Target) + } + } + require.Equal(t, 1, opaqueCount) +} + +func TestApplyLayerTreeWhiteoutSemantics(t *testing.T) { + root := t.TempDir() + targetDir := filepath.Join(root, "target") + layerDir := filepath.Join(root, "layer") + + // Lower state contributed by earlier layers. + require.NoError(t, os.MkdirAll(filepath.Join(targetDir, "opqdir"), 0755)) + require.NoError(t, os.MkdirAll(filepath.Join(targetDir, "swapdir"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(targetDir, "victim.txt"), []byte("old"), 0644)) + require.NoError(t, os.MkdirAll(filepath.Join(targetDir, "removedir"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(targetDir, "removedir", "inner.txt"), []byte("old"), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(targetDir, "keep.txt"), []byte("old"), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(targetDir, "opqdir", "stale.txt"), []byte("stale"), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(targetDir, "swapdir", "inner.txt"), []byte("inner"), 0644)) + + // Layer: whiteout victim.txt, opaque opqdir, replace swapdir with a file, + // and whiteout-then-recreate added/foo within the same layer. + require.NoError(t, os.MkdirAll(filepath.Join(layerDir, "opqdir"), 0755)) + require.NoError(t, os.MkdirAll(filepath.Join(layerDir, "added"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(layerDir, ".wh.victim.txt"), nil, 0644)) + require.NoError(t, os.WriteFile(filepath.Join(layerDir, ".wh.removedir"), nil, 0644)) + require.NoError(t, os.WriteFile(filepath.Join(layerDir, "keep.txt"), []byte("new"), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(layerDir, "opqdir", ".wh..wh..opq"), nil, 0644)) + require.NoError(t, os.WriteFile(filepath.Join(layerDir, "opqdir", "fresh.txt"), []byte("fresh"), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(layerDir, "swapdir"), []byte("now a file"), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(layerDir, "added", ".wh.foo"), nil, 0644)) + require.NoError(t, os.WriteFile(filepath.Join(layerDir, "added", "foo"), []byte("new"), 0644)) + + require.NoError(t, applyLayerTree(layerDir, targetDir)) + + // Whiteout removed the lower entry. + _, err := os.Lstat(filepath.Join(targetDir, "victim.txt")) + require.True(t, os.IsNotExist(err), "whiteout must delete the lower entry") + _, err = os.Lstat(filepath.Join(targetDir, "removedir")) + require.True(t, os.IsNotExist(err), "directory whiteout must delete the complete lower directory") + + // Regular file replacement. + data, err := os.ReadFile(filepath.Join(targetDir, "keep.txt")) + require.NoError(t, err) + require.Equal(t, "new", string(data)) + + // Opaque directory: stale content gone, layer content present. + _, err = os.Lstat(filepath.Join(targetDir, "opqdir", "stale.txt")) + require.True(t, os.IsNotExist(err), "opaque dir must hide lower contents") + data, err = os.ReadFile(filepath.Join(targetDir, "opqdir", "fresh.txt")) + require.NoError(t, err) + require.Equal(t, "fresh", string(data)) + + // Directory replaced by a file. + info, err := os.Lstat(filepath.Join(targetDir, "swapdir")) + require.NoError(t, err) + require.False(t, info.IsDir()) + + // Whiteout followed by recreate in the same layer keeps the new entry. + data, err = os.ReadFile(filepath.Join(targetDir, "added", "foo")) + require.NoError(t, err) + require.Equal(t, "new", string(data)) + + // Whiteout marker files never leak into the composed tree. + leaked := make([]string, 0) + require.NoError(t, filepath.WalkDir(targetDir, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if strings.HasPrefix(entry.Name(), whiteoutPrefix) { + leaked = append(leaked, path) + } + return nil + })) + require.Empty(t, leaked) +} + +func TestUnpackLayerBlobIncludesTrailingTarPaddingInDiffID(t *testing.T) { + root := t.TempDir() + blobPath := filepath.Join(root, "layer.tar.gz") + + var tarData bytes.Buffer + tw := tar.NewWriter(&tarData) + require.NoError(t, tw.WriteHeader(&tar.Header{Name: "file", Typeflag: tar.TypeReg, Mode: 0644, Size: 1})) + _, err := tw.Write([]byte("x")) + require.NoError(t, err) + require.NoError(t, tw.Close()) + _, err = tarData.Write(make([]byte, 512)) + require.NoError(t, err) + + var compressed bytes.Buffer + gzw := gzip.NewWriter(&compressed) + _, err = gzw.Write(tarData.Bytes()) + require.NoError(t, err) + require.NoError(t, gzw.Close()) + require.NoError(t, os.WriteFile(blobPath, compressed.Bytes(), 0644)) + + stats, err := unpackLayerBlob(blobPath, "application/vnd.oci.image.layer.v1.tar+gzip", filepath.Join(root, "dest")) + require.NoError(t, err) + want := sha256.Sum256(tarData.Bytes()) + require.Equal(t, "sha256:"+fmt.Sprintf("%x", want), stats.diffID) +} + +func TestUnpackLayerBlobRejectsSymlinkTraversal(t *testing.T) { + root := t.TempDir() + blobPath := filepath.Join(root, "layer.tar.gz") + var buf bytes.Buffer + gzw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gzw) + require.NoError(t, tw.WriteHeader(&tar.Header{Name: "link", Typeflag: tar.TypeSymlink, Linkname: "outside"})) + require.NoError(t, tw.WriteHeader(&tar.Header{Name: "link/escape", Typeflag: tar.TypeReg, Mode: 0644, Size: 1})) + _, err := tw.Write([]byte("x")) + require.NoError(t, err) + require.NoError(t, tw.Close()) + require.NoError(t, gzw.Close()) + require.NoError(t, os.WriteFile(blobPath, buf.Bytes(), 0644)) + + _, err = unpackLayerBlob(blobPath, "application/vnd.oci.image.layer.v1.tar+gzip", filepath.Join(root, "dest")) + require.ErrorContains(t, err, "symlink") +} + +func TestApplyLayerTreeSymlinksAndHardlinks(t *testing.T) { + root := t.TempDir() + targetDir := filepath.Join(root, "target") + layerDir := filepath.Join(root, "layer") + require.NoError(t, os.MkdirAll(targetDir, 0755)) + require.NoError(t, os.MkdirAll(layerDir, 0755)) + + // A symlink in the lower tree pointing at a file the new layer deletes: + // the symlink itself must be removed, never followed. + require.NoError(t, os.WriteFile(filepath.Join(targetDir, "real.txt"), []byte("real"), 0644)) + require.NoError(t, os.Symlink("real.txt", filepath.Join(targetDir, "alias"))) + + require.NoError(t, os.WriteFile(filepath.Join(layerDir, ".wh.alias"), nil, 0644)) + require.NoError(t, os.WriteFile(filepath.Join(layerDir, "a.txt"), []byte("content"), 0644)) + require.NoError(t, os.Link(filepath.Join(layerDir, "a.txt"), filepath.Join(layerDir, "b.txt"))) + require.NoError(t, os.Symlink("a.txt", filepath.Join(layerDir, "link-to-a"))) + + require.NoError(t, applyLayerTree(layerDir, targetDir)) + + _, err := os.Lstat(filepath.Join(targetDir, "alias")) + require.True(t, os.IsNotExist(err), "symlink whiteout must remove the link itself") + _, err = os.Lstat(filepath.Join(targetDir, "real.txt")) + require.NoError(t, err, "symlink target must survive an unrelated whiteout") + + infoA, err := os.Stat(filepath.Join(targetDir, "a.txt")) + require.NoError(t, err) + infoB, err := os.Stat(filepath.Join(targetDir, "b.txt")) + require.NoError(t, err) + require.Equal(t, int64(7), infoA.Size()) + require.True(t, os.SameFile(infoA, infoB), "hardlinks within the layer must stay linked") + + linkTarget, err := os.Readlink(filepath.Join(targetDir, "link-to-a")) + require.NoError(t, err) + require.Equal(t, "a.txt", linkTarget) +} + +func TestCopyXattrs(t *testing.T) { + root := t.TempDir() + src := filepath.Join(root, "src") + dst := filepath.Join(root, "dst") + require.NoError(t, os.WriteFile(src, []byte("payload"), 0644)) + require.NoError(t, os.WriteFile(dst, []byte("payload"), 0644)) + + err := unix.Lsetxattr(src, "user.one", []byte("1"), 0) + if errors.Is(err, unix.ENOTSUP) || errors.Is(err, unix.EPERM) { + t.Skip("filesystem does not support user xattrs") + } + require.NoError(t, err) + require.NoError(t, unix.Lsetxattr(src, "user.two", []byte("22"), 0)) + + require.NoError(t, copyXattrs(src, dst)) + + for name, want := range map[string]string{"user.one": "1", "user.two": "22"} { + size, err := unix.Lgetxattr(dst, name, nil) + require.NoError(t, err, "xattr %s must be copied", name) + value := make([]byte, size) + n, err := unix.Lgetxattr(dst, name, value) + require.NoError(t, err) + require.Equal(t, want, string(value[:n])) + } +} + +func TestUnpackLayerBlobPreservesDirMtime(t *testing.T) { + root := t.TempDir() + blobPath := filepath.Join(root, "layer.tar") + dirTime := time.Now().Add(-time.Hour).Truncate(time.Second) + + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + require.NoError(t, tw.WriteHeader(&tar.Header{Name: "d/", Typeflag: tar.TypeDir, Mode: 0755, ModTime: dirTime})) + require.NoError(t, tw.WriteHeader(&tar.Header{Name: "d/file.txt", Typeflag: tar.TypeReg, Mode: 0644, Size: 1})) + _, err := tw.Write([]byte("x")) + require.NoError(t, err) + require.NoError(t, tw.Close()) + require.NoError(t, os.WriteFile(blobPath, buf.Bytes(), 0644)) + + dest := filepath.Join(root, "dest") + _, err = unpackLayerBlob(blobPath, "application/vnd.oci.image.layer.v1.tar", dest) + require.NoError(t, err) + + info, err := os.Stat(filepath.Join(dest, "d")) + require.NoError(t, err) + require.True(t, info.ModTime().Equal(dirTime), "dir mtime must come from the tar header") +} + +func TestApplyLayerTreePreservesDirMtime(t *testing.T) { + root := t.TempDir() + targetDir := filepath.Join(root, "target") + layerDir := filepath.Join(root, "layer") + + old := time.Now().Add(-time.Hour).Truncate(time.Second) + require.NoError(t, os.MkdirAll(filepath.Join(layerDir, "d"), 0700)) + require.NoError(t, os.WriteFile(filepath.Join(layerDir, "d", "file.txt"), []byte("x"), 0644)) + require.NoError(t, os.Chtimes(filepath.Join(layerDir, "d"), old, old)) + + require.NoError(t, applyLayerTree(layerDir, targetDir)) + + info, err := os.Stat(filepath.Join(targetDir, "d")) + require.NoError(t, err) + require.True(t, info.ModTime().Equal(old), "dir mtime must survive the merge") +} diff --git a/lib/paths/paths.go b/lib/paths/paths.go index fc3f221eb..b47bdbbf1 100644 --- a/lib/paths/paths.go +++ b/lib/paths/paths.go @@ -177,6 +177,28 @@ func (p *Paths) ImageRepositoryTagSymlink(repository, tag string) string { return filepath.Join(p.ImageRepositoriesDir(), repository, tag) } +// ImageLayersDir returns the root directory of the per-layer artifact store. +// Layer artifacts are content-addressed by the compressed layer blob digest. +func (p *Paths) ImageLayersDir() string { + return filepath.Join(p.dataDir, "images", "layers") +} + +// ImageLayerDir returns the artifact directory for one layer digest. +func (p *Paths) ImageLayerDir(layerHex string) string { + return filepath.Join(p.ImageLayersDir(), layerHex) +} + +// ImageLayerArtifactForFormat returns the path to a materialized layer artifact. +func (p *Paths) ImageLayerArtifactForFormat(layerHex, format string) string { + return filepath.Join(p.ImageLayerDir(layerHex), "layer."+format) +} + +// ImageLayerRecord returns the path to the artifact record describing one +// materialized layer (format, options, whiteout inventory). +func (p *Paths) ImageLayerRecord(layerHex string) string { + return filepath.Join(p.ImageLayerDir(layerHex), "artifact.json") +} + // ImageDigestDir returns the directory for a specific image digest. func (p *Paths) ImageDigestDir(repository, digestHex string) string { return filepath.Join(p.dataDir, "images", repository, digestHex) From 285a8c6d09fabfe62ed49980db6117504b326457 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 1 Sep 2026 16:34:42 +0000 Subject: [PATCH 02/41] Harden layer artifact materialization --- lib/images/keyed_mutex.go | 39 +++++++++ lib/images/layer_artifact.go | 162 ++++++++++++++++++++++++++++++++--- lib/images/manager.go | 1 + 3 files changed, 192 insertions(+), 10 deletions(-) create mode 100644 lib/images/keyed_mutex.go diff --git a/lib/images/keyed_mutex.go b/lib/images/keyed_mutex.go new file mode 100644 index 000000000..609fd3017 --- /dev/null +++ b/lib/images/keyed_mutex.go @@ -0,0 +1,39 @@ +package images + +import "sync" + +type keyedMutex struct { + mu sync.Mutex + locks map[string]*keyedLock +} + +type keyedLock struct { + mu sync.Mutex + refs int +} + +func (m *keyedMutex) lock(key string) func() { + m.mu.Lock() + if m.locks == nil { + m.locks = make(map[string]*keyedLock) + } + lock := m.locks[key] + if lock == nil { + lock = &keyedLock{} + m.locks[key] = lock + } + lock.refs++ + m.mu.Unlock() + + lock.mu.Lock() + return func() { + lock.mu.Unlock() + + m.mu.Lock() + lock.refs-- + if lock.refs == 0 { + delete(m.locks, key) + } + m.mu.Unlock() + } +} diff --git a/lib/images/layer_artifact.go b/lib/images/layer_artifact.go index 8ca313ea4..dab913152 100644 --- a/lib/images/layer_artifact.go +++ b/lib/images/layer_artifact.go @@ -31,7 +31,11 @@ const ( opaqueWhiteout = ".wh..wh..opq" ) -const layerRecordSchemaVersion = 1 +const ( + layerRecordSchemaVersion = 1 + maxLayerEntries = 1_000_000 + maxLayerUnpackedBytes = 100 << 30 +) // layerArtifact is the persisted record for one materialized layer artifact. // The key is the compressed layer blob digest plus the artifact format, so the @@ -95,7 +99,7 @@ func layerArtifactFormat() string { case FormatExt4: return layerFormatExt4 default: - return layerFormatErofs + return "" } } @@ -131,6 +135,12 @@ func readLayerRecord(p *paths.Paths, layerHex string) (*layerArtifact, error) { // only temp files that the next attempt replaces. No production caller yet: // pull integration and composition land in later changes. func (m *manager) materializeLayerArtifact(desc layerDescriptor) (*layerArtifact, error) { + unlock := m.layerLocks.lock(desc.Digest) + defer unlock() + + if layerArtifactFormat() == "" { + return nil, fmt.Errorf("unsupported layer artifact format: %s", DefaultImageFormat) + } layerHex := strings.TrimPrefix(desc.Digest, "sha256:") if err := paths.ValidatePathComponent(layerHex); err != nil { return nil, fmt.Errorf("invalid layer digest: %s", desc.Digest) @@ -222,6 +232,9 @@ type unpackStats struct { // unpackLayerBlob extracts one compressed layer blob into dest, preserving // whiteout marker files and recording them. Paths are confined to dest. func unpackLayerBlob(blobPath, mediaType, dest string) (*unpackStats, error) { + if err := os.MkdirAll(dest, 0755); err != nil { + return nil, fmt.Errorf("create extraction root: %w", err) + } blob, err := os.Open(blobPath) if err != nil { return nil, fmt.Errorf("open blob: %w", err) @@ -239,7 +252,9 @@ func unpackLayerBlob(blobPath, mediaType, dest string) (*unpackStats, error) { // Directory metadata is re-applied after extraction, once children // exist, so tar directory mtimes are not overwritten by later writes. var pendingDirs []pendingDir - hashedReader := io.TeeReader(reader, hash) + var pendingHardlinks []pendingHardlink + limitedReader := &io.LimitedReader{R: reader, N: maxLayerUnpackedBytes + 1} + hashedReader := io.TeeReader(limitedReader, hash) tr := tar.NewReader(hashedReader) for { header, err := tr.Next() @@ -254,6 +269,12 @@ func unpackLayerBlob(blobPath, mediaType, dest string) (*unpackStats, error) { if err != nil { return nil, err } + if stats.entries >= maxLayerEntries { + return nil, fmt.Errorf("layer exceeds maximum entry count of %d", maxLayerEntries) + } + if header.Size > maxLayerUnpackedBytes-stats.unpackedBytes { + return nil, fmt.Errorf("layer exceeds maximum unpacked size of %d bytes", maxLayerUnpackedBytes) + } stats.entries++ dir, base := filepath.Dir(header.Name), filepath.Base(header.Name) @@ -273,16 +294,24 @@ func unpackLayerBlob(blobPath, mediaType, dest string) (*unpackStats, error) { if header.Typeflag == tar.TypeDir { pendingDirs = append(pendingDirs, pendingDir{target: target, header: header}) } - if err := extractTarEntry(tr, header, dest, target); err != nil { + if header.Typeflag == tar.TypeLink { + pendingHardlinks = append(pendingHardlinks, pendingHardlink{target: target, linkname: header.Linkname}) + } else if err := extractTarEntry(tr, header, dest, target); err != nil { return nil, fmt.Errorf("extract %s: %w", header.Name, err) } - if header.Typeflag == tar.TypeReg { + if header.Typeflag == tar.TypeReg || header.Typeflag == tar.TypeGNUSparse { stats.unpackedBytes += header.Size } } if _, err := io.Copy(io.Discard, hashedReader); err != nil { return nil, fmt.Errorf("drain layer: %w", err) } + if limitedReader.N == 0 { + return nil, fmt.Errorf("layer exceeds maximum unpacked size of %d bytes", maxLayerUnpackedBytes) + } + if err := resolveHardlinks(dest, pendingHardlinks); err != nil { + return nil, err + } for _, dir := range pendingDirs { if err := applyTarMetadata(dir.target, dir.header); err != nil { return nil, fmt.Errorf("restore dir metadata %s: %w", dir.target, err) @@ -297,6 +326,43 @@ type pendingDir struct { header *tar.Header } +type pendingHardlink struct { + target string + linkname string +} + +func resolveHardlinks(root string, pending []pendingHardlink) error { + for len(pending) > 0 { + resolved := 0 + remaining := make([]pendingHardlink, 0, len(pending)) + for _, link := range pending { + linkTarget, err := safeJoin(root, link.linkname) + if err != nil { + return err + } + if _, err := os.Lstat(linkTarget); err != nil { + if os.IsNotExist(err) { + remaining = append(remaining, link) + continue + } + return err + } + if err := prepareTarTarget(link.target); err != nil { + return err + } + if err := os.Link(linkTarget, link.target); err != nil { + return fmt.Errorf("create hardlink %s: %w", link.target, err) + } + resolved++ + } + if resolved == 0 { + return fmt.Errorf("hardlink target not found") + } + pending = remaining + } + return nil +} + // decompressLayer wraps the blob in the reader for its layer media type. Both // OCI-style suffixes (+gzip, +zstd) and docker-style media types (tar.gzip, // tar.zstd) are matched so neither encoding falls through to the raw path. @@ -338,6 +404,13 @@ func (c multiCloser) Close() error { // do not exist yet, so their targets are checked by resolve in // validateSymlinkTarget instead. func safeJoin(root, name string) (string, error) { + rootInfo, err := os.Lstat(root) + if err != nil { + return "", fmt.Errorf("inspect extraction root: %w", err) + } + if !rootInfo.IsDir() || rootInfo.Mode()&os.ModeSymlink != 0 { + return "", fmt.Errorf("extraction root is not a directory: %s", root) + } if filepath.IsAbs(name) { return "", fmt.Errorf("tar entry escapes root: %s", name) } @@ -390,7 +463,7 @@ func extractTarEntry(tr *tar.Reader, header *tar.Header, root, target string) er switch header.Typeflag { case tar.TypeDir: return extractTarDir(target) - case tar.TypeReg: + case tar.TypeReg, tar.TypeGNUSparse: return extractTarFile(tr, target, header) case tar.TypeSymlink: return extractTarSymlink(root, target, header) @@ -540,13 +613,23 @@ func removePath(path string) error { // whiteout files are interpreted here rather than passed through, because // overlayfs does not understand them. No production caller yet; composition // lands in a later change. -func applyLayerTree(layerDir, targetDir string) error { +func applyLayerTree(layerDir, targetDir string) (err error) { if err := os.MkdirAll(targetDir, 0755); err != nil { return fmt.Errorf("create target directory: %w", err) } + originalModes := make(map[string]fs.FileMode) + defer func() { + if restoreErr := restoreDirectoryModes(originalModes); restoreErr != nil { + if err == nil { + err = fmt.Errorf("restore directory modes: %w", restoreErr) + } else { + err = errors.Join(err, fmt.Errorf("restore directory modes: %w", restoreErr)) + } + } + }() // Phase 1: apply whiteouts against what is already in the target. - err := filepath.WalkDir(layerDir, func(path string, entry fs.DirEntry, err error) error { + err = filepath.WalkDir(layerDir, func(path string, entry fs.DirEntry, err error) error { if err != nil { return err } @@ -562,6 +645,9 @@ func applyLayerTree(layerDir, targetDir string) error { if err != nil { return err } + if err := makePathWritable(targetDir, targetParent, originalModes); err != nil { + return err + } if base == opaqueWhiteout { return clearDirContents(targetParent) } @@ -602,6 +688,9 @@ func applyLayerTree(layerDir, targetDir string) error { if err != nil { return err } + if err := makePathWritable(targetDir, filepath.Dir(target), originalModes); err != nil { + return err + } if entry.IsDir() { info, err := entry.Info() if err != nil { @@ -609,7 +698,13 @@ func applyLayerTree(layerDir, targetDir string) error { } pendingDirs = append(pendingDirs, dirMeta{src: path, dst: target, info: info}) } - return copyEntryInto(path, target, hardlinks) + if err := copyEntryInto(path, target, hardlinks); err != nil { + return err + } + if entry.IsDir() { + return makePathWritable(targetDir, target, originalModes) + } + return nil }) if err != nil { return fmt.Errorf("copy layer tree: %w", err) @@ -622,6 +717,49 @@ func applyLayerTree(layerDir, targetDir string) error { return nil } +func makePathWritable(root, path string, originalModes map[string]fs.FileMode) error { + root = filepath.Clean(root) + path = filepath.Clean(path) + if path != root && !strings.HasPrefix(path, root+string(filepath.Separator)) { + return fmt.Errorf("path is outside target root: %s", path) + } + for current := path; ; current = filepath.Dir(current) { + info, err := os.Lstat(current) + if err == nil { + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + return fmt.Errorf("target parent is not a directory: %s", current) + } + if _, recorded := originalModes[current]; !recorded { + originalModes[current] = info.Mode() + if err := os.Chmod(current, info.Mode().Perm()|0700); err != nil { + return err + } + } + } else if !os.IsNotExist(err) { + return err + } + if current == root { + return nil + } + } +} + +func restoreDirectoryModes(originalModes map[string]fs.FileMode) error { + var restoreErr error + for path, mode := range originalModes { + if _, err := os.Lstat(path); os.IsNotExist(err) { + continue + } else if err != nil { + restoreErr = errors.Join(restoreErr, err) + continue + } + if err := os.Chmod(path, mode.Perm()|mode&(os.ModeSetuid|os.ModeSetgid|os.ModeSticky)); err != nil { + restoreErr = errors.Join(restoreErr, err) + } + } + return restoreErr +} + type dirMeta struct { src, dst string info os.FileInfo @@ -717,7 +855,11 @@ func copySymlinkEntry(src, dst string) error { if err := os.Symlink(linkTarget, dst); err != nil { return err } - return nil + info, err := os.Lstat(src) + if err != nil { + return err + } + return copyEntryMetadata(src, dst, info) } func copySpecialEntry(src, dst string, info os.FileInfo) error { diff --git a/lib/images/manager.go b/lib/images/manager.go index 8b905a417..a5ea4b732 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -75,6 +75,7 @@ type manager struct { ociClient *ociClient queue *queue.Queue createMu sync.Mutex + layerLocks keyedMutex diskUsageMu sync.RWMutex tagGenerations map[string]uint64 requestedTags map[string]string // newest pull's digest per requested tag From 820d196b41432785bc727f5282317ee509a0aee0 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:52:16 +0000 Subject: [PATCH 03/41] Address layer artifact review findings --- lib/images/layer_artifact.go | 35 +++++++++++++++---------------- lib/images/layer_artifact_test.go | 33 +++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/lib/images/layer_artifact.go b/lib/images/layer_artifact.go index dab913152..52d378829 100644 --- a/lib/images/layer_artifact.go +++ b/lib/images/layer_artifact.go @@ -251,8 +251,8 @@ func unpackLayerBlob(blobPath, mediaType, dest string) (*unpackStats, error) { stats := &unpackStats{whiteouts: make([]whiteoutRecord, 0)} // Directory metadata is re-applied after extraction, once children // exist, so tar directory mtimes are not overwritten by later writes. - var pendingDirs []pendingDir - var pendingHardlinks []pendingHardlink + pendingDirs := make([]pendingDir, 0) + pendingHardlinks := make([]pendingHardlink, 0) limitedReader := &io.LimitedReader{R: reader, N: maxLayerUnpackedBytes + 1} hashedReader := io.TeeReader(limitedReader, hash) tr := tar.NewReader(hashedReader) @@ -295,6 +295,8 @@ func unpackLayerBlob(blobPath, mediaType, dest string) (*unpackStats, error) { pendingDirs = append(pendingDirs, pendingDir{target: target, header: header}) } if header.Typeflag == tar.TypeLink { + // Hardlinks may reference entries that appear later in the tar, so + // resolve them after extracting all non-link entries. pendingHardlinks = append(pendingHardlinks, pendingHardlink{target: target, linkname: header.Linkname}) } else if err := extractTarEntry(tr, header, dest, target); err != nil { return nil, fmt.Errorf("extract %s: %w", header.Name, err) @@ -401,7 +403,7 @@ func (c multiCloser) Close() error { // parents: extraction must never create an entry through a symlink an earlier // tar entry planted, so existing parents are Lstat-walked and rejected rather // than resolved. Symlink entries themselves may legitimately name paths that -// do not exist yet, so their targets are checked by resolve in +// do not exist yet, so their targets are checked by // validateSymlinkTarget instead. func safeJoin(root, name string) (string, error) { rootInfo, err := os.Lstat(root) @@ -447,6 +449,10 @@ func safeJoin(root, name string) (string, error) { return target, nil } +// validateSymlinkTarget permits absolute targets because OCI images may use +// them and the image filesystem must preserve their link text. Extraction +// still rejects symlink traversal for later entries, while composition copies +// symlink text without following it. func validateSymlinkTarget(root, target, linkname string) error { if filepath.IsAbs(linkname) { return nil @@ -467,8 +473,6 @@ func extractTarEntry(tr *tar.Reader, header *tar.Header, root, target string) er return extractTarFile(tr, target, header) case tar.TypeSymlink: return extractTarSymlink(root, target, header) - case tar.TypeLink: - return extractTarHardlink(root, target, header) case tar.TypeChar, tar.TypeBlock: return extractTarDevice(target, header) case tar.TypeFifo: @@ -525,17 +529,6 @@ func extractTarSymlink(root, target string, header *tar.Header) error { return applyTarMetadata(target, header) } -func extractTarHardlink(root, target string, header *tar.Header) error { - linkTarget, err := safeJoin(root, header.Linkname) - if err != nil { - return err - } - if err := prepareTarTarget(target); err != nil { - return err - } - return os.Link(linkTarget, target) -} - func extractTarDevice(target string, header *tar.Header) error { if err := prepareTarTarget(target); err != nil { return err @@ -665,7 +658,7 @@ func applyLayerTree(layerDir, targetDir string) (err error) { // Phase 2: copy the layer's own entries, skipping whiteout markers. // Directory metadata is deferred until all children are copied, so tar // directory mtimes survive the merge. - var pendingDirs []dirMeta + pendingDirs := make([]dirMeta, 0) hardlinks := make(map[hardlinkIdentity]string) err = filepath.WalkDir(layerDir, func(path string, entry fs.DirEntry, err error) error { if err != nil { @@ -897,6 +890,8 @@ func specialFileMode(mode fs.FileMode) (uint32, error) { return syscall.S_IFBLK, nil case fs.ModeNamedPipe: return syscall.S_IFIFO, nil + case fs.ModeSocket: + return syscall.S_IFSOCK, nil default: return 0, fmt.Errorf("unsupported file mode") } @@ -913,7 +908,11 @@ func copyEntryMetadata(src, dst string, info os.FileInfo) error { if err := os.Chmod(dst, mode); err != nil { return err } - if err := os.Chtimes(dst, info.ModTime(), info.ModTime()); err != nil { + atime := info.ModTime() + if stat, ok := info.Sys().(*syscall.Stat_t); ok { + atime = time.Unix(stat.Atim.Sec, stat.Atim.Nsec) + } + if err := os.Chtimes(dst, atime, info.ModTime()); err != nil { return err } if err := copyXattrs(src, dst); err != nil { diff --git a/lib/images/layer_artifact_test.go b/lib/images/layer_artifact_test.go index d92940b7a..610b7d490 100644 --- a/lib/images/layer_artifact_test.go +++ b/lib/images/layer_artifact_test.go @@ -13,6 +13,7 @@ import ( "os/exec" "path/filepath" "strings" + "syscall" "testing" "time" @@ -287,6 +288,12 @@ func TestUnpackLayerBlobRejectsSymlinkTraversal(t *testing.T) { require.ErrorContains(t, err, "symlink") } +func TestValidateSymlinkTargetAllowsAbsoluteTargets(t *testing.T) { + root := t.TempDir() + target := filepath.Join(root, "link") + require.NoError(t, validateSymlinkTarget(root, target, "/etc/resolv.conf")) +} + func TestApplyLayerTreeSymlinksAndHardlinks(t *testing.T) { root := t.TempDir() targetDir := filepath.Join(root, "target") @@ -372,6 +379,32 @@ func TestUnpackLayerBlobPreservesDirMtime(t *testing.T) { require.True(t, info.ModTime().Equal(dirTime), "dir mtime must come from the tar header") } +func TestCopyEntryMetadataPreservesAccessTime(t *testing.T) { + root := t.TempDir() + src := filepath.Join(root, "src") + dst := filepath.Join(root, "dst") + require.NoError(t, os.WriteFile(src, []byte("payload"), 0644)) + require.NoError(t, os.WriteFile(dst, nil, 0644)) + + atime := time.Now().Add(-2 * time.Hour).Truncate(time.Second) + mtime := time.Now().Add(-time.Hour).Truncate(time.Second) + require.NoError(t, os.Chtimes(src, atime, mtime)) + info, err := os.Lstat(src) + require.NoError(t, err) + require.NoError(t, copyEntryMetadata(src, dst, info)) + + var stat unix.Stat_t + require.NoError(t, unix.Lstat(dst, &stat)) + require.True(t, time.Unix(stat.Atim.Sec, stat.Atim.Nsec).Equal(atime), "atime must be preserved") + require.True(t, time.Unix(stat.Mtim.Sec, stat.Mtim.Nsec).Equal(mtime), "mtime must be preserved") +} + +func TestSpecialFileModeSupportsSockets(t *testing.T) { + mode, err := specialFileMode(fs.ModeSocket) + require.NoError(t, err) + require.Equal(t, uint32(syscall.S_IFSOCK), mode) +} + func TestApplyLayerTreePreservesDirMtime(t *testing.T) { root := t.TempDir() targetDir := filepath.Join(root, "target") From 480372f6f6b8995328f89e5fd3f914c38440dde3 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Wed, 2 Sep 2026 13:30:05 +0000 Subject: [PATCH 04/41] Fix layer artifact composition edge cases --- lib/images/disk_usage.go | 29 +- lib/images/disk_usage_test.go | 15 + lib/images/layer_artifact.go | 597 ++++++------------------------ lib/images/layer_artifact_test.go | 112 +++++- lib/images/layer_tree.go | 487 ++++++++++++++++++++++++ lib/images/manager.go | 2 +- lib/paths/paths.go | 8 +- 7 files changed, 740 insertions(+), 510 deletions(-) create mode 100644 lib/images/layer_tree.go diff --git a/lib/images/disk_usage.go b/lib/images/disk_usage.go index b65dba1ac..8d098a46d 100644 --- a/lib/images/disk_usage.go +++ b/lib/images/disk_usage.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "syscall" ) @@ -82,6 +83,28 @@ func totalReadyImageBytesFromMetadata(imagesDir string) (int64, error) { return total, nil } +// totalLayerArtifactBytesFromFilesystem sums materialized layer artifacts. +func totalLayerArtifactBytesFromFilesystem(layersDir string) (int64, error) { + var total int64 + err := filepath.Walk(layersDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + if info.IsDir() || !strings.HasPrefix(info.Name(), "layer.") { + return nil + } + total += info.Size() + return nil + }) + if err != nil && !os.IsNotExist(err) { + return 0, fmt.Errorf("walk layer artifacts: %w", err) + } + return total, nil +} + // totalOCICacheBlobBytesFromFilesystem sums blob sizes directly from the OCI cache blob store. // This counts the actual bytes on disk, including any blob files that are currently // present but no longer referenced by the OCI layout index. @@ -158,7 +181,11 @@ func (m *manager) computeDiskUsageTotals() (int64, int64, error) { if err != nil { return 0, 0, err } - return readyImageBytes, ociCacheBytes, nil + layerArtifactBytes, err := totalLayerArtifactBytesFromFilesystem(m.paths.ImageLayersDir()) + if err != nil { + return 0, 0, err + } + return readyImageBytes, ociCacheBytes + layerArtifactBytes, nil } func totalRootfsBytesInDigestDir(digestDir string) (int64, error) { diff --git a/lib/images/disk_usage_test.go b/lib/images/disk_usage_test.go index bd6056fef..5b6a441ed 100644 --- a/lib/images/disk_usage_test.go +++ b/lib/images/disk_usage_test.go @@ -64,6 +64,21 @@ func TestTotalReadyImageBytesFromMetadata_DeduplicatesMalformedAliases(t *testin require.Equal(t, int64(len("shared-rootfs")), total) } +func TestTotalLayerArtifactBytesFromFilesystem(t *testing.T) { + t.Parallel() + + layersDir := t.TempDir() + digestDir := filepath.Join(layersDir, "a", "b") + require.NoError(t, os.MkdirAll(digestDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(digestDir, "layer.erofs"), []byte("erofs"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(digestDir, "layer.ext4"), []byte("ext4"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(digestDir, "artifact.erofs.json"), []byte("record"), 0o644)) + + total, err := totalLayerArtifactBytesFromFilesystem(layersDir) + require.NoError(t, err) + require.Equal(t, int64(len("erofs")+len("ext4")), total) +} + func TestTotalReadyImageBytesFromMetadata_UsesRootfsFallbackForReadyImageWithoutSize(t *testing.T) { t.Parallel() diff --git a/lib/images/layer_artifact.go b/lib/images/layer_artifact.go index 52d378829..fa51003a2 100644 --- a/lib/images/layer_artifact.go +++ b/lib/images/layer_artifact.go @@ -2,14 +2,12 @@ package images import ( "archive/tar" - "bytes" "compress/gzip" "crypto/sha256" "encoding/json" "errors" "fmt" "io" - "io/fs" "os" "path/filepath" "strings" @@ -37,19 +35,20 @@ const ( maxLayerUnpackedBytes = 100 << 30 ) +var errCorruptLayerRecord = errors.New("corrupt layer record") + // layerArtifact is the persisted record for one materialized layer artifact. // The key is the compressed layer blob digest plus the artifact format, so the // same layer can coexist in several materializations. type layerArtifact struct { - SchemaVersion int `json:"schema_version"` - Digest string `json:"digest"` // compressed layer blob digest, sha256:... - DiffID string `json:"diff_id,omitempty"` - Format string `json:"format"` - SizeBytes int64 `json:"size_bytes"` // artifact bytes on disk - UnpackedBytes int64 `json:"unpacked_bytes"` - Entries int `json:"entries"` - Whiteouts []whiteoutRecord `json:"whiteouts,omitempty"` - CreatedAt time.Time `json:"created_at"` + SchemaVersion int `json:"schema_version"` + Digest string `json:"digest"` // compressed layer blob digest, sha256:... + DiffID string `json:"diff_id,omitempty"` + Format string `json:"format"` + SizeBytes int64 `json:"size_bytes"` // artifact bytes on disk + UnpackedBytes int64 `json:"unpacked_bytes"` + Entries int `json:"entries"` + CreatedAt time.Time `json:"created_at"` } // validate checks a record read back from disk. The format fully determines @@ -62,7 +61,7 @@ func (a *layerArtifact) validate() error { if a.Digest == "" { return fmt.Errorf("missing digest") } - if a.Format != layerFormatErofs && a.Format != layerFormatExt4 { + if a.Format != string(FormatErofs) && a.Format != string(FormatExt4) { return fmt.Errorf("invalid format: %s", a.Format) } if a.SizeBytes < 0 || a.UnpackedBytes < 0 || a.Entries < 0 { @@ -71,15 +70,6 @@ func (a *layerArtifact) validate() error { return nil } -// whiteoutRecord describes one whiteout marker found in a layer. Dir is the -// directory containing the marker relative to the layer root ("" for root). -// For opaque markers, Target is empty and the whole directory is hidden. -type whiteoutRecord struct { - Dir string `json:"dir"` - Target string `json:"target,omitempty"` - Opaque bool `json:"opaque,omitempty"` -} - func (a *layerArtifact) matches(desc layerDescriptor) bool { if a.Digest != desc.Digest || a.Format != layerArtifactFormat() { return false @@ -87,17 +77,10 @@ func (a *layerArtifact) matches(desc layerDescriptor) bool { return desc.DiffID == "" || a.DiffID == desc.DiffID } -const ( - layerFormatErofs = "erofs" - layerFormatExt4 = "ext4" -) - func layerArtifactFormat() string { switch DefaultImageFormat { - case FormatErofs: - return layerFormatErofs - case FormatExt4: - return layerFormatExt4 + case FormatErofs, FormatExt4: + return string(DefaultImageFormat) default: return "" } @@ -107,11 +90,15 @@ func layerArtifactPath(p *paths.Paths, layerHex string) string { return p.ImageLayerArtifactForFormat(layerHex, layerArtifactFormat()) } +func layerArtifactRecordPath(p *paths.Paths, layerHex string) string { + return p.ImageLayerRecordForFormat(layerHex, layerArtifactFormat()) +} + // readLayerRecord loads the artifact record for a layer digest, if present. // A missing record returns (nil, nil): the layer simply was never // materialized. func readLayerRecord(p *paths.Paths, layerHex string) (*layerArtifact, error) { - data, err := os.ReadFile(p.ImageLayerRecord(layerHex)) + data, err := os.ReadFile(layerArtifactRecordPath(p, layerHex)) if err != nil { if os.IsNotExist(err) { return nil, nil @@ -120,14 +107,26 @@ func readLayerRecord(p *paths.Paths, layerHex string) (*layerArtifact, error) { } var record layerArtifact if err := json.Unmarshal(data, &record); err != nil { - return nil, fmt.Errorf("unmarshal layer record: %w", err) + return nil, fmt.Errorf("%w: unmarshal layer record: %v", errCorruptLayerRecord, err) } if err := record.validate(); err != nil { - return nil, fmt.Errorf("invalid layer record: %w", err) + return nil, fmt.Errorf("%w: invalid layer record: %v", errCorruptLayerRecord, err) } return &record, nil } +func discardLayerCache(p *paths.Paths, layerHex string) error { + for _, path := range []string{ + layerArtifactRecordPath(p, layerHex), + layerArtifactPath(p, layerHex), + } { + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + return err + } + } + return nil +} + // materializeLayerArtifact ensures a layer has a materialized artifact keyed // by its blob digest, building it from the shared OCI cache blob when absent. // The layer is unpacked into an isolated temp directory, converted to the @@ -143,11 +142,16 @@ func (m *manager) materializeLayerArtifact(desc layerDescriptor) (*layerArtifact } layerHex := strings.TrimPrefix(desc.Digest, "sha256:") if err := paths.ValidatePathComponent(layerHex); err != nil { - return nil, fmt.Errorf("invalid layer digest: %s", desc.Digest) + return nil, fmt.Errorf("invalid layer digest %s: %w", desc.Digest, err) } if record, err := readLayerRecord(m.paths, layerHex); err != nil { - return nil, err + if !errors.Is(err, errCorruptLayerRecord) { + return nil, err + } + if discardErr := discardLayerCache(m.paths, layerHex); discardErr != nil { + return nil, fmt.Errorf("discard corrupt layer cache: %w", discardErr) + } } else if record != nil && record.matches(desc) { if _, statErr := os.Stat(layerArtifactPath(m.paths, layerHex)); statErr == nil { return record, nil @@ -192,10 +196,7 @@ func (m *manager) installLayerArtifact(desc layerDescriptor, layerHex, unpackDir Format: layerArtifactFormat(), UnpackedBytes: stats.unpackedBytes, Entries: stats.entries, - // Nothing reads Whiteouts yet; composition re-derives whiteouts - // from the unpacked tree once it lands. - Whiteouts: stats.whiteouts, - CreatedAt: time.Now(), + CreatedAt: time.Now(), } if err := installAtomically(layerArtifactPath(m.paths, layerHex), func(path string) error { @@ -215,7 +216,7 @@ func (m *manager) installLayerArtifact(desc layerDescriptor, layerHex, unpackDir if err != nil { return nil, fmt.Errorf("marshal layer record: %w", err) } - if err := writeJSONAtomic(m.paths.ImageLayerRecord(layerHex), data); err != nil { + if err := writeJSONAtomic(layerArtifactRecordPath(m.paths, layerHex), data); err != nil { _ = os.Remove(layerArtifactPath(m.paths, layerHex)) return nil, fmt.Errorf("write layer record: %w", err) } @@ -226,11 +227,12 @@ type unpackStats struct { entries int unpackedBytes int64 diffID string - whiteouts []whiteoutRecord } -// unpackLayerBlob extracts one compressed layer blob into dest, preserving -// whiteout marker files and recording them. Paths are confined to dest. +// unpackLayerBlob extracts one compressed layer blob into dest while preserving +// whiteout marker files. It intentionally does not use umoci's layer unpacker: +// umoci consumes whiteouts while this store must retain them for composition. +// Paths are confined to dest. func unpackLayerBlob(blobPath, mediaType, dest string) (*unpackStats, error) { if err := os.MkdirAll(dest, 0755); err != nil { return nil, fmt.Errorf("create extraction root: %w", err) @@ -248,7 +250,7 @@ func unpackLayerBlob(blobPath, mediaType, dest string) (*unpackStats, error) { defer closer.Close() hash := sha256.New() - stats := &unpackStats{whiteouts: make([]whiteoutRecord, 0)} + stats := &unpackStats{} // Directory metadata is re-applied after extraction, once children // exist, so tar directory mtimes are not overwritten by later writes. pendingDirs := make([]pendingDir, 0) @@ -277,18 +279,12 @@ func unpackLayerBlob(blobPath, mediaType, dest string) (*unpackStats, error) { } stats.entries++ - dir, base := filepath.Dir(header.Name), filepath.Base(header.Name) - if dir == "." { - dir = "" - } - if base == opaqueWhiteout { - stats.whiteouts = append(stats.whiteouts, whiteoutRecord{Dir: dir, Opaque: true}) - } else if strings.HasPrefix(base, whiteoutPrefix) { + base := filepath.Base(header.Name) + if strings.HasPrefix(base, whiteoutPrefix) && base != opaqueWhiteout { targetName := strings.TrimPrefix(base, whiteoutPrefix) if targetName == "" || targetName == "." || targetName == ".." { return nil, fmt.Errorf("invalid whiteout entry: %s", header.Name) } - stats.whiteouts = append(stats.whiteouts, whiteoutRecord{Dir: dir, Target: targetName}) } if header.Typeflag == tar.TypeDir { @@ -334,33 +330,42 @@ type pendingHardlink struct { } func resolveHardlinks(root string, pending []pendingHardlink) error { - for len(pending) > 0 { - resolved := 0 - remaining := make([]pendingHardlink, 0, len(pending)) - for _, link := range pending { - linkTarget, err := safeJoin(root, link.linkname) - if err != nil { - return err - } - if _, err := os.Lstat(linkTarget); err != nil { - if os.IsNotExist(err) { - remaining = append(remaining, link) - continue - } - return err - } - if err := prepareTarTarget(link.target); err != nil { - return err - } - if err := os.Link(linkTarget, link.target); err != nil { - return fmt.Errorf("create hardlink %s: %w", link.target, err) - } - resolved++ + waiting := make(map[string][]pendingHardlink) + ready := make([]pendingHardlink, 0, len(pending)) + for _, link := range pending { + linkTarget, err := safeJoin(root, link.linkname) + if err != nil { + return err } - if resolved == 0 { - return fmt.Errorf("hardlink target not found") + if _, err := os.Lstat(linkTarget); err == nil { + ready = append(ready, link) + } else if os.IsNotExist(err) { + waiting[linkTarget] = append(waiting[linkTarget], link) + } else { + return err + } + } + + resolved := 0 + for len(ready) > 0 { + link := ready[0] + ready = ready[1:] + if err := prepareTarTarget(link.target); err != nil { + return err + } + if linkTarget, err := safeJoin(root, link.linkname); err != nil { + return err + } else if err := os.Link(linkTarget, link.target); err != nil { + return fmt.Errorf("create hardlink %s -> %s: %w", link.target, link.linkname, err) + } + resolved++ + ready = append(ready, waiting[link.target]...) + delete(waiting, link.target) + } + if resolved != len(pending) { + for _, links := range waiting { + return fmt.Errorf("hardlink target not found for %s", links[0].linkname) } - pending = remaining } return nil } @@ -542,12 +547,8 @@ func extractTarDevice(target string, header *tar.Header) error { if !errors.Is(err, unix.EPERM) { return fmt.Errorf("mknod: %w", err) } - file, openErr := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|syscall.O_NOFOLLOW, 0644) - if openErr != nil { - return fmt.Errorf("create rootless device placeholder: %w", openErr) - } - if closeErr := file.Close(); closeErr != nil { - return closeErr + if err := createRootlessDevicePlaceholder(target); err != nil { + return err } } return applyTarMetadata(target, header) @@ -563,417 +564,37 @@ func extractTarFIFO(target string, header *tar.Header) error { return applyTarMetadata(target, header) } -func applyTarMetadata(path string, header *tar.Header) error { - if err := os.Lchown(path, header.Uid, header.Gid); err != nil && !errors.Is(err, os.ErrPermission) && !errors.Is(err, unix.EPERM) { - return fmt.Errorf("restore ownership: %w", err) - } - if header.Typeflag != tar.TypeSymlink { - if err := os.Chmod(path, os.FileMode(header.Mode)); err != nil { - return fmt.Errorf("restore mode: %w", err) - } - mtime := header.ModTime - if mtime.IsZero() { - mtime = time.Now() - } - atime := header.AccessTime - if atime.IsZero() { - atime = mtime - } - if err := os.Chtimes(path, atime, mtime); err != nil { - return fmt.Errorf("restore timestamps: %w", err) - } - } - for name, value := range header.Xattrs { - if err := unix.Lsetxattr(path, name, []byte(value), 0); err != nil && !errors.Is(err, unix.ENOTSUP) && !errors.Is(err, unix.EPERM) { - return fmt.Errorf("restore xattr %s: %w", name, err) - } - } - return nil -} - -// removePath removes whatever entry occupies path, including non-empty -// directories, and tolerates a missing path. -func removePath(path string) error { - if err := os.RemoveAll(path); err != nil && !os.IsNotExist(err) { - return err - } - return nil -} - -// applyLayerTree merges one unpacked layer directory into targetDir following -// OCI whiteout semantics: whiteouts and opaque markers remove what lower layers -// contributed, then the layer's own entries are copied on top. Raw tar -// whiteout files are interpreted here rather than passed through, because -// overlayfs does not understand them. No production caller yet; composition -// lands in a later change. -func applyLayerTree(layerDir, targetDir string) (err error) { - if err := os.MkdirAll(targetDir, 0755); err != nil { - return fmt.Errorf("create target directory: %w", err) - } - originalModes := make(map[string]fs.FileMode) - defer func() { - if restoreErr := restoreDirectoryModes(originalModes); restoreErr != nil { - if err == nil { - err = fmt.Errorf("restore directory modes: %w", restoreErr) - } else { - err = errors.Join(err, fmt.Errorf("restore directory modes: %w", restoreErr)) - } - } - }() - - // Phase 1: apply whiteouts against what is already in the target. - err = filepath.WalkDir(layerDir, func(path string, entry fs.DirEntry, err error) error { - if err != nil { - return err - } - base := entry.Name() - if path == layerDir || !strings.HasPrefix(base, whiteoutPrefix) { - return nil - } - rel, err := filepath.Rel(layerDir, path) - if err != nil { - return err - } - targetParent, err := safeJoin(targetDir, filepath.Dir(rel)) - if err != nil { - return err - } - if err := makePathWritable(targetDir, targetParent, originalModes); err != nil { - return err - } - if base == opaqueWhiteout { - return clearDirContents(targetParent) - } - hidden := strings.TrimPrefix(base, whiteoutPrefix) - target, err := safeJoin(targetDir, filepath.Join(filepath.Dir(rel), hidden)) - if err != nil { - return err - } - return removePath(target) - }) - if err != nil { - return fmt.Errorf("apply whiteouts: %w", err) - } - - // Phase 2: copy the layer's own entries, skipping whiteout markers. - // Directory metadata is deferred until all children are copied, so tar - // directory mtimes survive the merge. - pendingDirs := make([]dirMeta, 0) - hardlinks := make(map[hardlinkIdentity]string) - err = filepath.WalkDir(layerDir, func(path string, entry fs.DirEntry, err error) error { - if err != nil { - return err - } - if path == layerDir { - return nil - } - if strings.HasPrefix(entry.Name(), whiteoutPrefix) { - if entry.IsDir() { - return fs.SkipDir - } - return nil - } - rel, err := filepath.Rel(layerDir, path) - if err != nil { - return err - } - target, err := safeJoin(targetDir, rel) - if err != nil { - return err - } - if err := makePathWritable(targetDir, filepath.Dir(target), originalModes); err != nil { - return err - } - if entry.IsDir() { - info, err := entry.Info() - if err != nil { - return err - } - pendingDirs = append(pendingDirs, dirMeta{src: path, dst: target, info: info}) - } - if err := copyEntryInto(path, target, hardlinks); err != nil { - return err - } - if entry.IsDir() { - return makePathWritable(targetDir, target, originalModes) - } - return nil - }) +func createRootlessDevicePlaceholder(path string) error { + file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|syscall.O_NOFOLLOW, 0644) if err != nil { - return fmt.Errorf("copy layer tree: %w", err) - } - for _, dir := range pendingDirs { - if err := copyEntryMetadata(dir.src, dir.dst, dir.info); err != nil { - return fmt.Errorf("restore dir metadata %s: %w", dir.dst, err) - } - } - return nil -} - -func makePathWritable(root, path string, originalModes map[string]fs.FileMode) error { - root = filepath.Clean(root) - path = filepath.Clean(path) - if path != root && !strings.HasPrefix(path, root+string(filepath.Separator)) { - return fmt.Errorf("path is outside target root: %s", path) - } - for current := path; ; current = filepath.Dir(current) { - info, err := os.Lstat(current) - if err == nil { - if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { - return fmt.Errorf("target parent is not a directory: %s", current) - } - if _, recorded := originalModes[current]; !recorded { - originalModes[current] = info.Mode() - if err := os.Chmod(current, info.Mode().Perm()|0700); err != nil { - return err - } - } - } else if !os.IsNotExist(err) { - return err - } - if current == root { - return nil - } - } -} - -func restoreDirectoryModes(originalModes map[string]fs.FileMode) error { - var restoreErr error - for path, mode := range originalModes { - if _, err := os.Lstat(path); os.IsNotExist(err) { - continue - } else if err != nil { - restoreErr = errors.Join(restoreErr, err) - continue - } - if err := os.Chmod(path, mode.Perm()|mode&(os.ModeSetuid|os.ModeSetgid|os.ModeSticky)); err != nil { - restoreErr = errors.Join(restoreErr, err) - } + return fmt.Errorf("create rootless device placeholder: %w", err) } - return restoreErr -} - -type dirMeta struct { - src, dst string - info os.FileInfo -} - -// clearDirContents removes everything inside dir without removing dir itself, -// and without following symlinks. -func clearDirContents(dir string) error { - info, err := os.Lstat(dir) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - if !info.IsDir() { - return removePath(dir) - } - entries, err := os.ReadDir(dir) - if err != nil { - return err - } - for _, entry := range entries { - if err := os.RemoveAll(filepath.Join(dir, entry.Name())); err != nil { - return err - } - } - return nil -} - -type hardlinkIdentity struct { - dev uint64 - ino uint64 -} - -// copyEntryInto copies one filesystem entry from src to dst, replacing any -// conflicting entry and preserving hardlinks within the layer. -func copyEntryInto(src, dst string, hardlinks map[hardlinkIdentity]string) error { - info, err := os.Lstat(src) - if err != nil { - return err - } - if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { - return err - } - - switch info.Mode() & fs.ModeType { - case 0: - return copyRegularEntry(src, dst, info, hardlinks) - case fs.ModeDir: - return copyDirectoryEntry(src, dst, info) - case fs.ModeSymlink: - return copySymlinkEntry(src, dst) - default: - return copySpecialEntry(src, dst, info) - } -} - -func copyRegularEntry(src, dst string, info os.FileInfo, hardlinks map[hardlinkIdentity]string) error { - if err := removePath(dst); err != nil { - return err - } - if stat, ok := info.Sys().(*syscall.Stat_t); ok && stat.Nlink > 1 { - identity := hardlinkIdentity{dev: uint64(stat.Dev), ino: uint64(stat.Ino)} - if first, seen := hardlinks[identity]; seen { - return os.Link(first, dst) - } - hardlinks[identity] = dst - } - if err := copyFileContents(src, dst); err != nil { - return err - } - return copyEntryMetadata(src, dst, info) -} - -func copyDirectoryEntry(src, dst string, info os.FileInfo) error { - if existing, err := os.Lstat(dst); err == nil && !existing.IsDir() { - if err := removePath(dst); err != nil { - return err - } - } - return os.MkdirAll(dst, info.Mode().Perm()) -} - -func copySymlinkEntry(src, dst string) error { - linkTarget, err := os.Readlink(src) - if err != nil { - return err - } - if err := removePath(dst); err != nil { - return err - } - if err := os.Symlink(linkTarget, dst); err != nil { - return err - } - info, err := os.Lstat(src) - if err != nil { - return err - } - return copyEntryMetadata(src, dst, info) -} - -func copySpecialEntry(src, dst string, info os.FileInfo) error { - if err := removePath(dst); err != nil { + if err := file.Close(); err != nil { return err } - stat, ok := info.Sys().(*syscall.Stat_t) - if !ok { - return fmt.Errorf("unsupported entry type for %s", src) - } - mode, err := specialFileMode(info.Mode() & fs.ModeType) - if err != nil { - return fmt.Errorf("unsupported entry type for %s: %w", src, err) - } - if err := unix.Mknod(dst, mode|uint32(info.Mode().Perm()), int(stat.Rdev)); err != nil { - if !errors.Is(err, unix.EPERM) { - return fmt.Errorf("mknod: %w", err) - } - file, openErr := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|syscall.O_NOFOLLOW, 0644) - if openErr != nil { - return fmt.Errorf("create rootless device placeholder: %w", openErr) - } - if closeErr := file.Close(); closeErr != nil { - return closeErr - } - } - return copyEntryMetadata(src, dst, info) -} - -func specialFileMode(mode fs.FileMode) (uint32, error) { - switch mode { - case fs.ModeCharDevice: - return syscall.S_IFCHR, nil - case fs.ModeDevice: - return syscall.S_IFBLK, nil - case fs.ModeNamedPipe: - return syscall.S_IFIFO, nil - case fs.ModeSocket: - return syscall.S_IFSOCK, nil - default: - return 0, fmt.Errorf("unsupported file mode") - } -} - -func copyEntryMetadata(src, dst string, info os.FileInfo) error { - if stat, ok := info.Sys().(*syscall.Stat_t); ok { - if err := os.Lchown(dst, int(stat.Uid), int(stat.Gid)); err != nil && !errors.Is(err, os.ErrPermission) && !errors.Is(err, unix.EPERM) { - return err - } - } - if info.Mode()&os.ModeSymlink == 0 { - mode := info.Mode().Perm() | info.Mode()&(os.ModeSetuid|os.ModeSetgid|os.ModeSticky) - if err := os.Chmod(dst, mode); err != nil { - return err - } - atime := info.ModTime() - if stat, ok := info.Sys().(*syscall.Stat_t); ok { - atime = time.Unix(stat.Atim.Sec, stat.Atim.Nsec) - } - if err := os.Chtimes(dst, atime, info.ModTime()); err != nil { - return err - } - if err := copyXattrs(src, dst); err != nil { - return err - } - } return nil } -func copyXattrs(src, dst string) error { - size, err := unix.Llistxattr(src, nil) - if err != nil { - if errors.Is(err, unix.ENOTSUP) || errors.Is(err, unix.EPERM) { - return nil - } - return err - } - names := make([]byte, size) - if size > 0 { - n, err := unix.Llistxattr(src, names) - if err != nil { - return err - } - names = names[:n] - } - for _, attr := range bytes.Split(bytes.TrimSuffix(names, []byte{0}), []byte{0}) { - if len(attr) == 0 { - continue - } - name := string(attr) - size, err := unix.Lgetxattr(src, name, nil) - if err != nil { - if errors.Is(err, unix.ENOTSUP) || errors.Is(err, unix.EPERM) || errors.Is(err, unix.ENODATA) { - continue - } - return err - } - value := make([]byte, size) - if _, err := unix.Lgetxattr(src, name, value); err != nil { - return err - } - if err := unix.Lsetxattr(dst, name, value, 0); err != nil && !errors.Is(err, unix.ENOTSUP) && !errors.Is(err, unix.EPERM) { - return err - } - } - return nil +func applyTarMetadata(path string, header *tar.Header) error { + mtime := header.ModTime + if mtime.IsZero() { + mtime = time.Now() + } + atime := header.AccessTime + if atime.IsZero() { + atime = mtime + } + return applyEntryMetadata(path, entryMetadata{ + uid: header.Uid, + gid: header.Gid, + hasOwner: true, + mode: os.FileMode(header.Mode), + symlink: header.Typeflag == tar.TypeSymlink, + atime: atime, + mtime: mtime, + xattrs: tarXattrs(header), + }) } -func copyFileContents(src, dst string) error { - in, err := os.Open(src) - if err != nil { - return err - } - defer in.Close() - out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC|syscall.O_NOFOLLOW, 0644) - if err != nil { - return err - } - if _, err := io.Copy(out, in); err != nil { - out.Close() - return err - } - return out.Close() -} +// removePath removes whatever entry occupies path, including non-empty +// directories, and tolerates a missing path. diff --git a/lib/images/layer_artifact_test.go b/lib/images/layer_artifact_test.go index 610b7d490..924eaf091 100644 --- a/lib/images/layer_artifact_test.go +++ b/lib/images/layer_artifact_test.go @@ -5,6 +5,7 @@ import ( "bytes" "compress/gzip" "crypto/sha256" + "encoding/json" "errors" "fmt" "io" @@ -93,6 +94,43 @@ func TestMaterializeLayerArtifact(t *testing.T) { require.Equal(t, artifactInfo.ModTime(), artifactInfoAfter.ModTime(), "reuse must not rebuild") } +func TestMaterializeLayerArtifactRecoversCorruptRecord(t *testing.T) { + if _, err := exec.LookPath("mkfs.ext4"); err != nil { + t.Skip("mkfs.ext4 not available") + } + originalFormat := DefaultImageFormat + DefaultImageFormat = FormatExt4 + t.Cleanup(func() { DefaultImageFormat = originalFormat }) + + p := paths.New(t.TempDir()) + img, err := mutate.AppendLayers(empty.Image, syntheticLayer(t, "base.txt", "base layer content")) + require.NoError(t, err) + writeLayerTestLayout(t, p, img) + + desc := layerDescFromImage(t, img, 0) + m := &manager{paths: p} + first, err := m.materializeLayerArtifact(desc) + require.NoError(t, err) + + layerHex := desc.Digest[len("sha256:"):] + require.NoError(t, os.WriteFile( + p.ImageLayerRecordForFormat(layerHex, layerArtifactFormat()), + []byte("{not valid json"), + 0600, + )) + + second, err := m.materializeLayerArtifact(desc) + require.NoError(t, err) + require.NotEqual(t, first.CreatedAt, second.CreatedAt, "corrupt record must trigger a rebuild") + require.FileExists(t, p.ImageLayerArtifactForFormat(layerHex, layerArtifactFormat())) + + data, err := os.ReadFile(p.ImageLayerRecordForFormat(layerHex, layerArtifactFormat())) + require.NoError(t, err) + var record layerArtifact + require.NoError(t, json.Unmarshal(data, &record)) + require.NoError(t, record.validate()) +} + func TestMaterializeLayerArtifactMissingBlob(t *testing.T) { p := paths.New(t.TempDir()) m := &manager{paths: p} @@ -141,7 +179,7 @@ func whiteoutLayer(t *testing.T) gcr.Layer { return layer } -func TestMaterializeLayerRecordsWhiteouts(t *testing.T) { +func TestMaterializeLayerDoesNotPersistWhiteoutInventory(t *testing.T) { if _, err := exec.LookPath("mkfs.erofs"); err != nil { t.Skip("mkfs.erofs not available") } @@ -154,22 +192,12 @@ func TestMaterializeLayerRecordsWhiteouts(t *testing.T) { desc := layerDescFromImage(t, img, 0) m := &manager{paths: p} - record, err := m.materializeLayerArtifact(desc) + _, err = m.materializeLayerArtifact(desc) require.NoError(t, err) - require.Contains(t, record.Whiteouts, whiteoutRecord{Dir: "gone", Target: "deleted.txt"}) - require.Contains(t, record.Whiteouts, whiteoutRecord{Dir: "opq", Opaque: true}) - require.Contains(t, record.Whiteouts, whiteoutRecord{Dir: "added", Target: "foo"}) - - // Opaque and whiteout markers are recorded distinctly. - opaqueCount := 0 - for _, whiteout := range record.Whiteouts { - if whiteout.Opaque { - opaqueCount++ - require.Empty(t, whiteout.Target) - } - } - require.Equal(t, 1, opaqueCount) + data, err := os.ReadFile(p.ImageLayerRecordForFormat(desc.Digest[len("sha256:"):], layerArtifactFormat())) + require.NoError(t, err) + require.NotContains(t, string(data), "whiteouts") } func TestApplyLayerTreeWhiteoutSemantics(t *testing.T) { @@ -244,6 +272,38 @@ func TestApplyLayerTreeWhiteoutSemantics(t *testing.T) { require.Empty(t, leaked) } +func TestApplyLayerTreePreservesUpperDirectoryMode(t *testing.T) { + root := t.TempDir() + targetDir := filepath.Join(root, "target") + layerDir := filepath.Join(root, "layer") + + require.NoError(t, os.MkdirAll(filepath.Join(targetDir, "etc"), 0755)) + require.NoError(t, os.MkdirAll(filepath.Join(layerDir, "etc"), 0750)) + require.NoError(t, os.WriteFile(filepath.Join(layerDir, "etc", "new.txt"), []byte("new"), 0644)) + + require.NoError(t, applyLayerTree(layerDir, targetDir)) + + info, err := os.Stat(filepath.Join(targetDir, "etc")) + require.NoError(t, err) + require.Equal(t, os.FileMode(0750), info.Mode().Perm()) +} + +func TestApplyLayerTreeWhiteoutRemovesReadOnlyDirectory(t *testing.T) { + root := t.TempDir() + targetDir := filepath.Join(root, "target") + layerDir := filepath.Join(root, "layer") + + require.NoError(t, os.MkdirAll(filepath.Join(targetDir, "gone"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(targetDir, "gone", "inner.txt"), []byte("old"), 0644)) + require.NoError(t, os.Chmod(filepath.Join(targetDir, "gone"), 0555)) + require.NoError(t, os.MkdirAll(layerDir, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(layerDir, ".wh.gone"), nil, 0644)) + + require.NoError(t, applyLayerTree(layerDir, targetDir)) + _, err := os.Lstat(filepath.Join(targetDir, "gone")) + require.ErrorIs(t, err, os.ErrNotExist) +} + func TestUnpackLayerBlobIncludesTrailingTarPaddingInDiffID(t *testing.T) { root := t.TempDir() blobPath := filepath.Join(root, "layer.tar.gz") @@ -270,6 +330,24 @@ func TestUnpackLayerBlobIncludesTrailingTarPaddingInDiffID(t *testing.T) { require.Equal(t, "sha256:"+fmt.Sprintf("%x", want), stats.diffID) } +func TestResolveHardlinksResolvesDependencyChain(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "source"), []byte("content"), 0644)) + + pending := []pendingHardlink{ + {target: filepath.Join(root, "third"), linkname: "second"}, + {target: filepath.Join(root, "second"), linkname: "first"}, + {target: filepath.Join(root, "first"), linkname: "source"}, + } + require.NoError(t, resolveHardlinks(root, pending)) + + source, err := os.Stat(filepath.Join(root, "source")) + require.NoError(t, err) + third, err := os.Stat(filepath.Join(root, "third")) + require.NoError(t, err) + require.True(t, os.SameFile(source, third)) +} + func TestUnpackLayerBlobRejectsSymlinkTraversal(t *testing.T) { root := t.TempDir() blobPath := filepath.Join(root, "layer.tar.gz") @@ -344,7 +422,9 @@ func TestCopyXattrs(t *testing.T) { require.NoError(t, err) require.NoError(t, unix.Lsetxattr(src, "user.two", []byte("22"), 0)) - require.NoError(t, copyXattrs(src, dst)) + xattrs, err := readXattrs(src) + require.NoError(t, err) + require.NoError(t, applyXattrs(dst, xattrs)) for name, want := range map[string]string{"user.one": "1", "user.two": "22"} { size, err := unix.Lgetxattr(dst, name, nil) diff --git a/lib/images/layer_tree.go b/lib/images/layer_tree.go new file mode 100644 index 000000000..5ff887896 --- /dev/null +++ b/lib/images/layer_tree.go @@ -0,0 +1,487 @@ +package images + +import ( + "archive/tar" + "bytes" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "syscall" + "time" + + "golang.org/x/sys/unix" +) + +func removePath(path string) error { + if err := makeTreeWritable(path); err != nil && !os.IsNotExist(err) { + return err + } + if err := os.RemoveAll(path); err != nil && !os.IsNotExist(err) { + return err + } + return nil +} + +func makeTreeWritable(path string) error { + info, err := os.Lstat(path) + if err != nil { + return err + } + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return nil + } + if err := os.Chmod(path, info.Mode().Perm()|0700); err != nil { + return err + } + return filepath.WalkDir(path, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() && entry.Type()&os.ModeSymlink == 0 { + info, err := entry.Info() + if err != nil { + return err + } + return os.Chmod(path, info.Mode().Perm()|0700) + } + return nil + }) +} + +// applyLayerTree merges one unpacked layer directory into targetDir following +// OCI whiteout semantics: whiteouts and opaque markers remove what lower layers +// contributed, then the layer's own entries are copied on top. Raw tar +// whiteout files are interpreted here rather than passed through, because +// overlayfs does not understand them. No production caller yet; composition +// lands in a later change. +func applyLayerTree(layerDir, targetDir string) (err error) { + if err := os.MkdirAll(targetDir, 0755); err != nil { + return fmt.Errorf("create target directory: %w", err) + } + originalModes := make(map[string]fs.FileMode) + defer func() { + if restoreErr := restoreDirectoryModes(originalModes); restoreErr != nil { + if err == nil { + err = fmt.Errorf("restore directory modes: %w", restoreErr) + } else { + err = errors.Join(err, fmt.Errorf("restore directory modes: %w", restoreErr)) + } + } + }() + + // Phase 1: apply whiteouts against what is already in the target. + err = filepath.WalkDir(layerDir, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + base := entry.Name() + if path == layerDir || !strings.HasPrefix(base, whiteoutPrefix) { + return nil + } + rel, err := filepath.Rel(layerDir, path) + if err != nil { + return err + } + targetParent, err := safeJoin(targetDir, filepath.Dir(rel)) + if err != nil { + return err + } + if err := makePathWritable(targetDir, targetParent, originalModes); err != nil { + return err + } + if base == opaqueWhiteout { + return clearDirContents(targetParent) + } + hidden := strings.TrimPrefix(base, whiteoutPrefix) + target, err := safeJoin(targetDir, filepath.Join(filepath.Dir(rel), hidden)) + if err != nil { + return err + } + if info, err := os.Lstat(target); err == nil && info.IsDir() && info.Mode()&os.ModeSymlink == 0 { + if err := makePathWritable(targetDir, target, originalModes); err != nil { + return err + } + } else if err != nil && !os.IsNotExist(err) { + return err + } + return removePath(target) + }) + if err != nil { + return fmt.Errorf("apply whiteouts: %w", err) + } + + // Phase 2: copy the layer's own entries, skipping whiteout markers. + // Directory metadata is deferred until all children are copied, so tar + // directory mtimes survive the merge. + pendingDirs := make([]dirMeta, 0) + hardlinks := make(map[hardlinkIdentity]string) + err = filepath.WalkDir(layerDir, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if path == layerDir { + return nil + } + if strings.HasPrefix(entry.Name(), whiteoutPrefix) { + if entry.IsDir() { + return fs.SkipDir + } + return nil + } + rel, err := filepath.Rel(layerDir, path) + if err != nil { + return err + } + target, err := safeJoin(targetDir, rel) + if err != nil { + return err + } + if err := makePathWritable(targetDir, filepath.Dir(target), originalModes); err != nil { + return err + } + if entry.IsDir() { + info, err := entry.Info() + if err != nil { + return err + } + pendingDirs = append(pendingDirs, dirMeta{src: path, dst: target, info: info}) + } + if err := copyEntryInto(path, target, hardlinks); err != nil { + return err + } + if entry.IsDir() { + return makePathWritable(targetDir, target, originalModes) + } + return nil + }) + if err != nil { + return fmt.Errorf("copy layer tree: %w", err) + } + for _, dir := range pendingDirs { + if err := copyEntryMetadata(dir.src, dir.dst, dir.info); err != nil { + return fmt.Errorf("restore dir metadata %s: %w", dir.dst, err) + } + delete(originalModes, dir.dst) + } + return nil +} + +func makePathWritable(root, path string, originalModes map[string]fs.FileMode) error { + root = filepath.Clean(root) + path = filepath.Clean(path) + if path != root && !strings.HasPrefix(path, root+string(filepath.Separator)) { + return fmt.Errorf("path is outside target root: %s", path) + } + for current := path; ; current = filepath.Dir(current) { + info, err := os.Lstat(current) + if err == nil { + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + return fmt.Errorf("target parent is not a directory: %s", current) + } + if _, recorded := originalModes[current]; !recorded { + originalModes[current] = info.Mode() + if err := os.Chmod(current, info.Mode().Perm()|0700); err != nil { + return err + } + } + } else if !os.IsNotExist(err) { + return err + } + if current == root { + return nil + } + } +} + +func restoreDirectoryModes(originalModes map[string]fs.FileMode) error { + var restoreErr error + for path, mode := range originalModes { + if _, err := os.Lstat(path); os.IsNotExist(err) { + continue + } else if err != nil { + restoreErr = errors.Join(restoreErr, err) + continue + } + if err := os.Chmod(path, mode.Perm()|mode&(os.ModeSetuid|os.ModeSetgid|os.ModeSticky)); err != nil { + restoreErr = errors.Join(restoreErr, err) + } + } + return restoreErr +} + +type dirMeta struct { + src, dst string + info os.FileInfo +} + +// clearDirContents removes everything inside dir without removing dir itself, +// and without following symlinks. +func clearDirContents(dir string) error { + info, err := os.Lstat(dir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + if !info.IsDir() { + return removePath(dir) + } + entries, err := os.ReadDir(dir) + if err != nil { + return err + } + for _, entry := range entries { + if err := removePath(filepath.Join(dir, entry.Name())); err != nil { + return err + } + } + return nil +} + +type hardlinkIdentity struct { + dev uint64 + ino uint64 +} + +// copyEntryInto copies one filesystem entry from src to dst, replacing any +// conflicting entry and preserving hardlinks within the layer. +func copyEntryInto(src, dst string, hardlinks map[hardlinkIdentity]string) error { + info, err := os.Lstat(src) + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { + return err + } + + switch info.Mode() & fs.ModeType { + case 0: + return copyRegularEntry(src, dst, info, hardlinks) + case fs.ModeDir: + return copyDirectoryEntry(src, dst, info) + case fs.ModeSymlink: + return copySymlinkEntry(src, dst) + default: + return copySpecialEntry(src, dst, info) + } +} + +func copyRegularEntry(src, dst string, info os.FileInfo, hardlinks map[hardlinkIdentity]string) error { + if err := removePath(dst); err != nil { + return err + } + if stat, ok := info.Sys().(*syscall.Stat_t); ok && stat.Nlink > 1 { + identity := hardlinkIdentity{dev: uint64(stat.Dev), ino: uint64(stat.Ino)} + if first, seen := hardlinks[identity]; seen { + return os.Link(first, dst) + } + hardlinks[identity] = dst + } + if err := copyFileContents(src, dst); err != nil { + return err + } + return copyEntryMetadata(src, dst, info) +} + +func copyDirectoryEntry(src, dst string, info os.FileInfo) error { + if existing, err := os.Lstat(dst); err == nil && !existing.IsDir() { + if err := removePath(dst); err != nil { + return err + } + } + return os.MkdirAll(dst, info.Mode().Perm()) +} + +func copySymlinkEntry(src, dst string) error { + linkTarget, err := os.Readlink(src) + if err != nil { + return err + } + if err := removePath(dst); err != nil { + return err + } + if err := os.Symlink(linkTarget, dst); err != nil { + return err + } + info, err := os.Lstat(src) + if err != nil { + return err + } + return copyEntryMetadata(src, dst, info) +} + +func copySpecialEntry(src, dst string, info os.FileInfo) error { + if err := removePath(dst); err != nil { + return err + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return fmt.Errorf("unsupported entry type for %s", src) + } + mode, err := specialFileMode(info.Mode() & fs.ModeType) + if err != nil { + return fmt.Errorf("unsupported entry type for %s: %w", src, err) + } + if err := unix.Mknod(dst, mode|uint32(info.Mode().Perm()), int(stat.Rdev)); err != nil { + if !errors.Is(err, unix.EPERM) { + return fmt.Errorf("mknod: %w", err) + } + if err := createRootlessDevicePlaceholder(dst); err != nil { + return err + } + } + return copyEntryMetadata(src, dst, info) +} + +func specialFileMode(mode fs.FileMode) (uint32, error) { + switch mode { + case fs.ModeCharDevice: + return syscall.S_IFCHR, nil + case fs.ModeDevice: + return syscall.S_IFBLK, nil + case fs.ModeNamedPipe: + return syscall.S_IFIFO, nil + case fs.ModeSocket: + return syscall.S_IFSOCK, nil + default: + return 0, fmt.Errorf("unsupported file mode") + } +} + +type entryMetadata struct { + uid, gid int + hasOwner bool + mode fs.FileMode + symlink bool + atime time.Time + mtime time.Time + xattrs map[string][]byte +} + +func applyEntryMetadata(path string, metadata entryMetadata) error { + if metadata.hasOwner { + if err := os.Lchown(path, metadata.uid, metadata.gid); err != nil && !errors.Is(err, os.ErrPermission) && !errors.Is(err, unix.EPERM) { + return err + } + } + if metadata.symlink { + return nil + } + mode := metadata.mode.Perm() | metadata.mode&(os.ModeSetuid|os.ModeSetgid|os.ModeSticky) + if err := os.Chmod(path, mode); err != nil { + return err + } + if err := os.Chtimes(path, metadata.atime, metadata.mtime); err != nil { + return err + } + return applyXattrs(path, metadata.xattrs) +} + +func copyEntryMetadata(src, dst string, info os.FileInfo) error { + metadata := entryMetadata{ + mode: info.Mode(), + symlink: info.Mode()&os.ModeSymlink != 0, + atime: info.ModTime(), + mtime: info.ModTime(), + } + if stat, ok := info.Sys().(*syscall.Stat_t); ok { + metadata.uid = int(stat.Uid) + metadata.gid = int(stat.Gid) + metadata.hasOwner = true + metadata.atime = time.Unix(stat.Atim.Sec, stat.Atim.Nsec) + } + if !metadata.symlink { + xattrs, err := readXattrs(src) + if err != nil { + return err + } + metadata.xattrs = xattrs + } + return applyEntryMetadata(dst, metadata) +} + +func readXattrs(src string) (map[string][]byte, error) { + size, err := unix.Llistxattr(src, nil) + if err != nil { + if errors.Is(err, unix.ENOTSUP) || errors.Is(err, unix.EPERM) { + return nil, nil + } + return nil, err + } + names := make([]byte, size) + if size > 0 { + n, err := unix.Llistxattr(src, names) + if err != nil { + return nil, err + } + names = names[:n] + } + xattrs := make(map[string][]byte) + for _, attr := range bytes.Split(bytes.TrimSuffix(names, []byte{0}), []byte{0}) { + if len(attr) == 0 { + continue + } + name := string(attr) + size, err := unix.Lgetxattr(src, name, nil) + if err != nil { + if errors.Is(err, unix.ENOTSUP) || errors.Is(err, unix.EPERM) || errors.Is(err, unix.ENODATA) { + continue + } + return nil, err + } + value := make([]byte, size) + if _, err := unix.Lgetxattr(src, name, value); err != nil { + return nil, err + } + xattrs[name] = value + } + return xattrs, nil +} + +func tarXattrs(header *tar.Header) map[string][]byte { + const paxXattrPrefix = "SCHILY.xattr." + xattrs := make(map[string][]byte, len(header.Xattrs)) + for name, value := range header.Xattrs { + xattrs[name] = []byte(value) + } + for key, value := range header.PAXRecords { + if strings.HasPrefix(key, paxXattrPrefix) { + name := strings.TrimPrefix(key, paxXattrPrefix) + if _, exists := xattrs[name]; !exists { + xattrs[name] = []byte(value) + } + } + } + return xattrs +} + +func applyXattrs(path string, xattrs map[string][]byte) error { + for name, value := range xattrs { + if err := unix.Lsetxattr(path, name, value, 0); err != nil && !errors.Is(err, unix.ENOTSUP) && !errors.Is(err, unix.EPERM) { + return fmt.Errorf("restore xattr %s: %w", name, err) + } + } + return nil +} + +func copyFileContents(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC|syscall.O_NOFOLLOW, 0644) + if err != nil { + return err + } + if _, err := io.Copy(out, in); err != nil { + out.Close() + return err + } + return out.Close() +} diff --git a/lib/images/manager.go b/lib/images/manager.go index a5ea4b732..12fe1c244 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -55,7 +55,7 @@ type Manager interface { // TotalImageBytes returns the total size of all ready images on disk. // Used by the resource manager for disk capacity tracking. TotalImageBytes(ctx context.Context) (int64, error) - // TotalOCICacheBytes returns the total size of the OCI layer cache. + // TotalOCICacheBytes returns the total size of the OCI and materialized layer caches. // Used by the resource manager for disk capacity tracking. TotalOCICacheBytes(ctx context.Context) (int64, error) // WaitForReady blocks until the image identified by name reaches a terminal diff --git a/lib/paths/paths.go b/lib/paths/paths.go index b47bdbbf1..b294f2417 100644 --- a/lib/paths/paths.go +++ b/lib/paths/paths.go @@ -193,10 +193,10 @@ func (p *Paths) ImageLayerArtifactForFormat(layerHex, format string) string { return filepath.Join(p.ImageLayerDir(layerHex), "layer."+format) } -// ImageLayerRecord returns the path to the artifact record describing one -// materialized layer (format, options, whiteout inventory). -func (p *Paths) ImageLayerRecord(layerHex string) string { - return filepath.Join(p.ImageLayerDir(layerHex), "artifact.json") +// ImageLayerRecordForFormat returns the path to the artifact record for one +// materialized layer and format. +func (p *Paths) ImageLayerRecordForFormat(layerHex, format string) string { + return filepath.Join(p.ImageLayerDir(layerHex), "artifact."+format+".json") } // ImageDigestDir returns the directory for a specific image digest. From a0ea61ae1aac7ff95983fe657cbdfd1a04431ac4 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Wed, 2 Sep 2026 17:09:47 +0000 Subject: [PATCH 05/41] Fix layer artifact composition and metadata handling edge cases --- lib/images/disk_usage.go | 15 ++++- lib/images/disk_usage_test.go | 5 ++ lib/images/layer_artifact.go | 24 ++++--- lib/images/layer_artifact_test.go | 103 ++++++++++++++++++++++++++++++ lib/images/layer_tree.go | 78 +++++++++++++++------- 5 files changed, 187 insertions(+), 38 deletions(-) diff --git a/lib/images/disk_usage.go b/lib/images/disk_usage.go index 8d098a46d..b9b91bf19 100644 --- a/lib/images/disk_usage.go +++ b/lib/images/disk_usage.go @@ -3,6 +3,7 @@ package images import ( "encoding/json" "fmt" + "io/fs" "os" "path/filepath" "strings" @@ -86,16 +87,26 @@ func totalReadyImageBytesFromMetadata(imagesDir string) (int64, error) { // totalLayerArtifactBytesFromFilesystem sums materialized layer artifacts. func totalLayerArtifactBytesFromFilesystem(layersDir string) (int64, error) { var total int64 - err := filepath.Walk(layersDir, func(path string, info os.FileInfo, err error) error { + err := filepath.WalkDir(layersDir, func(path string, d fs.DirEntry, err error) error { if err != nil { if os.IsNotExist(err) { return nil } return err } - if info.IsDir() || !strings.HasPrefix(info.Name(), "layer.") { + if d.IsDir() { + if strings.HasPrefix(d.Name(), ".") && path != layersDir { + return filepath.SkipDir + } + return nil + } + if !strings.HasPrefix(d.Name(), "layer.") { return nil } + info, err := d.Info() + if err != nil { + return err + } total += info.Size() return nil }) diff --git a/lib/images/disk_usage_test.go b/lib/images/disk_usage_test.go index 5b6a441ed..7de35871b 100644 --- a/lib/images/disk_usage_test.go +++ b/lib/images/disk_usage_test.go @@ -74,6 +74,11 @@ func TestTotalLayerArtifactBytesFromFilesystem(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(digestDir, "layer.ext4"), []byte("ext4"), 0o644)) require.NoError(t, os.WriteFile(filepath.Join(digestDir, "artifact.erofs.json"), []byte("record"), 0o644)) + // In-progress temp dirs should be skipped. + unpackDir := filepath.Join(digestDir, ".unpack-tmp") + require.NoError(t, os.MkdirAll(unpackDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(unpackDir, "layer.bin"), []byte("unpacked-content"), 0o644)) + total, err := totalLayerArtifactBytesFromFilesystem(layersDir) require.NoError(t, err) require.Equal(t, int64(len("erofs")+len("ext4")), total) diff --git a/lib/images/layer_artifact.go b/lib/images/layer_artifact.go index fa51003a2..572500c12 100644 --- a/lib/images/layer_artifact.go +++ b/lib/images/layer_artifact.go @@ -175,7 +175,7 @@ func (m *manager) materializeLayerArtifact(desc layerDescriptor) (*layerArtifact if err != nil { return nil, fmt.Errorf("create unpack directory: %w", err) } - defer os.RemoveAll(unpackDir) + defer removePath(unpackDir) stats, err := unpackLayerBlob(blobPath, desc.MediaType, unpackDir) if err != nil { @@ -310,7 +310,8 @@ func unpackLayerBlob(blobPath, mediaType, dest string) (*unpackStats, error) { if err := resolveHardlinks(dest, pendingHardlinks); err != nil { return nil, err } - for _, dir := range pendingDirs { + for i := len(pendingDirs) - 1; i >= 0; i-- { + dir := pendingDirs[i] if err := applyTarMetadata(dir.target, dir.header); err != nil { return nil, fmt.Errorf("restore dir metadata %s: %w", dir.target, err) } @@ -325,8 +326,9 @@ type pendingDir struct { } type pendingHardlink struct { - target string - linkname string + target string + linkname string + linkTarget string } func resolveHardlinks(root string, pending []pendingHardlink) error { @@ -337,6 +339,7 @@ func resolveHardlinks(root string, pending []pendingHardlink) error { if err != nil { return err } + link.linkTarget = linkTarget if _, err := os.Lstat(linkTarget); err == nil { ready = append(ready, link) } else if os.IsNotExist(err) { @@ -353,9 +356,7 @@ func resolveHardlinks(root string, pending []pendingHardlink) error { if err := prepareTarTarget(link.target); err != nil { return err } - if linkTarget, err := safeJoin(root, link.linkname); err != nil { - return err - } else if err := os.Link(linkTarget, link.target); err != nil { + if err := os.Link(link.linkTarget, link.target); err != nil { return fmt.Errorf("create hardlink %s -> %s: %w", link.target, link.linkname, err) } resolved++ @@ -411,6 +412,7 @@ func (c multiCloser) Close() error { // do not exist yet, so their targets are checked by // validateSymlinkTarget instead. func safeJoin(root, name string) (string, error) { + root = filepath.Clean(root) rootInfo, err := os.Lstat(root) if err != nil { return "", fmt.Errorf("inspect extraction root: %w", err) @@ -423,12 +425,11 @@ func safeJoin(root, name string) (string, error) { } clean := filepath.Clean(name) if clean == "." { - return filepath.Clean(root), nil + return root, nil } if clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { return "", fmt.Errorf("tar entry escapes root: %s", name) } - root = filepath.Clean(root) target := filepath.Join(root, clean) if target == root { return target, nil @@ -588,13 +589,10 @@ func applyTarMetadata(path string, header *tar.Header) error { uid: header.Uid, gid: header.Gid, hasOwner: true, - mode: os.FileMode(header.Mode), + mode: header.FileInfo().Mode(), symlink: header.Typeflag == tar.TypeSymlink, atime: atime, mtime: mtime, xattrs: tarXattrs(header), }) } - -// removePath removes whatever entry occupies path, including non-empty -// directories, and tolerates a missing path. diff --git a/lib/images/layer_artifact_test.go b/lib/images/layer_artifact_test.go index 924eaf091..cc798bfeb 100644 --- a/lib/images/layer_artifact_test.go +++ b/lib/images/layer_artifact_test.go @@ -501,3 +501,106 @@ func TestApplyLayerTreePreservesDirMtime(t *testing.T) { require.NoError(t, err) require.True(t, info.ModTime().Equal(old), "dir mtime must survive the merge") } + +func TestSpecialFileModeSupportsCharDevice(t *testing.T) { + mode, err := specialFileMode(fs.ModeCharDevice | fs.ModeDevice) + require.NoError(t, err) + require.Equal(t, uint32(syscall.S_IFCHR), mode) +} + +func TestApplyTarMetadataPreservesSpecialBits(t *testing.T) { + root := t.TempDir() + target := filepath.Join(root, "suid-bin") + require.NoError(t, os.WriteFile(target, []byte("bin"), 0755)) + + header := &tar.Header{ + Name: "suid-bin", + Typeflag: tar.TypeReg, + Mode: 04755, + } + require.NoError(t, applyTarMetadata(target, header)) + + info, err := os.Stat(target) + require.NoError(t, err) + require.Equal(t, os.ModeSetuid, info.Mode()&os.ModeSetuid, "setuid bit must be preserved") +} + +func TestApplyLayerTreeSanitizesWhiteouts(t *testing.T) { + root := t.TempDir() + targetDir := filepath.Join(root, "target") + layerDir := filepath.Join(root, "layer") + + require.NoError(t, os.MkdirAll(filepath.Join(targetDir, "parent", "child"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(targetDir, "parent", "victim.txt"), []byte("v"), 0644)) + require.NoError(t, os.MkdirAll(filepath.Join(layerDir, "parent"), 0755)) + // Malformed whiteouts that must be safely ignored: + require.NoError(t, os.WriteFile(filepath.Join(layerDir, "parent", ".wh."), nil, 0644)) + require.NoError(t, os.WriteFile(filepath.Join(layerDir, "parent", ".wh.."), nil, 0644)) + require.NoError(t, os.WriteFile(filepath.Join(layerDir, "parent", ".wh..."), nil, 0644)) + + require.NoError(t, applyLayerTree(layerDir, targetDir)) + + // Parent directory must not be deleted by malformed whiteout paths. + info, err := os.Stat(filepath.Join(targetDir, "parent")) + require.NoError(t, err) + require.True(t, info.IsDir()) + require.FileExists(t, filepath.Join(targetDir, "parent", "victim.txt")) +} + +func TestApplyLayerTreeRestoresRestrictiveNestedDirModes(t *testing.T) { + root := t.TempDir() + t.Cleanup(func() { _ = makeTreeWritable(root) }) + targetDir := filepath.Join(root, "target") + layerDir := filepath.Join(root, "layer") + + require.NoError(t, os.MkdirAll(filepath.Join(layerDir, "outer", "inner"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(layerDir, "outer", "inner", "file.txt"), []byte("data"), 0644)) + require.NoError(t, os.Chmod(filepath.Join(layerDir, "outer", "inner"), 0500)) + require.NoError(t, os.Chmod(filepath.Join(layerDir, "outer"), 0500)) + + require.NoError(t, applyLayerTree(layerDir, targetDir)) + + infoOuter, err := os.Stat(filepath.Join(targetDir, "outer")) + require.NoError(t, err) + require.Equal(t, os.FileMode(0500), infoOuter.Mode().Perm()) + + infoInner, err := os.Stat(filepath.Join(targetDir, "outer", "inner")) + require.NoError(t, err) + require.Equal(t, os.FileMode(0500), infoInner.Mode().Perm()) +} + +func TestMakePathWritableTopDown(t *testing.T) { + root := t.TempDir() + t.Cleanup(func() { _ = makeTreeWritable(root) }) + parent := filepath.Join(root, "restricted") + child := filepath.Join(parent, "child") + require.NoError(t, os.MkdirAll(child, 0755)) + require.NoError(t, os.Chmod(parent, 0400)) // read-only, no execute + + originalModes := make(map[string]fs.FileMode) + err := makePathWritable(root, child, originalModes) + require.NoError(t, err) + + info, err := os.Stat(parent) + require.NoError(t, err) + require.True(t, info.Mode().Perm()&0700 == 0700) + + require.NoError(t, restoreDirectoryModes(originalModes)) + infoAfter, err := os.Stat(parent) + require.NoError(t, err) + require.Equal(t, os.FileMode(0400), infoAfter.Mode().Perm()) +} + +func TestSafeJoinCleanRootSymlink(t *testing.T) { + root := t.TempDir() + realDir := filepath.Join(root, "real") + symDir := filepath.Join(root, "sym") + require.NoError(t, os.Mkdir(realDir, 0755)) + require.NoError(t, os.Symlink("real", symDir)) + + // Calling safeJoin with trailing slash on symlink root must be rejected as symlink. + _, err := safeJoin(symDir+"/", "file.txt") + require.Error(t, err) + require.Contains(t, err.Error(), "extraction root is not a directory") +} + diff --git a/lib/images/layer_tree.go b/lib/images/layer_tree.go index 5ff887896..fd120d86b 100644 --- a/lib/images/layer_tree.go +++ b/lib/images/layer_tree.go @@ -9,6 +9,7 @@ import ( "io/fs" "os" "path/filepath" + "sort" "strings" "syscall" "time" @@ -37,7 +38,7 @@ func makeTreeWritable(path string) error { if err := os.Chmod(path, info.Mode().Perm()|0700); err != nil { return err } - return filepath.WalkDir(path, func(path string, entry fs.DirEntry, err error) error { + return filepath.WalkDir(path, func(p string, entry fs.DirEntry, err error) error { if err != nil { return err } @@ -46,7 +47,7 @@ func makeTreeWritable(path string) error { if err != nil { return err } - return os.Chmod(path, info.Mode().Perm()|0700) + return os.Chmod(p, info.Mode().Perm()|0700) } return nil }) @@ -97,6 +98,9 @@ func applyLayerTree(layerDir, targetDir string) (err error) { return clearDirContents(targetParent) } hidden := strings.TrimPrefix(base, whiteoutPrefix) + if hidden == "" || hidden == "." || hidden == ".." { + return nil + } target, err := safeJoin(targetDir, filepath.Join(filepath.Dir(rel), hidden)) if err != nil { return err @@ -161,7 +165,8 @@ func applyLayerTree(layerDir, targetDir string) (err error) { if err != nil { return fmt.Errorf("copy layer tree: %w", err) } - for _, dir := range pendingDirs { + for i := len(pendingDirs) - 1; i >= 0; i-- { + dir := pendingDirs[i] if err := copyEntryMetadata(dir.src, dir.dst, dir.info); err != nil { return fmt.Errorf("restore dir metadata %s: %w", dir.dst, err) } @@ -176,30 +181,57 @@ func makePathWritable(root, path string, originalModes map[string]fs.FileMode) e if path != root && !strings.HasPrefix(path, root+string(filepath.Separator)) { return fmt.Errorf("path is outside target root: %s", path) } - for current := path; ; current = filepath.Dir(current) { - info, err := os.Lstat(current) - if err == nil { - if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { - return fmt.Errorf("target parent is not a directory: %s", current) - } - if _, recorded := originalModes[current]; !recorded { - originalModes[current] = info.Mode() - if err := os.Chmod(current, info.Mode().Perm()|0700); err != nil { - return err - } - } - } else if !os.IsNotExist(err) { + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + if err := makeDirWritable(root, originalModes); err != nil { + return err + } + if rel == "." { + return nil + } + current := root + for _, part := range strings.Split(rel, string(filepath.Separator)) { + current = filepath.Join(current, part) + if err := makeDirWritable(current, originalModes); err != nil { return err } - if current == root { + } + return nil +} + +func makeDirWritable(dir string, originalModes map[string]fs.FileMode) error { + info, err := os.Lstat(dir) + if err != nil { + if os.IsNotExist(err) { return nil } + return err + } + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + return fmt.Errorf("target parent is not a directory: %s", dir) } + if _, recorded := originalModes[dir]; !recorded { + originalModes[dir] = info.Mode() + if err := os.Chmod(dir, info.Mode().Perm()|0700); err != nil { + return err + } + } + return nil } func restoreDirectoryModes(originalModes map[string]fs.FileMode) error { + paths := make([]string, 0, len(originalModes)) + for path := range originalModes { + paths = append(paths, path) + } + sort.Slice(paths, func(i, j int) bool { + return len(paths[i]) > len(paths[j]) + }) var restoreErr error - for path, mode := range originalModes { + for _, path := range paths { + mode := originalModes[path] if _, err := os.Lstat(path); os.IsNotExist(err) { continue } else if err != nil { @@ -339,14 +371,14 @@ func copySpecialEntry(src, dst string, info os.FileInfo) error { } func specialFileMode(mode fs.FileMode) (uint32, error) { - switch mode { - case fs.ModeCharDevice: + switch { + case mode&fs.ModeCharDevice != 0: return syscall.S_IFCHR, nil - case fs.ModeDevice: + case mode&fs.ModeDevice != 0: return syscall.S_IFBLK, nil - case fs.ModeNamedPipe: + case mode&fs.ModeNamedPipe != 0: return syscall.S_IFIFO, nil - case fs.ModeSocket: + case mode&fs.ModeSocket != 0: return syscall.S_IFSOCK, nil default: return 0, fmt.Errorf("unsupported file mode") From be91308b9e9bbe9cf590db99b70e7074506da4a8 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Wed, 2 Sep 2026 19:43:00 +0000 Subject: [PATCH 06/41] Harden layer artifact composition --- lib/images/fs_entry.go | 33 +++++++++++++++++++++ lib/images/layer_artifact.go | 22 ++------------ lib/images/layer_artifact_test.go | 25 +++++++++++++++- lib/images/layer_tree.go | 48 ++++++++++++++++++++++++++----- lib/images/manager.go | 2 +- lib/resources/resource.go | 2 +- 6 files changed, 103 insertions(+), 29 deletions(-) create mode 100644 lib/images/fs_entry.go diff --git a/lib/images/fs_entry.go b/lib/images/fs_entry.go new file mode 100644 index 000000000..72443077f --- /dev/null +++ b/lib/images/fs_entry.go @@ -0,0 +1,33 @@ +package images + +import ( + "errors" + "fmt" + "os" + "syscall" + + "golang.org/x/sys/unix" +) + +func mknodWithRootlessFallback(path string, mode uint32, dev int) error { + if err := unix.Mknod(path, mode, dev); err != nil { + if !errors.Is(err, unix.EPERM) { + return fmt.Errorf("mknod: %w", err) + } + if err := createRootlessDevicePlaceholder(path); err != nil { + return err + } + } + return nil +} + +func createRootlessDevicePlaceholder(path string) error { + file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|syscall.O_NOFOLLOW, 0644) + if err != nil { + return fmt.Errorf("create rootless device placeholder: %w", err) + } + if err := file.Close(); err != nil { + return err + } + return nil +} diff --git a/lib/images/layer_artifact.go b/lib/images/layer_artifact.go index 572500c12..3c05383ad 100644 --- a/lib/images/layer_artifact.go +++ b/lib/images/layer_artifact.go @@ -192,7 +192,7 @@ func (m *manager) installLayerArtifact(desc layerDescriptor, layerHex, unpackDir record := &layerArtifact{ SchemaVersion: layerRecordSchemaVersion, Digest: desc.Digest, - DiffID: desc.DiffID, + DiffID: stats.diffID, Format: layerArtifactFormat(), UnpackedBytes: stats.unpackedBytes, Entries: stats.entries, @@ -544,13 +544,8 @@ func extractTarDevice(target string, header *tar.Header) error { mode = uint32(syscall.S_IFBLK) } dev := int(unix.Mkdev(uint32(header.Devmajor), uint32(header.Devminor))) - if err := unix.Mknod(target, mode|uint32(header.FileInfo().Mode().Perm()), dev); err != nil { - if !errors.Is(err, unix.EPERM) { - return fmt.Errorf("mknod: %w", err) - } - if err := createRootlessDevicePlaceholder(target); err != nil { - return err - } + if err := mknodWithRootlessFallback(target, mode|uint32(header.FileInfo().Mode().Perm()), dev); err != nil { + return err } return applyTarMetadata(target, header) } @@ -565,17 +560,6 @@ func extractTarFIFO(target string, header *tar.Header) error { return applyTarMetadata(target, header) } -func createRootlessDevicePlaceholder(path string) error { - file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|syscall.O_NOFOLLOW, 0644) - if err != nil { - return fmt.Errorf("create rootless device placeholder: %w", err) - } - if err := file.Close(); err != nil { - return err - } - return nil -} - func applyTarMetadata(path string, header *tar.Header) error { mtime := header.ModTime if mtime.IsZero() { diff --git a/lib/images/layer_artifact_test.go b/lib/images/layer_artifact_test.go index cc798bfeb..691124d21 100644 --- a/lib/images/layer_artifact_test.go +++ b/lib/images/layer_artifact_test.go @@ -272,6 +272,30 @@ func TestApplyLayerTreeWhiteoutSemantics(t *testing.T) { require.Empty(t, leaked) } +func TestApplyLayerTreeReplacesSymlinkBeforeNestedWhiteout(t *testing.T) { + root := t.TempDir() + targetDir := filepath.Join(root, "target") + layerDir := filepath.Join(root, "layer") + lowerDir := filepath.Join(root, "lower") + + require.NoError(t, os.MkdirAll(targetDir, 0755)) + require.NoError(t, os.MkdirAll(lowerDir, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(lowerDir, "old.txt"), []byte("old"), 0644)) + require.NoError(t, os.Symlink("../lower", filepath.Join(targetDir, "replaced"))) + require.NoError(t, os.MkdirAll(filepath.Join(layerDir, "replaced"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(layerDir, "replaced", ".wh.old.txt"), nil, 0644)) + require.NoError(t, os.WriteFile(filepath.Join(layerDir, "replaced", "new.txt"), []byte("new"), 0644)) + + require.NoError(t, applyLayerTree(layerDir, targetDir)) + + info, err := os.Lstat(filepath.Join(targetDir, "replaced")) + require.NoError(t, err) + require.True(t, info.IsDir()) + require.FileExists(t, filepath.Join(targetDir, "replaced", "new.txt")) + _, err = os.Lstat(filepath.Join(targetDir, "replaced", "old.txt")) + require.ErrorIs(t, err, os.ErrNotExist) +} + func TestApplyLayerTreePreservesUpperDirectoryMode(t *testing.T) { root := t.TempDir() targetDir := filepath.Join(root, "target") @@ -603,4 +627,3 @@ func TestSafeJoinCleanRootSymlink(t *testing.T) { require.Error(t, err) require.Contains(t, err.Error(), "extraction root is not a directory") } - diff --git a/lib/images/layer_tree.go b/lib/images/layer_tree.go index fd120d86b..8b941f601 100644 --- a/lib/images/layer_tree.go +++ b/lib/images/layer_tree.go @@ -74,6 +74,12 @@ func applyLayerTree(layerDir, targetDir string) (err error) { } }() + // Prepare directories before whiteouts so nested markers are applied after + // the layer has replaced any conflicting lower-layer entry. + if err := prepareLayerDirectories(layerDir, targetDir, originalModes); err != nil { + return fmt.Errorf("prepare layer directories: %w", err) + } + // Phase 1: apply whiteouts against what is already in the target. err = filepath.WalkDir(layerDir, func(path string, entry fs.DirEntry, err error) error { if err != nil { @@ -175,6 +181,39 @@ func applyLayerTree(layerDir, targetDir string) (err error) { return nil } +func prepareLayerDirectories(layerDir, targetDir string, originalModes map[string]fs.FileMode) error { + return filepath.WalkDir(layerDir, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if path == layerDir || !entry.IsDir() { + return nil + } + if strings.HasPrefix(entry.Name(), whiteoutPrefix) { + return filepath.SkipDir + } + rel, err := filepath.Rel(layerDir, path) + if err != nil { + return err + } + target, err := safeJoin(targetDir, rel) + if err != nil { + return err + } + if err := makePathWritable(targetDir, filepath.Dir(target), originalModes); err != nil { + return err + } + info, err := entry.Info() + if err != nil { + return err + } + if err := copyDirectoryEntry(path, target, info); err != nil { + return err + } + return makePathWritable(targetDir, target, originalModes) + }) +} + func makePathWritable(root, path string, originalModes map[string]fs.FileMode) error { root = filepath.Clean(root) path = filepath.Clean(path) @@ -359,13 +398,8 @@ func copySpecialEntry(src, dst string, info os.FileInfo) error { if err != nil { return fmt.Errorf("unsupported entry type for %s: %w", src, err) } - if err := unix.Mknod(dst, mode|uint32(info.Mode().Perm()), int(stat.Rdev)); err != nil { - if !errors.Is(err, unix.EPERM) { - return fmt.Errorf("mknod: %w", err) - } - if err := createRootlessDevicePlaceholder(dst); err != nil { - return err - } + if err := mknodWithRootlessFallback(dst, mode|uint32(info.Mode().Perm()), int(stat.Rdev)); err != nil { + return err } return copyEntryMetadata(src, dst, info) } diff --git a/lib/images/manager.go b/lib/images/manager.go index 12fe1c244..abd77a3fc 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -855,7 +855,7 @@ func (m *manager) TotalImageBytes(ctx context.Context) (int64, error) { return readyImageBytes, nil } -// TotalOCICacheBytes returns the total size of the OCI layer cache. +// TotalOCICacheBytes returns the total size of the OCI and materialized layer caches. func (m *manager) TotalOCICacheBytes(ctx context.Context) (int64, error) { _, ociCacheBytes, err := m.getDiskUsageTotals() if err != nil { diff --git a/lib/resources/resource.go b/lib/resources/resource.go index d674c5d01..5943cb0bc 100644 --- a/lib/resources/resource.go +++ b/lib/resources/resource.go @@ -132,7 +132,7 @@ type InstanceAllocation struct { type ImageLister interface { // TotalImageBytes returns the total size of all images on disk. TotalImageBytes(ctx context.Context) (int64, error) - // TotalOCICacheBytes returns the total size of the OCI layer cache. + // TotalOCICacheBytes returns the total size of the OCI and materialized layer caches. TotalOCICacheBytes(ctx context.Context) (int64, error) } From 122c28c183e5b0e367d69242c15677421989b517 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Thu, 3 Sep 2026 01:39:31 +0000 Subject: [PATCH 07/41] Harden layer tree portability and file copying --- lib/images/layer_artifact.go | 7 ++--- lib/images/layer_artifact_test.go | 41 +++++++++++++++++++++++++--- lib/images/layer_atime_darwin.go | 12 +++++++++ lib/images/layer_atime_linux.go | 12 +++++++++ lib/images/layer_tree.go | 44 +++++++++++++++++++++---------- 5 files changed, 96 insertions(+), 20 deletions(-) create mode 100644 lib/images/layer_atime_darwin.go create mode 100644 lib/images/layer_atime_linux.go diff --git a/lib/images/layer_artifact.go b/lib/images/layer_artifact.go index 3c05383ad..679c27b37 100644 --- a/lib/images/layer_artifact.go +++ b/lib/images/layer_artifact.go @@ -130,9 +130,10 @@ func discardLayerCache(p *paths.Paths, layerHex string) error { // materializeLayerArtifact ensures a layer has a materialized artifact keyed // by its blob digest, building it from the shared OCI cache blob when absent. // The layer is unpacked into an isolated temp directory, converted to the -// default image format, and installed atomically; an interrupted build leaves -// only temp files that the next attempt replaces. No production caller yet: -// pull integration and composition land in later changes. +// default image format, and installed atomically. Normal failures remove the +// temp directory; lifecycle reconciliation removes stale temp directories after +// an interrupted build. No production caller yet: pull integration and +// composition land in later changes. func (m *manager) materializeLayerArtifact(desc layerDescriptor) (*layerArtifact, error) { unlock := m.layerLocks.lock(desc.Digest) defer unlock() diff --git a/lib/images/layer_artifact_test.go b/lib/images/layer_artifact_test.go index 691124d21..986609ffa 100644 --- a/lib/images/layer_artifact_test.go +++ b/lib/images/layer_artifact_test.go @@ -200,6 +200,28 @@ func TestMaterializeLayerDoesNotPersistWhiteoutInventory(t *testing.T) { require.NotContains(t, string(data), "whiteouts") } +func TestApplyLayerTreeCopiesRestrictiveFiles(t *testing.T) { + root := t.TempDir() + targetDir := filepath.Join(root, "target") + layerDir := filepath.Join(root, "layer") + file := filepath.Join(layerDir, "secret.txt") + + require.NoError(t, os.MkdirAll(layerDir, 0755)) + require.NoError(t, os.WriteFile(file, []byte("secret"), 0644)) + require.NoError(t, os.Chmod(file, 0000)) + + require.NoError(t, applyLayerTree(layerDir, targetDir)) + + target := filepath.Join(targetDir, "secret.txt") + info, err := os.Stat(target) + require.NoError(t, err) + require.Equal(t, os.FileMode(0000), info.Mode().Perm()) + require.NoError(t, os.Chmod(target, 0644)) + data, err := os.ReadFile(target) + require.NoError(t, err) + require.Equal(t, "secret", string(data)) +} + func TestApplyLayerTreeWhiteoutSemantics(t *testing.T) { root := t.TempDir() targetDir := filepath.Join(root, "target") @@ -549,7 +571,7 @@ func TestApplyTarMetadataPreservesSpecialBits(t *testing.T) { require.Equal(t, os.ModeSetuid, info.Mode()&os.ModeSetuid, "setuid bit must be preserved") } -func TestApplyLayerTreeSanitizesWhiteouts(t *testing.T) { +func TestApplyLayerTreeRejectsMalformedWhiteouts(t *testing.T) { root := t.TempDir() targetDir := filepath.Join(root, "target") layerDir := filepath.Join(root, "layer") @@ -557,12 +579,13 @@ func TestApplyLayerTreeSanitizesWhiteouts(t *testing.T) { require.NoError(t, os.MkdirAll(filepath.Join(targetDir, "parent", "child"), 0755)) require.NoError(t, os.WriteFile(filepath.Join(targetDir, "parent", "victim.txt"), []byte("v"), 0644)) require.NoError(t, os.MkdirAll(filepath.Join(layerDir, "parent"), 0755)) - // Malformed whiteouts that must be safely ignored: + // Malformed whiteouts must be rejected without deleting lower-layer entries. require.NoError(t, os.WriteFile(filepath.Join(layerDir, "parent", ".wh."), nil, 0644)) require.NoError(t, os.WriteFile(filepath.Join(layerDir, "parent", ".wh.."), nil, 0644)) require.NoError(t, os.WriteFile(filepath.Join(layerDir, "parent", ".wh..."), nil, 0644)) - require.NoError(t, applyLayerTree(layerDir, targetDir)) + err := applyLayerTree(layerDir, targetDir) + require.ErrorContains(t, err, "invalid whiteout entry") // Parent directory must not be deleted by malformed whiteout paths. info, err := os.Stat(filepath.Join(targetDir, "parent")) @@ -571,6 +594,18 @@ func TestApplyLayerTreeSanitizesWhiteouts(t *testing.T) { require.FileExists(t, filepath.Join(targetDir, "parent", "victim.txt")) } +func TestApplyLayerTreeRejectsEscapingSymlink(t *testing.T) { + root := t.TempDir() + targetDir := filepath.Join(root, "target") + layerDir := filepath.Join(root, "layer") + require.NoError(t, os.MkdirAll(targetDir, 0755)) + require.NoError(t, os.MkdirAll(layerDir, 0755)) + require.NoError(t, os.Symlink("../../outside", filepath.Join(layerDir, "link"))) + + err := applyLayerTree(layerDir, targetDir) + require.ErrorContains(t, err, "symlink target escapes root") +} + func TestApplyLayerTreeRestoresRestrictiveNestedDirModes(t *testing.T) { root := t.TempDir() t.Cleanup(func() { _ = makeTreeWritable(root) }) diff --git a/lib/images/layer_atime_darwin.go b/lib/images/layer_atime_darwin.go new file mode 100644 index 000000000..670259c4c --- /dev/null +++ b/lib/images/layer_atime_darwin.go @@ -0,0 +1,12 @@ +//go:build darwin + +package images + +import ( + "syscall" + "time" +) + +func statAtime(stat *syscall.Stat_t) time.Time { + return time.Unix(stat.Atimespec.Sec, stat.Atimespec.Nsec) +} diff --git a/lib/images/layer_atime_linux.go b/lib/images/layer_atime_linux.go new file mode 100644 index 000000000..e2912da06 --- /dev/null +++ b/lib/images/layer_atime_linux.go @@ -0,0 +1,12 @@ +//go:build linux + +package images + +import ( + "syscall" + "time" +) + +func statAtime(stat *syscall.Stat_t) time.Time { + return time.Unix(stat.Atim.Sec, stat.Atim.Nsec) +} diff --git a/lib/images/layer_tree.go b/lib/images/layer_tree.go index 8b941f601..58ae47bba 100644 --- a/lib/images/layer_tree.go +++ b/lib/images/layer_tree.go @@ -105,13 +105,13 @@ func applyLayerTree(layerDir, targetDir string) (err error) { } hidden := strings.TrimPrefix(base, whiteoutPrefix) if hidden == "" || hidden == "." || hidden == ".." { - return nil + return fmt.Errorf("invalid whiteout entry: %s", rel) } target, err := safeJoin(targetDir, filepath.Join(filepath.Dir(rel), hidden)) if err != nil { return err } - if info, err := os.Lstat(target); err == nil && info.IsDir() && info.Mode()&os.ModeSymlink == 0 { + if info, err := os.Lstat(target); err == nil && info.IsDir() { if err := makePathWritable(targetDir, target, originalModes); err != nil { return err } @@ -160,7 +160,7 @@ func applyLayerTree(layerDir, targetDir string) (err error) { } pendingDirs = append(pendingDirs, dirMeta{src: path, dst: target, info: info}) } - if err := copyEntryInto(path, target, hardlinks); err != nil { + if err := copyEntryInto(path, target, hardlinks, targetDir); err != nil { return err } if entry.IsDir() { @@ -321,22 +321,18 @@ type hardlinkIdentity struct { // copyEntryInto copies one filesystem entry from src to dst, replacing any // conflicting entry and preserving hardlinks within the layer. -func copyEntryInto(src, dst string, hardlinks map[hardlinkIdentity]string) error { +func copyEntryInto(src, dst string, hardlinks map[hardlinkIdentity]string, root string) error { info, err := os.Lstat(src) if err != nil { return err } - if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { - return err - } - switch info.Mode() & fs.ModeType { case 0: return copyRegularEntry(src, dst, info, hardlinks) case fs.ModeDir: return copyDirectoryEntry(src, dst, info) case fs.ModeSymlink: - return copySymlinkEntry(src, dst) + return copySymlinkEntry(src, dst, root) default: return copySpecialEntry(src, dst, info) } @@ -368,11 +364,14 @@ func copyDirectoryEntry(src, dst string, info os.FileInfo) error { return os.MkdirAll(dst, info.Mode().Perm()) } -func copySymlinkEntry(src, dst string) error { +func copySymlinkEntry(src, dst, root string) error { linkTarget, err := os.Readlink(src) if err != nil { return err } + if err := validateSymlinkTarget(root, dst, linkTarget); err != nil { + return err + } if err := removePath(dst); err != nil { return err } @@ -459,7 +458,7 @@ func copyEntryMetadata(src, dst string, info os.FileInfo) error { metadata.uid = int(stat.Uid) metadata.gid = int(stat.Gid) metadata.hasOwner = true - metadata.atime = time.Unix(stat.Atim.Sec, stat.Atim.Nsec) + metadata.atime = statAtime(stat) } if !metadata.symlink { xattrs, err := readXattrs(src) @@ -501,10 +500,11 @@ func readXattrs(src string) (map[string][]byte, error) { return nil, err } value := make([]byte, size) - if _, err := unix.Lgetxattr(src, name, value); err != nil { + n, err := unix.Lgetxattr(src, name, value) + if err != nil { return nil, err } - xattrs[name] = value + xattrs[name] = value[:n] } return xattrs, nil } @@ -535,7 +535,23 @@ func applyXattrs(path string, xattrs map[string][]byte) error { return nil } -func copyFileContents(src, dst string) error { +func copyFileContents(src, dst string) (err error) { + info, err := os.Lstat(src) + if err != nil { + return err + } + sourceMode := info.Mode().Perm() | info.Mode()&(os.ModeSetuid|os.ModeSetgid|os.ModeSticky) + if sourceMode&0400 == 0 { + if err := os.Chmod(src, sourceMode|0400); err != nil { + return err + } + defer func() { + if restoreErr := os.Chmod(src, sourceMode); err == nil { + err = restoreErr + } + }() + } + in, err := os.Open(src) if err != nil { return err From 841674677a2e0349e825af572ecf27343cdd5222 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Thu, 3 Sep 2026 13:50:20 +0000 Subject: [PATCH 08/41] Harden layer materialization and metadata restoration --- lib/images/context_reader.go | 18 ++++++++++++ lib/images/disk.go | 7 ++++- lib/images/layer_artifact.go | 49 +++++++++++++++++++++++++++---- lib/images/layer_artifact_test.go | 34 +++++++++++++++++++++ lib/images/layer_tree.go | 11 +++---- lib/images/manager.go | 3 +- 6 files changed, 109 insertions(+), 13 deletions(-) create mode 100644 lib/images/context_reader.go diff --git a/lib/images/context_reader.go b/lib/images/context_reader.go new file mode 100644 index 000000000..8aec8b2e4 --- /dev/null +++ b/lib/images/context_reader.go @@ -0,0 +1,18 @@ +package images + +import ( + "context" + "io" +) + +type contextReader struct { + ctx context.Context + reader io.Reader +} + +func (r contextReader) Read(p []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.reader.Read(p) +} diff --git a/lib/images/disk.go b/lib/images/disk.go index 896d10f2a..97e77db53 100644 --- a/lib/images/disk.go +++ b/lib/images/disk.go @@ -1,6 +1,7 @@ package images import ( + "context" "fmt" "os" "os/exec" @@ -194,6 +195,10 @@ func convertToExt4(rootfsDir, diskPath string) (int64, error) { // convertToErofs converts a rootfs directory to an erofs disk image using mkfs.erofs func convertToErofs(rootfsDir, diskPath string) (int64, error) { + return convertToErofsContext(context.Background(), rootfsDir, diskPath) +} + +func convertToErofsContext(ctx context.Context, rootfsDir, diskPath string) (int64, error) { // Ensure parent directory exists if err := os.MkdirAll(filepath.Dir(diskPath), 0755); err != nil { return 0, fmt.Errorf("create disk parent dir: %w", err) @@ -202,7 +207,7 @@ func convertToErofs(rootfsDir, diskPath string) (int64, error) { // Create erofs image with LZ4 fast compression // -zlz4: LZ4 fast compression (~20-25% space savings, faster builds) // erofs doesn't need pre-allocation, creates file directly - cmd := exec.Command("mkfs.erofs", "-zlz4", diskPath, rootfsDir) + cmd := exec.CommandContext(ctx, "mkfs.erofs", "-zlz4", diskPath, rootfsDir) output, err := cmd.CombinedOutput() if err != nil { return 0, fmt.Errorf("mkfs.erofs failed: %w, output: %s", err, output) diff --git a/lib/images/layer_artifact.go b/lib/images/layer_artifact.go index 679c27b37..20187e29f 100644 --- a/lib/images/layer_artifact.go +++ b/lib/images/layer_artifact.go @@ -3,6 +3,7 @@ package images import ( "archive/tar" "compress/gzip" + "context" "crypto/sha256" "encoding/json" "errors" @@ -135,9 +136,26 @@ func discardLayerCache(p *paths.Paths, layerHex string) error { // an interrupted build. No production caller yet: pull integration and // composition land in later changes. func (m *manager) materializeLayerArtifact(desc layerDescriptor) (*layerArtifact, error) { - unlock := m.layerLocks.lock(desc.Digest) - defer unlock() + return m.materializeLayerArtifactContext(context.Background(), desc) +} +func (m *manager) materializeLayerArtifactContext(ctx context.Context, desc layerDescriptor) (*layerArtifact, error) { + key := desc.Digest + "\x00" + layerArtifactFormat() + result := m.layerFlights.DoChan(key, func() (any, error) { + return m.materializeLayerArtifactOnce(ctx, desc) + }) + select { + case <-ctx.Done(): + return nil, ctx.Err() + case shared := <-result: + if shared.Err != nil { + return nil, shared.Err + } + return shared.Val.(*layerArtifact), nil + } +} + +func (m *manager) materializeLayerArtifactOnce(ctx context.Context, desc layerDescriptor) (*layerArtifact, error) { if layerArtifactFormat() == "" { return nil, fmt.Errorf("unsupported layer artifact format: %s", DefaultImageFormat) } @@ -178,7 +196,7 @@ func (m *manager) materializeLayerArtifact(desc layerDescriptor) (*layerArtifact } defer removePath(unpackDir) - stats, err := unpackLayerBlob(blobPath, desc.MediaType, unpackDir) + stats, err := unpackLayerBlobContext(ctx, blobPath, desc.MediaType, unpackDir) if err != nil { return nil, fmt.Errorf("unpack layer %s: %w", desc.Digest, err) } @@ -186,10 +204,14 @@ func (m *manager) materializeLayerArtifact(desc layerDescriptor) (*layerArtifact return nil, fmt.Errorf("layer %s diff id mismatch: got %s, want %s", desc.Digest, stats.diffID, desc.DiffID) } - return m.installLayerArtifact(desc, layerHex, unpackDir, stats) + return m.installLayerArtifactContext(ctx, desc, layerHex, unpackDir, stats) } func (m *manager) installLayerArtifact(desc layerDescriptor, layerHex, unpackDir string, stats *unpackStats) (*layerArtifact, error) { + return m.installLayerArtifactContext(context.Background(), desc, layerHex, unpackDir, stats) +} + +func (m *manager) installLayerArtifactContext(ctx context.Context, desc layerDescriptor, layerHex, unpackDir string, stats *unpackStats) (*layerArtifact, error) { record := &layerArtifact{ SchemaVersion: layerRecordSchemaVersion, Digest: desc.Digest, @@ -203,7 +225,13 @@ func (m *manager) installLayerArtifact(desc layerDescriptor, layerHex, unpackDir if err := installAtomically(layerArtifactPath(m.paths, layerHex), func(path string) error { // The artifact intentionally retains .wh. marker files so // composition can re-derive whiteouts from the tree itself. - size, convErr := ExportRootfs(unpackDir, path, DefaultImageFormat) + var size int64 + var convErr error + if DefaultImageFormat == FormatErofs { + size, convErr = convertToErofsContext(ctx, unpackDir, path) + } else { + size, convErr = ExportRootfs(unpackDir, path, DefaultImageFormat) + } if convErr != nil { return convErr } @@ -235,6 +263,10 @@ type unpackStats struct { // umoci consumes whiteouts while this store must retain them for composition. // Paths are confined to dest. func unpackLayerBlob(blobPath, mediaType, dest string) (*unpackStats, error) { + return unpackLayerBlobContext(context.Background(), blobPath, mediaType, dest) +} + +func unpackLayerBlobContext(ctx context.Context, blobPath, mediaType, dest string) (*unpackStats, error) { if err := os.MkdirAll(dest, 0755); err != nil { return nil, fmt.Errorf("create extraction root: %w", err) } @@ -249,6 +281,7 @@ func unpackLayerBlob(blobPath, mediaType, dest string) (*unpackStats, error) { return nil, err } defer closer.Close() + reader = contextReader{ctx: ctx, reader: reader} hash := sha256.New() stats := &unpackStats{} @@ -336,7 +369,11 @@ func resolveHardlinks(root string, pending []pendingHardlink) error { waiting := make(map[string][]pendingHardlink) ready := make([]pendingHardlink, 0, len(pending)) for _, link := range pending { - linkTarget, err := safeJoin(root, link.linkname) + linkname := filepath.Clean(link.linkname) + if filepath.IsAbs(linkname) { + linkname = strings.TrimPrefix(linkname, string(filepath.Separator)) + } + linkTarget, err := safeJoin(root, linkname) if err != nil { return err } diff --git a/lib/images/layer_artifact_test.go b/lib/images/layer_artifact_test.go index 986609ffa..0654e8699 100644 --- a/lib/images/layer_artifact_test.go +++ b/lib/images/layer_artifact_test.go @@ -4,6 +4,7 @@ import ( "archive/tar" "bytes" "compress/gzip" + "context" "crypto/sha256" "encoding/json" "errors" @@ -222,6 +223,22 @@ func TestApplyLayerTreeCopiesRestrictiveFiles(t *testing.T) { require.Equal(t, "secret", string(data)) } +func TestRestoreDirectoryModesSkipsReplacedSymlink(t *testing.T) { + root := t.TempDir() + dir := filepath.Join(root, "dir") + target := filepath.Join(root, "target") + require.NoError(t, os.Mkdir(dir, 0750)) + require.NoError(t, os.WriteFile(target, []byte("target"), 0644)) + require.NoError(t, os.Remove(dir)) + require.NoError(t, os.Symlink("target", dir)) + + require.NoError(t, restoreDirectoryModes(map[string]fs.FileMode{dir: 0750})) + + info, err := os.Stat(target) + require.NoError(t, err) + require.Equal(t, os.FileMode(0644), info.Mode().Perm()) +} + func TestApplyLayerTreeWhiteoutSemantics(t *testing.T) { root := t.TempDir() targetDir := filepath.Join(root, "target") @@ -350,6 +367,23 @@ func TestApplyLayerTreeWhiteoutRemovesReadOnlyDirectory(t *testing.T) { require.ErrorIs(t, err, os.ErrNotExist) } +func TestUnpackLayerBlobContextHonorsCancellation(t *testing.T) { + root := t.TempDir() + blobPath := filepath.Join(root, "layer.tar") + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + require.NoError(t, tw.WriteHeader(&tar.Header{Name: "file", Typeflag: tar.TypeReg, Mode: 0644, Size: 1})) + _, err := tw.Write([]byte("x")) + require.NoError(t, err) + require.NoError(t, tw.Close()) + require.NoError(t, os.WriteFile(blobPath, buf.Bytes(), 0644)) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err = unpackLayerBlobContext(ctx, blobPath, "application/vnd.oci.image.layer.v1.tar", filepath.Join(root, "dest")) + require.ErrorIs(t, err, context.Canceled) +} + func TestUnpackLayerBlobIncludesTrailingTarPaddingInDiffID(t *testing.T) { root := t.TempDir() blobPath := filepath.Join(root, "layer.tar.gz") diff --git a/lib/images/layer_tree.go b/lib/images/layer_tree.go index 58ae47bba..c48b1a2a1 100644 --- a/lib/images/layer_tree.go +++ b/lib/images/layer_tree.go @@ -271,7 +271,8 @@ func restoreDirectoryModes(originalModes map[string]fs.FileMode) error { var restoreErr error for _, path := range paths { mode := originalModes[path] - if _, err := os.Lstat(path); os.IsNotExist(err) { + info, err := os.Lstat(path) + if os.IsNotExist(err) || (err == nil && (info.Mode()&os.ModeSymlink != 0 || !info.IsDir())) { continue } else if err != nil { restoreErr = errors.Join(restoreErr, err) @@ -437,14 +438,14 @@ func applyEntryMetadata(path string, metadata entryMetadata) error { if metadata.symlink { return nil } - mode := metadata.mode.Perm() | metadata.mode&(os.ModeSetuid|os.ModeSetgid|os.ModeSticky) - if err := os.Chmod(path, mode); err != nil { + if err := applyXattrs(path, metadata.xattrs); err != nil { return err } - if err := os.Chtimes(path, metadata.atime, metadata.mtime); err != nil { + mode := metadata.mode.Perm() | metadata.mode&(os.ModeSetuid|os.ModeSetgid|os.ModeSticky) + if err := os.Chmod(path, mode); err != nil { return err } - return applyXattrs(path, metadata.xattrs) + return os.Chtimes(path, metadata.atime, metadata.mtime) } func copyEntryMetadata(src, dst string, info os.FileInfo) error { diff --git a/lib/images/manager.go b/lib/images/manager.go index abd77a3fc..bceeb0f05 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -19,6 +19,7 @@ import ( "github.com/kernel/hypeman/lib/queue" "github.com/kernel/hypeman/lib/tags" "go.opentelemetry.io/otel/metric" + "golang.org/x/sync/singleflight" ) var errStaleBuild = errors.New("stale image build") @@ -75,7 +76,7 @@ type manager struct { ociClient *ociClient queue *queue.Queue createMu sync.Mutex - layerLocks keyedMutex + layerFlights singleflight.Group diskUsageMu sync.RWMutex tagGenerations map[string]uint64 requestedTags map[string]string // newest pull's digest per requested tag From c2e927e3fa6bd5fa9229aff0d53fdae2da2dc9b0 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Thu, 3 Sep 2026 13:51:13 +0000 Subject: [PATCH 09/41] Share context-aware cached layer unpacking --- lib/images/layer_artifact.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/lib/images/layer_artifact.go b/lib/images/layer_artifact.go index 20187e29f..08fb080bf 100644 --- a/lib/images/layer_artifact.go +++ b/lib/images/layer_artifact.go @@ -262,6 +262,32 @@ type unpackStats struct { // whiteout marker files. It intentionally does not use umoci's layer unpacker: // umoci consumes whiteouts while this store must retain them for composition. // Paths are confined to dest. +func unpackCachedLayer(cacheDir string, desc layerDescriptor, dest string) (*unpackStats, error) { + return unpackCachedLayerContext(context.Background(), cacheDir, desc, dest) +} + +func unpackCachedLayerContext(ctx context.Context, cacheDir string, desc layerDescriptor, dest string) (*unpackStats, error) { + layerHex := strings.TrimPrefix(desc.Digest, "sha256:") + if err := paths.ValidatePathComponent(layerHex); err != nil { + return nil, fmt.Errorf("invalid layer digest: %s", desc.Digest) + } + blobPath := filepath.Join(cacheDir, "blobs", "sha256", layerHex) + if _, err := os.Stat(blobPath); err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("layer blob missing from oci cache: %s", desc.Digest) + } + return nil, fmt.Errorf("stat layer blob: %w", err) + } + stats, err := unpackLayerBlobContext(ctx, blobPath, desc.MediaType, dest) + if err != nil { + return nil, fmt.Errorf("unpack layer %s: %w", desc.Digest, err) + } + if desc.DiffID != "" && stats.diffID != desc.DiffID { + return nil, fmt.Errorf("layer %s diff id mismatch: got %s, want %s", desc.Digest, stats.diffID, desc.DiffID) + } + return stats, nil +} + func unpackLayerBlob(blobPath, mediaType, dest string) (*unpackStats, error) { return unpackLayerBlobContext(context.Background(), blobPath, mediaType, dest) } From 889290092b9a32b76e03966b8bb521c4ec1c4e65 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Thu, 3 Sep 2026 13:52:37 +0000 Subject: [PATCH 10/41] Serialize layer store writes --- lib/images/keyed_mutex.go | 39 ------------------------------------ lib/images/layer_artifact.go | 3 +++ lib/images/manager.go | 1 + 3 files changed, 4 insertions(+), 39 deletions(-) delete mode 100644 lib/images/keyed_mutex.go diff --git a/lib/images/keyed_mutex.go b/lib/images/keyed_mutex.go deleted file mode 100644 index 609fd3017..000000000 --- a/lib/images/keyed_mutex.go +++ /dev/null @@ -1,39 +0,0 @@ -package images - -import "sync" - -type keyedMutex struct { - mu sync.Mutex - locks map[string]*keyedLock -} - -type keyedLock struct { - mu sync.Mutex - refs int -} - -func (m *keyedMutex) lock(key string) func() { - m.mu.Lock() - if m.locks == nil { - m.locks = make(map[string]*keyedLock) - } - lock := m.locks[key] - if lock == nil { - lock = &keyedLock{} - m.locks[key] = lock - } - lock.refs++ - m.mu.Unlock() - - lock.mu.Lock() - return func() { - lock.mu.Unlock() - - m.mu.Lock() - lock.refs-- - if lock.refs == 0 { - delete(m.locks, key) - } - m.mu.Unlock() - } -} diff --git a/lib/images/layer_artifact.go b/lib/images/layer_artifact.go index 08fb080bf..32a11cdbd 100644 --- a/lib/images/layer_artifact.go +++ b/lib/images/layer_artifact.go @@ -156,6 +156,9 @@ func (m *manager) materializeLayerArtifactContext(ctx context.Context, desc laye } func (m *manager) materializeLayerArtifactOnce(ctx context.Context, desc layerDescriptor) (*layerArtifact, error) { + m.layerStoreMu.Lock() + defer m.layerStoreMu.Unlock() + if layerArtifactFormat() == "" { return nil, fmt.Errorf("unsupported layer artifact format: %s", DefaultImageFormat) } diff --git a/lib/images/manager.go b/lib/images/manager.go index bceeb0f05..f18fe8a8d 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -77,6 +77,7 @@ type manager struct { queue *queue.Queue createMu sync.Mutex layerFlights singleflight.Group + layerStoreMu sync.Mutex diskUsageMu sync.RWMutex tagGenerations map[string]uint64 requestedTags map[string]string // newest pull's digest per requested tag From 2b7928e792d5cc6943f434cad324d48a6d909dc7 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Thu, 3 Sep 2026 14:01:13 +0000 Subject: [PATCH 11/41] Preserve explicit layer directory metadata --- lib/images/layer_artifact.go | 119 +++++++++++++++++++++++++----- lib/images/layer_artifact_test.go | 19 +++++ lib/images/layer_tree.go | 62 ++++++++++++---- 3 files changed, 167 insertions(+), 33 deletions(-) diff --git a/lib/images/layer_artifact.go b/lib/images/layer_artifact.go index 32a11cdbd..8733557f1 100644 --- a/lib/images/layer_artifact.go +++ b/lib/images/layer_artifact.go @@ -259,6 +259,7 @@ type unpackStats struct { entries int unpackedBytes int64 diffID string + explicitDirs map[string]struct{} } // unpackLayerBlob extracts one compressed layer blob into dest while preserving @@ -313,7 +314,7 @@ func unpackLayerBlobContext(ctx context.Context, blobPath, mediaType, dest strin reader = contextReader{ctx: ctx, reader: reader} hash := sha256.New() - stats := &unpackStats{} + stats := &unpackStats{explicitDirs: make(map[string]struct{})} // Directory metadata is re-applied after extraction, once children // exist, so tar directory mtimes are not overwritten by later writes. pendingDirs := make([]pendingDir, 0) @@ -351,6 +352,7 @@ func unpackLayerBlobContext(ctx context.Context, blobPath, mediaType, dest strin } if header.Typeflag == tar.TypeDir { + stats.explicitDirs[filepath.Clean(header.Name)] = struct{}{} pendingDirs = append(pendingDirs, pendingDir{target: target, header: header}) } if header.Typeflag == tar.TypeLink { @@ -472,13 +474,38 @@ func (c multiCloser) Close() error { return firstErr } -// safeJoin resolves a tar entry name inside root and rejects symlinked -// parents: extraction must never create an entry through a symlink an earlier -// tar entry planted, so existing parents are Lstat-walked and rejected rather -// than resolved. Symlink entries themselves may legitimately name paths that -// do not exist yet, so their targets are checked by -// validateSymlinkTarget instead. +// safeJoin resolves a tar entry inside root and rejects symlinked parents +// while extracting a layer. This prevents one tar entry from changing where +// a later entry is written. func safeJoin(root, name string) (string, error) { + resolved, err := safeJoinForComposition(root, name) + if err != nil { + return "", err + } + root = filepath.Clean(root) + clean := filepath.Clean(name) + target := filepath.Join(root, clean) + for parent := filepath.Dir(target); parent != root; parent = filepath.Dir(parent) { + info, err := os.Lstat(parent) + if err != nil { + if os.IsNotExist(err) { + continue + } + return "", fmt.Errorf("inspect tar entry parent: %w", err) + } + if info.Mode()&os.ModeSymlink != 0 { + return "", fmt.Errorf("tar entry traverses symlink: %s", name) + } + if !info.IsDir() { + return "", fmt.Errorf("tar entry parent is not a directory: %s", parent) + } + } + return resolved, nil +} + +// safeJoinForComposition resolves an entry through existing parent symlinks, +// interpreting absolute link targets relative to the image root. +func safeJoinForComposition(root, name string) (string, error) { root = filepath.Clean(root) rootInfo, err := os.Lstat(root) if err != nil { @@ -498,28 +525,84 @@ func safeJoin(root, name string) (string, error) { return "", fmt.Errorf("tar entry escapes root: %s", name) } target := filepath.Join(root, clean) - if target == root { - return target, nil - } if !strings.HasPrefix(target, root+string(filepath.Separator)) { return "", fmt.Errorf("tar entry escapes root: %s", name) } - for parent := filepath.Dir(target); parent != root; parent = filepath.Dir(parent) { - info, err := os.Lstat(parent) + parent := filepath.Dir(target) + resolvedParent, err := resolveLayerPath(root, parent, 0) + if err != nil { + return "", fmt.Errorf("inspect tar entry parent: %w", err) + } + return filepath.Join(resolvedParent, filepath.Base(target)), nil +} + +func resolveLayerPath(root, path string, depth int) (string, error) { + if depth > 40 { + return "", fmt.Errorf("too many symlinks") + } + root = filepath.Clean(root) + path = filepath.Clean(path) + rel, err := filepath.Rel(root, path) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("path escapes root") + } + if rel == "." { + return root, nil + } + parts := strings.Split(rel, string(filepath.Separator)) + current := root + for i, part := range parts { + candidate := filepath.Join(current, part) + info, err := os.Lstat(candidate) if err != nil { if os.IsNotExist(err) { - continue + return filepath.Join(candidate, filepath.Join(parts[i+1:]...)), nil } - return "", fmt.Errorf("inspect tar entry parent: %w", err) + return "", err } if info.Mode()&os.ModeSymlink != 0 { - return "", fmt.Errorf("tar entry traverses symlink: %s", name) + link, err := os.Readlink(candidate) + if err != nil { + return "", err + } + var linkTarget string + if filepath.IsAbs(link) { + linkTarget = filepath.Join(root, strings.TrimPrefix(link, string(filepath.Separator))) + } else { + linkTarget = filepath.Join(filepath.Dir(candidate), link) + } + linkTarget = filepath.Clean(linkTarget) + if !pathWithinRoot(root, linkTarget) { + return "", fmt.Errorf("symlink target escapes root") + } + return resolveLayerPath(root, filepath.Join(linkTarget, filepath.Join(parts[i+1:]...)), depth+1) } - if !info.IsDir() { - return "", fmt.Errorf("tar entry parent is not a directory: %s", parent) + if i < len(parts)-1 && !info.IsDir() { + return "", fmt.Errorf("path parent is not a directory: %s", candidate) + } + current = candidate + } + return current, nil +} + +func pathWithinRoot(root, path string) bool { + rel, err := filepath.Rel(filepath.Clean(root), filepath.Clean(path)) + return err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) +} + +func resolveCompositionDirTarget(root, target string) (string, error) { + resolved, err := resolveLayerPath(root, target, 0) + if err == nil { + return resolved, nil + } + info, statErr := os.Lstat(target) + if statErr == nil && info.Mode()&os.ModeSymlink != 0 { + if removeErr := removePath(target); removeErr != nil { + return "", removeErr } + return target, nil } - return target, nil + return "", err } // validateSymlinkTarget permits absolute targets because OCI images may use diff --git a/lib/images/layer_artifact_test.go b/lib/images/layer_artifact_test.go index 0654e8699..78a7bbcd7 100644 --- a/lib/images/layer_artifact_test.go +++ b/lib/images/layer_artifact_test.go @@ -452,6 +452,25 @@ func TestValidateSymlinkTargetAllowsAbsoluteTargets(t *testing.T) { require.NoError(t, validateSymlinkTarget(root, target, "/etc/resolv.conf")) } +func TestApplyLayerTreeResolvesInRootSymlinkParents(t *testing.T) { + root := t.TempDir() + targetDir := filepath.Join(root, "target") + layerDir := filepath.Join(root, "layer") + require.NoError(t, os.MkdirAll(filepath.Join(targetDir, "usr", "bin"), 0755)) + require.NoError(t, os.Symlink("usr/bin", filepath.Join(targetDir, "bin"))) + require.NoError(t, os.MkdirAll(filepath.Join(layerDir, "bin"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(layerDir, "bin", "tool"), []byte("tool"), 0644)) + + require.NoError(t, applyLayerTree(layerDir, targetDir)) + + data, err := os.ReadFile(filepath.Join(targetDir, "usr", "bin", "tool")) + require.NoError(t, err) + require.Equal(t, "tool", string(data)) + info, err := os.Lstat(filepath.Join(targetDir, "bin")) + require.NoError(t, err) + require.Equal(t, os.ModeSymlink, info.Mode()&os.ModeSymlink) +} + func TestApplyLayerTreeSymlinksAndHardlinks(t *testing.T) { root := t.TempDir() targetDir := filepath.Join(root, "target") diff --git a/lib/images/layer_tree.go b/lib/images/layer_tree.go index c48b1a2a1..03358d11e 100644 --- a/lib/images/layer_tree.go +++ b/lib/images/layer_tree.go @@ -59,7 +59,11 @@ func makeTreeWritable(path string) error { // whiteout files are interpreted here rather than passed through, because // overlayfs does not understand them. No production caller yet; composition // lands in a later change. -func applyLayerTree(layerDir, targetDir string) (err error) { +func applyLayerTree(layerDir, targetDir string) error { + return applyLayerTreeWithExplicitDirs(layerDir, targetDir, nil) +} + +func applyLayerTreeWithExplicitDirs(layerDir, targetDir string, explicitDirs map[string]struct{}) (err error) { if err := os.MkdirAll(targetDir, 0755); err != nil { return fmt.Errorf("create target directory: %w", err) } @@ -93,7 +97,7 @@ func applyLayerTree(layerDir, targetDir string) (err error) { if err != nil { return err } - targetParent, err := safeJoin(targetDir, filepath.Dir(rel)) + targetParent, err := safeJoinForComposition(targetDir, filepath.Dir(rel)) if err != nil { return err } @@ -107,7 +111,7 @@ func applyLayerTree(layerDir, targetDir string) (err error) { if hidden == "" || hidden == "." || hidden == ".." { return fmt.Errorf("invalid whiteout entry: %s", rel) } - target, err := safeJoin(targetDir, filepath.Join(filepath.Dir(rel), hidden)) + target, err := safeJoinForComposition(targetDir, filepath.Join(filepath.Dir(rel), hidden)) if err != nil { return err } @@ -146,19 +150,33 @@ func applyLayerTree(layerDir, targetDir string) (err error) { if err != nil { return err } - target, err := safeJoin(targetDir, rel) + target, err := safeJoinForComposition(targetDir, rel) if err != nil { return err } + if entry.IsDir() { + target, err = resolveCompositionDirTarget(targetDir, target) + if err != nil { + return err + } + } if err := makePathWritable(targetDir, filepath.Dir(target), originalModes); err != nil { return err } if entry.IsDir() { - info, err := entry.Info() - if err != nil { - return err + if explicitDirs == nil { + info, err := entry.Info() + if err != nil { + return err + } + pendingDirs = append(pendingDirs, dirMeta{src: path, dst: target, info: info}) + } else if _, explicit := explicitDirs[filepath.Clean(rel)]; explicit { + info, err := entry.Info() + if err != nil { + return err + } + pendingDirs = append(pendingDirs, dirMeta{src: path, dst: target, info: info}) } - pendingDirs = append(pendingDirs, dirMeta{src: path, dst: target, info: info}) } if err := copyEntryInto(path, target, hardlinks, targetDir); err != nil { return err @@ -196,7 +214,11 @@ func prepareLayerDirectories(layerDir, targetDir string, originalModes map[strin if err != nil { return err } - target, err := safeJoin(targetDir, rel) + target, err := safeJoinForComposition(targetDir, rel) + if err != nil { + return err + } + target, err = resolveCompositionDirTarget(targetDir, target) if err != nil { return err } @@ -207,7 +229,7 @@ func prepareLayerDirectories(layerDir, targetDir string, originalModes map[strin if err != nil { return err } - if err := copyDirectoryEntry(path, target, info); err != nil { + if err := copyDirectoryEntry(path, target, info, targetDir); err != nil { return err } return makePathWritable(targetDir, target, originalModes) @@ -331,7 +353,7 @@ func copyEntryInto(src, dst string, hardlinks map[hardlinkIdentity]string, root case 0: return copyRegularEntry(src, dst, info, hardlinks) case fs.ModeDir: - return copyDirectoryEntry(src, dst, info) + return copyDirectoryEntry(src, dst, info, root) case fs.ModeSymlink: return copySymlinkEntry(src, dst, root) default: @@ -356,10 +378,20 @@ func copyRegularEntry(src, dst string, info os.FileInfo, hardlinks map[hardlinkI return copyEntryMetadata(src, dst, info) } -func copyDirectoryEntry(src, dst string, info os.FileInfo) error { - if existing, err := os.Lstat(dst); err == nil && !existing.IsDir() { - if err := removePath(dst); err != nil { - return err +func copyDirectoryEntry(src, dst string, info os.FileInfo, root string) error { + if existing, err := os.Lstat(dst); err == nil { + if existing.Mode()&os.ModeSymlink != 0 { + resolved, resolveErr := resolveLayerPath(root, dst, 0) + if resolveErr == nil && pathWithinRoot(root, resolved) { + return nil + } + if err := removePath(dst); err != nil { + return err + } + } else if !existing.IsDir() { + if err := removePath(dst); err != nil { + return err + } } } return os.MkdirAll(dst, info.Mode().Perm()) From 75f2d176b356f5dee487944aaf029a690da7ae22 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Thu, 3 Sep 2026 14:03:48 +0000 Subject: [PATCH 12/41] Use singleflight for layer materialization --- lib/images/layer_artifact.go | 3 --- lib/images/manager.go | 1 - 2 files changed, 4 deletions(-) diff --git a/lib/images/layer_artifact.go b/lib/images/layer_artifact.go index 8733557f1..4cae3d0e4 100644 --- a/lib/images/layer_artifact.go +++ b/lib/images/layer_artifact.go @@ -156,9 +156,6 @@ func (m *manager) materializeLayerArtifactContext(ctx context.Context, desc laye } func (m *manager) materializeLayerArtifactOnce(ctx context.Context, desc layerDescriptor) (*layerArtifact, error) { - m.layerStoreMu.Lock() - defer m.layerStoreMu.Unlock() - if layerArtifactFormat() == "" { return nil, fmt.Errorf("unsupported layer artifact format: %s", DefaultImageFormat) } diff --git a/lib/images/manager.go b/lib/images/manager.go index f18fe8a8d..bceeb0f05 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -77,7 +77,6 @@ type manager struct { queue *queue.Queue createMu sync.Mutex layerFlights singleflight.Group - layerStoreMu sync.Mutex diskUsageMu sync.RWMutex tagGenerations map[string]uint64 requestedTags map[string]string // newest pull's digest per requested tag From deb563dce5b5c0b8eff329ec8183638248c84c3f Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Thu, 3 Sep 2026 14:07:03 +0000 Subject: [PATCH 13/41] Test synthesized directory metadata preservation --- lib/images/layer_artifact_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lib/images/layer_artifact_test.go b/lib/images/layer_artifact_test.go index 78a7bbcd7..8f45d2476 100644 --- a/lib/images/layer_artifact_test.go +++ b/lib/images/layer_artifact_test.go @@ -452,6 +452,21 @@ func TestValidateSymlinkTargetAllowsAbsoluteTargets(t *testing.T) { require.NoError(t, validateSymlinkTarget(root, target, "/etc/resolv.conf")) } +func TestApplyLayerTreePreservesImplicitDirectoryMetadata(t *testing.T) { + root := t.TempDir() + targetDir := filepath.Join(root, "target") + layerDir := filepath.Join(root, "layer") + require.NoError(t, os.MkdirAll(filepath.Join(targetDir, "etc"), 0700)) + require.NoError(t, os.MkdirAll(filepath.Join(layerDir, "etc"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(layerDir, "etc", "app.conf"), []byte("config"), 0644)) + + require.NoError(t, applyLayerTreeWithExplicitDirs(layerDir, targetDir, map[string]struct{}{})) + + info, err := os.Stat(filepath.Join(targetDir, "etc")) + require.NoError(t, err) + require.Equal(t, os.FileMode(0700), info.Mode().Perm()) +} + func TestApplyLayerTreeResolvesInRootSymlinkParents(t *testing.T) { root := t.TempDir() targetDir := filepath.Join(root, "target") From 53437a71ad77b0ca2fc19b67ea070217641afe15 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Thu, 3 Sep 2026 14:27:35 +0000 Subject: [PATCH 14/41] Collapse layer helper wrappers and dead branches Drop the context-less wrapper pairs (materializeLayerArtifact, installLayerArtifact, unpackCachedLayer, unpackLayerBlob, applyLayerTree) in favor of single ctx-taking functions; two of the wrappers had no callers. Reuse unpackCachedLayer from materializeLayerArtifactOnce instead of duplicating the blob lookup and diff-id check. Split the lexical confinement out of safeJoinForComposition into confineToRoot so safeJoin no longer performs and discards a full symlink resolution per tar entry. safeJoin also returns early when the entry resolves to the root itself, which previously looped forever on a "." entry. Pass the walked DirEntry's FileInfo into the copy helpers instead of re-Lstat'ing, remove the unreachable symlink branch in copyDirectoryEntry (callers already resolve through resolveCompositionDirTarget), and lean on removePath's own tree-writability handling for whiteout removal. --- lib/images/disk_usage.go | 2 +- lib/images/layer_artifact.go | 114 ++++++++++++------------------ lib/images/layer_artifact_test.go | 44 ++++++------ lib/images/layer_tree.go | 90 +++++++---------------- 4 files changed, 97 insertions(+), 153 deletions(-) diff --git a/lib/images/disk_usage.go b/lib/images/disk_usage.go index b9b91bf19..0cb6a74d6 100644 --- a/lib/images/disk_usage.go +++ b/lib/images/disk_usage.go @@ -110,7 +110,7 @@ func totalLayerArtifactBytesFromFilesystem(layersDir string) (int64, error) { total += info.Size() return nil }) - if err != nil && !os.IsNotExist(err) { + if err != nil { return 0, fmt.Errorf("walk layer artifacts: %w", err) } return total, nil diff --git a/lib/images/layer_artifact.go b/lib/images/layer_artifact.go index 4cae3d0e4..761c62de4 100644 --- a/lib/images/layer_artifact.go +++ b/lib/images/layer_artifact.go @@ -135,11 +135,7 @@ func discardLayerCache(p *paths.Paths, layerHex string) error { // temp directory; lifecycle reconciliation removes stale temp directories after // an interrupted build. No production caller yet: pull integration and // composition land in later changes. -func (m *manager) materializeLayerArtifact(desc layerDescriptor) (*layerArtifact, error) { - return m.materializeLayerArtifactContext(context.Background(), desc) -} - -func (m *manager) materializeLayerArtifactContext(ctx context.Context, desc layerDescriptor) (*layerArtifact, error) { +func (m *manager) materializeLayerArtifact(ctx context.Context, desc layerDescriptor) (*layerArtifact, error) { key := desc.Digest + "\x00" + layerArtifactFormat() result := m.layerFlights.DoChan(key, func() (any, error) { return m.materializeLayerArtifactOnce(ctx, desc) @@ -178,14 +174,6 @@ func (m *manager) materializeLayerArtifactOnce(ctx context.Context, desc layerDe // Record without artifact: rebuild below. } - blobPath := m.paths.OCICacheBlob(layerHex) - if _, err := os.Stat(blobPath); err != nil { - if os.IsNotExist(err) { - return nil, fmt.Errorf("layer blob missing from oci cache: %s", desc.Digest) - } - return nil, fmt.Errorf("stat layer blob: %w", err) - } - layerDir := m.paths.ImageLayerDir(layerHex) if err := os.MkdirAll(layerDir, 0755); err != nil { return nil, fmt.Errorf("create layer directory: %w", err) @@ -196,22 +184,14 @@ func (m *manager) materializeLayerArtifactOnce(ctx context.Context, desc layerDe } defer removePath(unpackDir) - stats, err := unpackLayerBlobContext(ctx, blobPath, desc.MediaType, unpackDir) + stats, err := unpackCachedLayer(ctx, m.paths.SystemOCICache(), desc, unpackDir) if err != nil { - return nil, fmt.Errorf("unpack layer %s: %w", desc.Digest, err) - } - if desc.DiffID != "" && stats.diffID != desc.DiffID { - return nil, fmt.Errorf("layer %s diff id mismatch: got %s, want %s", desc.Digest, stats.diffID, desc.DiffID) + return nil, err } - - return m.installLayerArtifactContext(ctx, desc, layerHex, unpackDir, stats) + return m.installLayerArtifact(ctx, desc, layerHex, unpackDir, stats) } -func (m *manager) installLayerArtifact(desc layerDescriptor, layerHex, unpackDir string, stats *unpackStats) (*layerArtifact, error) { - return m.installLayerArtifactContext(context.Background(), desc, layerHex, unpackDir, stats) -} - -func (m *manager) installLayerArtifactContext(ctx context.Context, desc layerDescriptor, layerHex, unpackDir string, stats *unpackStats) (*layerArtifact, error) { +func (m *manager) installLayerArtifact(ctx context.Context, desc layerDescriptor, layerHex, unpackDir string, stats *unpackStats) (*layerArtifact, error) { record := &layerArtifact{ SchemaVersion: layerRecordSchemaVersion, Digest: desc.Digest, @@ -259,15 +239,9 @@ type unpackStats struct { explicitDirs map[string]struct{} } -// unpackLayerBlob extracts one compressed layer blob into dest while preserving -// whiteout marker files. It intentionally does not use umoci's layer unpacker: -// umoci consumes whiteouts while this store must retain them for composition. -// Paths are confined to dest. -func unpackCachedLayer(cacheDir string, desc layerDescriptor, dest string) (*unpackStats, error) { - return unpackCachedLayerContext(context.Background(), cacheDir, desc, dest) -} - -func unpackCachedLayerContext(ctx context.Context, cacheDir string, desc layerDescriptor, dest string) (*unpackStats, error) { +// unpackCachedLayer locates desc's blob in the shared OCI cache, unpacks it +// into dest, and verifies the diff ID when the descriptor carries one. +func unpackCachedLayer(ctx context.Context, cacheDir string, desc layerDescriptor, dest string) (*unpackStats, error) { layerHex := strings.TrimPrefix(desc.Digest, "sha256:") if err := paths.ValidatePathComponent(layerHex); err != nil { return nil, fmt.Errorf("invalid layer digest: %s", desc.Digest) @@ -279,7 +253,7 @@ func unpackCachedLayerContext(ctx context.Context, cacheDir string, desc layerDe } return nil, fmt.Errorf("stat layer blob: %w", err) } - stats, err := unpackLayerBlobContext(ctx, blobPath, desc.MediaType, dest) + stats, err := unpackLayerBlob(ctx, blobPath, desc.MediaType, dest) if err != nil { return nil, fmt.Errorf("unpack layer %s: %w", desc.Digest, err) } @@ -289,11 +263,11 @@ func unpackCachedLayerContext(ctx context.Context, cacheDir string, desc layerDe return stats, nil } -func unpackLayerBlob(blobPath, mediaType, dest string) (*unpackStats, error) { - return unpackLayerBlobContext(context.Background(), blobPath, mediaType, dest) -} - -func unpackLayerBlobContext(ctx context.Context, blobPath, mediaType, dest string) (*unpackStats, error) { +// unpackLayerBlob extracts one compressed layer blob into dest while preserving +// whiteout marker files. It intentionally does not use umoci's layer unpacker: +// umoci consumes whiteouts while this store must retain them for composition. +// Paths are confined to dest. +func unpackLayerBlob(ctx context.Context, blobPath, mediaType, dest string) (*unpackStats, error) { if err := os.MkdirAll(dest, 0755); err != nil { return nil, fmt.Errorf("create extraction root: %w", err) } @@ -471,17 +445,40 @@ func (c multiCloser) Close() error { return firstErr } +// confineToRoot lexically resolves an entry name inside root, rejecting +// absolute names and parent traversal. root must be an existing directory +// that is not itself a symlink. +func confineToRoot(root, name string) (string, error) { + root = filepath.Clean(root) + rootInfo, err := os.Lstat(root) + if err != nil { + return "", fmt.Errorf("inspect extraction root: %w", err) + } + if !rootInfo.IsDir() || rootInfo.Mode()&os.ModeSymlink != 0 { + return "", fmt.Errorf("extraction root is not a directory: %s", root) + } + if filepath.IsAbs(name) { + return "", fmt.Errorf("tar entry escapes root: %s", name) + } + target := filepath.Join(root, name) + if !pathWithinRoot(root, target) { + return "", fmt.Errorf("tar entry escapes root: %s", name) + } + return target, nil +} + // safeJoin resolves a tar entry inside root and rejects symlinked parents // while extracting a layer. This prevents one tar entry from changing where // a later entry is written. func safeJoin(root, name string) (string, error) { - resolved, err := safeJoinForComposition(root, name) + target, err := confineToRoot(root, name) if err != nil { return "", err } root = filepath.Clean(root) - clean := filepath.Clean(name) - target := filepath.Join(root, clean) + if target == root { + return root, nil + } for parent := filepath.Dir(target); parent != root; parent = filepath.Dir(parent) { info, err := os.Lstat(parent) if err != nil { @@ -497,36 +494,21 @@ func safeJoin(root, name string) (string, error) { return "", fmt.Errorf("tar entry parent is not a directory: %s", parent) } } - return resolved, nil + return target, nil } // safeJoinForComposition resolves an entry through existing parent symlinks, // interpreting absolute link targets relative to the image root. func safeJoinForComposition(root, name string) (string, error) { - root = filepath.Clean(root) - rootInfo, err := os.Lstat(root) + target, err := confineToRoot(root, name) if err != nil { - return "", fmt.Errorf("inspect extraction root: %w", err) - } - if !rootInfo.IsDir() || rootInfo.Mode()&os.ModeSymlink != 0 { - return "", fmt.Errorf("extraction root is not a directory: %s", root) - } - if filepath.IsAbs(name) { - return "", fmt.Errorf("tar entry escapes root: %s", name) + return "", err } - clean := filepath.Clean(name) - if clean == "." { + root = filepath.Clean(root) + if target == root { return root, nil } - if clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { - return "", fmt.Errorf("tar entry escapes root: %s", name) - } - target := filepath.Join(root, clean) - if !strings.HasPrefix(target, root+string(filepath.Separator)) { - return "", fmt.Errorf("tar entry escapes root: %s", name) - } - parent := filepath.Dir(target) - resolvedParent, err := resolveLayerPath(root, parent, 0) + resolvedParent, err := resolveLayerPath(root, filepath.Dir(target), 0) if err != nil { return "", fmt.Errorf("inspect tar entry parent: %w", err) } @@ -610,9 +592,7 @@ func validateSymlinkTarget(root, target, linkname string) error { if filepath.IsAbs(linkname) { return nil } - resolved := filepath.Clean(filepath.Join(filepath.Dir(target), linkname)) - root = filepath.Clean(root) - if resolved != root && !strings.HasPrefix(resolved, root+string(filepath.Separator)) { + if !pathWithinRoot(root, filepath.Join(filepath.Dir(target), linkname)) { return fmt.Errorf("symlink target escapes root: %s", linkname) } return nil diff --git a/lib/images/layer_artifact_test.go b/lib/images/layer_artifact_test.go index 8f45d2476..96f900e92 100644 --- a/lib/images/layer_artifact_test.go +++ b/lib/images/layer_artifact_test.go @@ -71,7 +71,7 @@ func TestMaterializeLayerArtifact(t *testing.T) { desc := layerDescFromImage(t, img, 0) m := &manager{paths: p} - record, err := m.materializeLayerArtifact(desc) + record, err := m.materializeLayerArtifact(context.Background(), desc) require.NoError(t, err) require.Equal(t, desc.Digest, record.Digest) require.Equal(t, desc.DiffID, record.DiffID) @@ -87,7 +87,7 @@ func TestMaterializeLayerArtifact(t *testing.T) { // A second materialization reuses the existing artifact. artifactInfo, err := os.Stat(p.ImageLayerArtifactForFormat(layerHex, layerArtifactFormat())) require.NoError(t, err) - reused, err := m.materializeLayerArtifact(desc) + reused, err := m.materializeLayerArtifact(context.Background(), desc) require.NoError(t, err) require.True(t, record.CreatedAt.Equal(reused.CreatedAt), "reuse must return the stored record") artifactInfoAfter, err := os.Stat(p.ImageLayerArtifactForFormat(layerHex, layerArtifactFormat())) @@ -110,7 +110,7 @@ func TestMaterializeLayerArtifactRecoversCorruptRecord(t *testing.T) { desc := layerDescFromImage(t, img, 0) m := &manager{paths: p} - first, err := m.materializeLayerArtifact(desc) + first, err := m.materializeLayerArtifact(context.Background(), desc) require.NoError(t, err) layerHex := desc.Digest[len("sha256:"):] @@ -120,7 +120,7 @@ func TestMaterializeLayerArtifactRecoversCorruptRecord(t *testing.T) { 0600, )) - second, err := m.materializeLayerArtifact(desc) + second, err := m.materializeLayerArtifact(context.Background(), desc) require.NoError(t, err) require.NotEqual(t, first.CreatedAt, second.CreatedAt, "corrupt record must trigger a rebuild") require.FileExists(t, p.ImageLayerArtifactForFormat(layerHex, layerArtifactFormat())) @@ -136,7 +136,7 @@ func TestMaterializeLayerArtifactMissingBlob(t *testing.T) { p := paths.New(t.TempDir()) m := &manager{paths: p} - _, err := m.materializeLayerArtifact(layerDescriptor{ + _, err := m.materializeLayerArtifact(context.Background(), layerDescriptor{ Digest: "sha256:abababababababababababababababababababababababababababababababab", MediaType: "application/vnd.oci.image.layer.v1.tar+gzip", }) @@ -193,7 +193,7 @@ func TestMaterializeLayerDoesNotPersistWhiteoutInventory(t *testing.T) { desc := layerDescFromImage(t, img, 0) m := &manager{paths: p} - _, err = m.materializeLayerArtifact(desc) + _, err = m.materializeLayerArtifact(context.Background(), desc) require.NoError(t, err) data, err := os.ReadFile(p.ImageLayerRecordForFormat(desc.Digest[len("sha256:"):], layerArtifactFormat())) @@ -211,7 +211,7 @@ func TestApplyLayerTreeCopiesRestrictiveFiles(t *testing.T) { require.NoError(t, os.WriteFile(file, []byte("secret"), 0644)) require.NoError(t, os.Chmod(file, 0000)) - require.NoError(t, applyLayerTree(layerDir, targetDir)) + require.NoError(t, applyLayerTree(layerDir, targetDir, nil)) target := filepath.Join(targetDir, "secret.txt") info, err := os.Stat(target) @@ -267,7 +267,7 @@ func TestApplyLayerTreeWhiteoutSemantics(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(layerDir, "added", ".wh.foo"), nil, 0644)) require.NoError(t, os.WriteFile(filepath.Join(layerDir, "added", "foo"), []byte("new"), 0644)) - require.NoError(t, applyLayerTree(layerDir, targetDir)) + require.NoError(t, applyLayerTree(layerDir, targetDir, nil)) // Whiteout removed the lower entry. _, err := os.Lstat(filepath.Join(targetDir, "victim.txt")) @@ -325,7 +325,7 @@ func TestApplyLayerTreeReplacesSymlinkBeforeNestedWhiteout(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(layerDir, "replaced", ".wh.old.txt"), nil, 0644)) require.NoError(t, os.WriteFile(filepath.Join(layerDir, "replaced", "new.txt"), []byte("new"), 0644)) - require.NoError(t, applyLayerTree(layerDir, targetDir)) + require.NoError(t, applyLayerTree(layerDir, targetDir, nil)) info, err := os.Lstat(filepath.Join(targetDir, "replaced")) require.NoError(t, err) @@ -344,7 +344,7 @@ func TestApplyLayerTreePreservesUpperDirectoryMode(t *testing.T) { require.NoError(t, os.MkdirAll(filepath.Join(layerDir, "etc"), 0750)) require.NoError(t, os.WriteFile(filepath.Join(layerDir, "etc", "new.txt"), []byte("new"), 0644)) - require.NoError(t, applyLayerTree(layerDir, targetDir)) + require.NoError(t, applyLayerTree(layerDir, targetDir, nil)) info, err := os.Stat(filepath.Join(targetDir, "etc")) require.NoError(t, err) @@ -362,7 +362,7 @@ func TestApplyLayerTreeWhiteoutRemovesReadOnlyDirectory(t *testing.T) { require.NoError(t, os.MkdirAll(layerDir, 0755)) require.NoError(t, os.WriteFile(filepath.Join(layerDir, ".wh.gone"), nil, 0644)) - require.NoError(t, applyLayerTree(layerDir, targetDir)) + require.NoError(t, applyLayerTree(layerDir, targetDir, nil)) _, err := os.Lstat(filepath.Join(targetDir, "gone")) require.ErrorIs(t, err, os.ErrNotExist) } @@ -380,7 +380,7 @@ func TestUnpackLayerBlobContextHonorsCancellation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - _, err = unpackLayerBlobContext(ctx, blobPath, "application/vnd.oci.image.layer.v1.tar", filepath.Join(root, "dest")) + _, err = unpackLayerBlob(ctx, blobPath, "application/vnd.oci.image.layer.v1.tar", filepath.Join(root, "dest")) require.ErrorIs(t, err, context.Canceled) } @@ -404,7 +404,7 @@ func TestUnpackLayerBlobIncludesTrailingTarPaddingInDiffID(t *testing.T) { require.NoError(t, gzw.Close()) require.NoError(t, os.WriteFile(blobPath, compressed.Bytes(), 0644)) - stats, err := unpackLayerBlob(blobPath, "application/vnd.oci.image.layer.v1.tar+gzip", filepath.Join(root, "dest")) + stats, err := unpackLayerBlob(context.Background(), blobPath, "application/vnd.oci.image.layer.v1.tar+gzip", filepath.Join(root, "dest")) require.NoError(t, err) want := sha256.Sum256(tarData.Bytes()) require.Equal(t, "sha256:"+fmt.Sprintf("%x", want), stats.diffID) @@ -442,7 +442,7 @@ func TestUnpackLayerBlobRejectsSymlinkTraversal(t *testing.T) { require.NoError(t, gzw.Close()) require.NoError(t, os.WriteFile(blobPath, buf.Bytes(), 0644)) - _, err = unpackLayerBlob(blobPath, "application/vnd.oci.image.layer.v1.tar+gzip", filepath.Join(root, "dest")) + _, err = unpackLayerBlob(context.Background(), blobPath, "application/vnd.oci.image.layer.v1.tar+gzip", filepath.Join(root, "dest")) require.ErrorContains(t, err, "symlink") } @@ -460,7 +460,7 @@ func TestApplyLayerTreePreservesImplicitDirectoryMetadata(t *testing.T) { require.NoError(t, os.MkdirAll(filepath.Join(layerDir, "etc"), 0755)) require.NoError(t, os.WriteFile(filepath.Join(layerDir, "etc", "app.conf"), []byte("config"), 0644)) - require.NoError(t, applyLayerTreeWithExplicitDirs(layerDir, targetDir, map[string]struct{}{})) + require.NoError(t, applyLayerTree(layerDir, targetDir, map[string]struct{}{})) info, err := os.Stat(filepath.Join(targetDir, "etc")) require.NoError(t, err) @@ -476,7 +476,7 @@ func TestApplyLayerTreeResolvesInRootSymlinkParents(t *testing.T) { require.NoError(t, os.MkdirAll(filepath.Join(layerDir, "bin"), 0755)) require.NoError(t, os.WriteFile(filepath.Join(layerDir, "bin", "tool"), []byte("tool"), 0644)) - require.NoError(t, applyLayerTree(layerDir, targetDir)) + require.NoError(t, applyLayerTree(layerDir, targetDir, nil)) data, err := os.ReadFile(filepath.Join(targetDir, "usr", "bin", "tool")) require.NoError(t, err) @@ -503,7 +503,7 @@ func TestApplyLayerTreeSymlinksAndHardlinks(t *testing.T) { require.NoError(t, os.Link(filepath.Join(layerDir, "a.txt"), filepath.Join(layerDir, "b.txt"))) require.NoError(t, os.Symlink("a.txt", filepath.Join(layerDir, "link-to-a"))) - require.NoError(t, applyLayerTree(layerDir, targetDir)) + require.NoError(t, applyLayerTree(layerDir, targetDir, nil)) _, err := os.Lstat(filepath.Join(targetDir, "alias")) require.True(t, os.IsNotExist(err), "symlink whiteout must remove the link itself") @@ -565,7 +565,7 @@ func TestUnpackLayerBlobPreservesDirMtime(t *testing.T) { require.NoError(t, os.WriteFile(blobPath, buf.Bytes(), 0644)) dest := filepath.Join(root, "dest") - _, err = unpackLayerBlob(blobPath, "application/vnd.oci.image.layer.v1.tar", dest) + _, err = unpackLayerBlob(context.Background(), blobPath, "application/vnd.oci.image.layer.v1.tar", dest) require.NoError(t, err) info, err := os.Stat(filepath.Join(dest, "d")) @@ -609,7 +609,7 @@ func TestApplyLayerTreePreservesDirMtime(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(layerDir, "d", "file.txt"), []byte("x"), 0644)) require.NoError(t, os.Chtimes(filepath.Join(layerDir, "d"), old, old)) - require.NoError(t, applyLayerTree(layerDir, targetDir)) + require.NoError(t, applyLayerTree(layerDir, targetDir, nil)) info, err := os.Stat(filepath.Join(targetDir, "d")) require.NoError(t, err) @@ -652,7 +652,7 @@ func TestApplyLayerTreeRejectsMalformedWhiteouts(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(layerDir, "parent", ".wh.."), nil, 0644)) require.NoError(t, os.WriteFile(filepath.Join(layerDir, "parent", ".wh..."), nil, 0644)) - err := applyLayerTree(layerDir, targetDir) + err := applyLayerTree(layerDir, targetDir, nil) require.ErrorContains(t, err, "invalid whiteout entry") // Parent directory must not be deleted by malformed whiteout paths. @@ -670,7 +670,7 @@ func TestApplyLayerTreeRejectsEscapingSymlink(t *testing.T) { require.NoError(t, os.MkdirAll(layerDir, 0755)) require.NoError(t, os.Symlink("../../outside", filepath.Join(layerDir, "link"))) - err := applyLayerTree(layerDir, targetDir) + err := applyLayerTree(layerDir, targetDir, nil) require.ErrorContains(t, err, "symlink target escapes root") } @@ -685,7 +685,7 @@ func TestApplyLayerTreeRestoresRestrictiveNestedDirModes(t *testing.T) { require.NoError(t, os.Chmod(filepath.Join(layerDir, "outer", "inner"), 0500)) require.NoError(t, os.Chmod(filepath.Join(layerDir, "outer"), 0500)) - require.NoError(t, applyLayerTree(layerDir, targetDir)) + require.NoError(t, applyLayerTree(layerDir, targetDir, nil)) infoOuter, err := os.Stat(filepath.Join(targetDir, "outer")) require.NoError(t, err) diff --git a/lib/images/layer_tree.go b/lib/images/layer_tree.go index 03358d11e..2d2d196a6 100644 --- a/lib/images/layer_tree.go +++ b/lib/images/layer_tree.go @@ -59,22 +59,18 @@ func makeTreeWritable(path string) error { // whiteout files are interpreted here rather than passed through, because // overlayfs does not understand them. No production caller yet; composition // lands in a later change. -func applyLayerTree(layerDir, targetDir string) error { - return applyLayerTreeWithExplicitDirs(layerDir, targetDir, nil) -} - -func applyLayerTreeWithExplicitDirs(layerDir, targetDir string, explicitDirs map[string]struct{}) (err error) { +// +// explicitDirs, when non-nil, names the directories the layer tar listed +// explicitly (relative, cleaned); only those have their metadata copied. +// A nil map treats every directory as explicit. +func applyLayerTree(layerDir, targetDir string, explicitDirs map[string]struct{}) (err error) { if err := os.MkdirAll(targetDir, 0755); err != nil { return fmt.Errorf("create target directory: %w", err) } originalModes := make(map[string]fs.FileMode) defer func() { if restoreErr := restoreDirectoryModes(originalModes); restoreErr != nil { - if err == nil { - err = fmt.Errorf("restore directory modes: %w", restoreErr) - } else { - err = errors.Join(err, fmt.Errorf("restore directory modes: %w", restoreErr)) - } + err = errors.Join(err, fmt.Errorf("restore directory modes: %w", restoreErr)) } }() @@ -115,13 +111,6 @@ func applyLayerTreeWithExplicitDirs(layerDir, targetDir string, explicitDirs map if err != nil { return err } - if info, err := os.Lstat(target); err == nil && info.IsDir() { - if err := makePathWritable(targetDir, target, originalModes); err != nil { - return err - } - } else if err != nil && !os.IsNotExist(err) { - return err - } return removePath(target) }) if err != nil { @@ -163,22 +152,16 @@ func applyLayerTreeWithExplicitDirs(layerDir, targetDir string, explicitDirs map if err := makePathWritable(targetDir, filepath.Dir(target), originalModes); err != nil { return err } + info, err := entry.Info() + if err != nil { + return err + } if entry.IsDir() { - if explicitDirs == nil { - info, err := entry.Info() - if err != nil { - return err - } - pendingDirs = append(pendingDirs, dirMeta{src: path, dst: target, info: info}) - } else if _, explicit := explicitDirs[filepath.Clean(rel)]; explicit { - info, err := entry.Info() - if err != nil { - return err - } + if _, explicit := explicitDirs[filepath.Clean(rel)]; explicitDirs == nil || explicit { pendingDirs = append(pendingDirs, dirMeta{src: path, dst: target, info: info}) } } - if err := copyEntryInto(path, target, hardlinks, targetDir); err != nil { + if err := copyEntryInto(path, target, info, hardlinks, targetDir); err != nil { return err } if entry.IsDir() { @@ -229,7 +212,7 @@ func prepareLayerDirectories(layerDir, targetDir string, originalModes map[strin if err != nil { return err } - if err := copyDirectoryEntry(path, target, info, targetDir); err != nil { + if err := copyDirectoryEntry(target, info); err != nil { return err } return makePathWritable(targetDir, target, originalModes) @@ -239,7 +222,7 @@ func prepareLayerDirectories(layerDir, targetDir string, originalModes map[strin func makePathWritable(root, path string, originalModes map[string]fs.FileMode) error { root = filepath.Clean(root) path = filepath.Clean(path) - if path != root && !strings.HasPrefix(path, root+string(filepath.Separator)) { + if !pathWithinRoot(root, path) { return fmt.Errorf("path is outside target root: %s", path) } rel, err := filepath.Rel(root, path) @@ -344,18 +327,14 @@ type hardlinkIdentity struct { // copyEntryInto copies one filesystem entry from src to dst, replacing any // conflicting entry and preserving hardlinks within the layer. -func copyEntryInto(src, dst string, hardlinks map[hardlinkIdentity]string, root string) error { - info, err := os.Lstat(src) - if err != nil { - return err - } +func copyEntryInto(src, dst string, info os.FileInfo, hardlinks map[hardlinkIdentity]string, root string) error { switch info.Mode() & fs.ModeType { case 0: return copyRegularEntry(src, dst, info, hardlinks) case fs.ModeDir: - return copyDirectoryEntry(src, dst, info, root) + return copyDirectoryEntry(dst, info) case fs.ModeSymlink: - return copySymlinkEntry(src, dst, root) + return copySymlinkEntry(src, dst, info, root) default: return copySpecialEntry(src, dst, info) } @@ -372,32 +351,25 @@ func copyRegularEntry(src, dst string, info os.FileInfo, hardlinks map[hardlinkI } hardlinks[identity] = dst } - if err := copyFileContents(src, dst); err != nil { + if err := copyFileContents(src, dst, info); err != nil { return err } return copyEntryMetadata(src, dst, info) } -func copyDirectoryEntry(src, dst string, info os.FileInfo, root string) error { - if existing, err := os.Lstat(dst); err == nil { - if existing.Mode()&os.ModeSymlink != 0 { - resolved, resolveErr := resolveLayerPath(root, dst, 0) - if resolveErr == nil && pathWithinRoot(root, resolved) { - return nil - } - if err := removePath(dst); err != nil { - return err - } - } else if !existing.IsDir() { - if err := removePath(dst); err != nil { - return err - } +// copyDirectoryEntry creates dst as a directory, replacing a conflicting +// non-directory. Callers resolve dst through resolveCompositionDirTarget +// first, so dst is never an existing symlink here. +func copyDirectoryEntry(dst string, info os.FileInfo) error { + if existing, err := os.Lstat(dst); err == nil && !existing.IsDir() { + if err := removePath(dst); err != nil { + return err } } return os.MkdirAll(dst, info.Mode().Perm()) } -func copySymlinkEntry(src, dst, root string) error { +func copySymlinkEntry(src, dst string, info os.FileInfo, root string) error { linkTarget, err := os.Readlink(src) if err != nil { return err @@ -411,10 +383,6 @@ func copySymlinkEntry(src, dst, root string) error { if err := os.Symlink(linkTarget, dst); err != nil { return err } - info, err := os.Lstat(src) - if err != nil { - return err - } return copyEntryMetadata(src, dst, info) } @@ -568,11 +536,7 @@ func applyXattrs(path string, xattrs map[string][]byte) error { return nil } -func copyFileContents(src, dst string) (err error) { - info, err := os.Lstat(src) - if err != nil { - return err - } +func copyFileContents(src, dst string, info os.FileInfo) (err error) { sourceMode := info.Mode().Perm() | info.Mode()&(os.ModeSetuid|os.ModeSetgid|os.ModeSticky) if sourceMode&0400 == 0 { if err := os.Chmod(src, sourceMode|0400); err != nil { From 21b7c15cc4f9702e10f339af904eec786a00da0b Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Thu, 3 Sep 2026 14:42:53 +0000 Subject: [PATCH 15/41] Drop access time preservation from layer metadata Restoring atime required per-platform Stat_t accessors for a value that tar headers almost never carry and that nothing reads back. Set atime to mtime on both the tar extraction and tree copy paths instead. --- lib/images/layer_artifact.go | 5 ----- lib/images/layer_artifact_test.go | 20 -------------------- lib/images/layer_atime_darwin.go | 12 ------------ lib/images/layer_atime_linux.go | 12 ------------ lib/images/layer_tree.go | 5 +---- 5 files changed, 1 insertion(+), 53 deletions(-) delete mode 100644 lib/images/layer_atime_darwin.go delete mode 100644 lib/images/layer_atime_linux.go diff --git a/lib/images/layer_artifact.go b/lib/images/layer_artifact.go index 761c62de4..dd156a6a2 100644 --- a/lib/images/layer_artifact.go +++ b/lib/images/layer_artifact.go @@ -692,17 +692,12 @@ func applyTarMetadata(path string, header *tar.Header) error { if mtime.IsZero() { mtime = time.Now() } - atime := header.AccessTime - if atime.IsZero() { - atime = mtime - } return applyEntryMetadata(path, entryMetadata{ uid: header.Uid, gid: header.Gid, hasOwner: true, mode: header.FileInfo().Mode(), symlink: header.Typeflag == tar.TypeSymlink, - atime: atime, mtime: mtime, xattrs: tarXattrs(header), }) diff --git a/lib/images/layer_artifact_test.go b/lib/images/layer_artifact_test.go index 96f900e92..eeff21fa5 100644 --- a/lib/images/layer_artifact_test.go +++ b/lib/images/layer_artifact_test.go @@ -573,26 +573,6 @@ func TestUnpackLayerBlobPreservesDirMtime(t *testing.T) { require.True(t, info.ModTime().Equal(dirTime), "dir mtime must come from the tar header") } -func TestCopyEntryMetadataPreservesAccessTime(t *testing.T) { - root := t.TempDir() - src := filepath.Join(root, "src") - dst := filepath.Join(root, "dst") - require.NoError(t, os.WriteFile(src, []byte("payload"), 0644)) - require.NoError(t, os.WriteFile(dst, nil, 0644)) - - atime := time.Now().Add(-2 * time.Hour).Truncate(time.Second) - mtime := time.Now().Add(-time.Hour).Truncate(time.Second) - require.NoError(t, os.Chtimes(src, atime, mtime)) - info, err := os.Lstat(src) - require.NoError(t, err) - require.NoError(t, copyEntryMetadata(src, dst, info)) - - var stat unix.Stat_t - require.NoError(t, unix.Lstat(dst, &stat)) - require.True(t, time.Unix(stat.Atim.Sec, stat.Atim.Nsec).Equal(atime), "atime must be preserved") - require.True(t, time.Unix(stat.Mtim.Sec, stat.Mtim.Nsec).Equal(mtime), "mtime must be preserved") -} - func TestSpecialFileModeSupportsSockets(t *testing.T) { mode, err := specialFileMode(fs.ModeSocket) require.NoError(t, err) diff --git a/lib/images/layer_atime_darwin.go b/lib/images/layer_atime_darwin.go deleted file mode 100644 index 670259c4c..000000000 --- a/lib/images/layer_atime_darwin.go +++ /dev/null @@ -1,12 +0,0 @@ -//go:build darwin - -package images - -import ( - "syscall" - "time" -) - -func statAtime(stat *syscall.Stat_t) time.Time { - return time.Unix(stat.Atimespec.Sec, stat.Atimespec.Nsec) -} diff --git a/lib/images/layer_atime_linux.go b/lib/images/layer_atime_linux.go deleted file mode 100644 index e2912da06..000000000 --- a/lib/images/layer_atime_linux.go +++ /dev/null @@ -1,12 +0,0 @@ -//go:build linux - -package images - -import ( - "syscall" - "time" -) - -func statAtime(stat *syscall.Stat_t) time.Time { - return time.Unix(stat.Atim.Sec, stat.Atim.Nsec) -} diff --git a/lib/images/layer_tree.go b/lib/images/layer_tree.go index 2d2d196a6..634a015b3 100644 --- a/lib/images/layer_tree.go +++ b/lib/images/layer_tree.go @@ -424,7 +424,6 @@ type entryMetadata struct { hasOwner bool mode fs.FileMode symlink bool - atime time.Time mtime time.Time xattrs map[string][]byte } @@ -445,21 +444,19 @@ func applyEntryMetadata(path string, metadata entryMetadata) error { if err := os.Chmod(path, mode); err != nil { return err } - return os.Chtimes(path, metadata.atime, metadata.mtime) + return os.Chtimes(path, metadata.mtime, metadata.mtime) } func copyEntryMetadata(src, dst string, info os.FileInfo) error { metadata := entryMetadata{ mode: info.Mode(), symlink: info.Mode()&os.ModeSymlink != 0, - atime: info.ModTime(), mtime: info.ModTime(), } if stat, ok := info.Sys().(*syscall.Stat_t); ok { metadata.uid = int(stat.Uid) metadata.gid = int(stat.Gid) metadata.hasOwner = true - metadata.atime = statAtime(stat) } if !metadata.symlink { xattrs, err := readXattrs(src) From f950306405955ad174270c196b69913ff2c415b0 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Thu, 3 Sep 2026 14:54:20 +0000 Subject: [PATCH 16/41] Extract layers with umoci instead of a private tar walker Replace the hand-rolled tar extractor and tree merger with umoci's layer.UnpackLayer, which the pull path already depends on. umoci confines entries with filepath-securejoin, interprets OCI whiteouts, and follows the same directory-over-symlink rule as containerd and docker (replace, do not write through), so composition no longer diverges from other runtimes. Per-layer artifacts are extracted in OverlayfsRootfs form: whiteouts become 0:0 character devices and opaque directories carry the overlayfs opaque xattr, the on-disk representation an overlayfs mount of stacked layers understands. Composition applies each layer directly onto the staging tree with DirRootfs, removing the unpack-then-copy pass and the directory mode bookkeeping it required. Ownership is preserved when running as root; unprivileged runs use umoci's rootless mode as before. The record drops the entry count, which is no longer observable without a second pass over the tar, and reports the decompressed tar size as unpacked bytes. --- lib/images/fs_entry.go | 33 -- lib/images/layer_artifact.go | 460 +++---------------- lib/images/layer_artifact_test.go | 719 ++++++++---------------------- lib/images/layer_tree.go | 563 ----------------------- 4 files changed, 258 insertions(+), 1517 deletions(-) delete mode 100644 lib/images/fs_entry.go delete mode 100644 lib/images/layer_tree.go diff --git a/lib/images/fs_entry.go b/lib/images/fs_entry.go deleted file mode 100644 index 72443077f..000000000 --- a/lib/images/fs_entry.go +++ /dev/null @@ -1,33 +0,0 @@ -package images - -import ( - "errors" - "fmt" - "os" - "syscall" - - "golang.org/x/sys/unix" -) - -func mknodWithRootlessFallback(path string, mode uint32, dev int) error { - if err := unix.Mknod(path, mode, dev); err != nil { - if !errors.Is(err, unix.EPERM) { - return fmt.Errorf("mknod: %w", err) - } - if err := createRootlessDevicePlaceholder(path); err != nil { - return err - } - } - return nil -} - -func createRootlessDevicePlaceholder(path string) error { - file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|syscall.O_NOFOLLOW, 0644) - if err != nil { - return fmt.Errorf("create rootless device placeholder: %w", err) - } - if err := file.Close(); err != nil { - return err - } - return nil -} diff --git a/lib/images/layer_artifact.go b/lib/images/layer_artifact.go index dd156a6a2..6d67a9c56 100644 --- a/lib/images/layer_artifact.go +++ b/lib/images/layer_artifact.go @@ -1,7 +1,6 @@ package images import ( - "archive/tar" "compress/gzip" "context" "crypto/sha256" @@ -9,30 +8,25 @@ import ( "errors" "fmt" "io" + "io/fs" "os" "path/filepath" "strings" - "syscall" "time" "github.com/kernel/hypeman/lib/paths" "github.com/klauspost/compress/zstd" - "golang.org/x/sys/unix" + "github.com/opencontainers/umoci/oci/layer" ) -// OCI whiteout marker files. A ".wh." entry in directory D removes -// "" from D as inherited from lower layers; a ".wh..wh..opq" entry marks -// its directory opaque, hiding everything below it from lower layers. These -// are tar-level conventions: they do not compose on overlayfs by themselves, -// so composition must interpret them explicitly (see applyLayerTree). -const ( - whiteoutPrefix = ".wh." - opaqueWhiteout = ".wh..wh..opq" -) +// whiteoutPrefix marks OCI whiteout entries (".wh." and ".wh..wh..opq"). +// umoci interprets them during extraction: DirRootfs applies them against the +// tree being composed, OverlayfsRootfs converts them to overlayfs whiteout +// inodes and opaque xattrs so per-layer artifacts can later be stacked. +const whiteoutPrefix = ".wh." const ( layerRecordSchemaVersion = 1 - maxLayerEntries = 1_000_000 maxLayerUnpackedBytes = 100 << 30 ) @@ -46,9 +40,8 @@ type layerArtifact struct { Digest string `json:"digest"` // compressed layer blob digest, sha256:... DiffID string `json:"diff_id,omitempty"` Format string `json:"format"` - SizeBytes int64 `json:"size_bytes"` // artifact bytes on disk - UnpackedBytes int64 `json:"unpacked_bytes"` - Entries int `json:"entries"` + SizeBytes int64 `json:"size_bytes"` // artifact bytes on disk + UnpackedBytes int64 `json:"unpacked_bytes"` // decompressed tar stream bytes CreatedAt time.Time `json:"created_at"` } @@ -65,8 +58,8 @@ func (a *layerArtifact) validate() error { if a.Format != string(FormatErofs) && a.Format != string(FormatExt4) { return fmt.Errorf("invalid format: %s", a.Format) } - if a.SizeBytes < 0 || a.UnpackedBytes < 0 || a.Entries < 0 { - return fmt.Errorf("invalid size or entry counts") + if a.SizeBytes < 0 || a.UnpackedBytes < 0 { + return fmt.Errorf("invalid size") } return nil } @@ -95,6 +88,25 @@ func layerArtifactRecordPath(p *paths.Paths, layerHex string) string { return p.ImageLayerRecordForFormat(layerHex, layerArtifactFormat()) } +// layerMapOptions preserves tar ownership when running as root. Otherwise +// umoci's rootless mode skips chown and stands in empty files for device nodes. +func layerMapOptions() layer.MapOptions { + return layer.MapOptions{Rootless: os.Geteuid() != 0} +} + +// layerArtifactOnDiskFormat is the extraction format for per-layer artifacts. +// Whiteouts become overlayfs whiteout inodes and opaque xattrs, the form an +// overlayfs mount of stacked layers understands, so the artifact retains the +// layer's deletions without a private marker format. +func layerArtifactOnDiskFormat() layer.OnDiskFormat { + return layer.OverlayfsRootfs{MapOptions: layerMapOptions()} +} + +// composeOnDiskFormat applies whiteouts against the tree being composed. +func composeOnDiskFormat() layer.OnDiskFormat { + return layer.DirRootfs{MapOptions: layerMapOptions()} +} + // readLayerRecord loads the artifact record for a layer digest, if present. // A missing record returns (nil, nil): the layer simply was never // materialized. @@ -184,7 +196,7 @@ func (m *manager) materializeLayerArtifactOnce(ctx context.Context, desc layerDe } defer removePath(unpackDir) - stats, err := unpackCachedLayer(ctx, m.paths.SystemOCICache(), desc, unpackDir) + stats, err := unpackCachedLayer(ctx, m.paths.SystemOCICache(), desc, unpackDir, layerArtifactOnDiskFormat()) if err != nil { return nil, err } @@ -198,13 +210,10 @@ func (m *manager) installLayerArtifact(ctx context.Context, desc layerDescriptor DiffID: stats.diffID, Format: layerArtifactFormat(), UnpackedBytes: stats.unpackedBytes, - Entries: stats.entries, CreatedAt: time.Now(), } if err := installAtomically(layerArtifactPath(m.paths, layerHex), func(path string) error { - // The artifact intentionally retains .wh. marker files so - // composition can re-derive whiteouts from the tree itself. var size int64 var convErr error if DefaultImageFormat == FormatErofs { @@ -233,15 +242,13 @@ func (m *manager) installLayerArtifact(ctx context.Context, desc layerDescriptor } type unpackStats struct { - entries int unpackedBytes int64 diffID string - explicitDirs map[string]struct{} } // unpackCachedLayer locates desc's blob in the shared OCI cache, unpacks it // into dest, and verifies the diff ID when the descriptor carries one. -func unpackCachedLayer(ctx context.Context, cacheDir string, desc layerDescriptor, dest string) (*unpackStats, error) { +func unpackCachedLayer(ctx context.Context, cacheDir string, desc layerDescriptor, dest string, onDisk layer.OnDiskFormat) (*unpackStats, error) { layerHex := strings.TrimPrefix(desc.Digest, "sha256:") if err := paths.ValidatePathComponent(layerHex); err != nil { return nil, fmt.Errorf("invalid layer digest: %s", desc.Digest) @@ -253,7 +260,7 @@ func unpackCachedLayer(ctx context.Context, cacheDir string, desc layerDescripto } return nil, fmt.Errorf("stat layer blob: %w", err) } - stats, err := unpackLayerBlob(ctx, blobPath, desc.MediaType, dest) + stats, err := unpackLayerBlob(ctx, blobPath, desc.MediaType, dest, onDisk) if err != nil { return nil, fmt.Errorf("unpack layer %s: %w", desc.Digest, err) } @@ -263,11 +270,10 @@ func unpackCachedLayer(ctx context.Context, cacheDir string, desc layerDescripto return stats, nil } -// unpackLayerBlob extracts one compressed layer blob into dest while preserving -// whiteout marker files. It intentionally does not use umoci's layer unpacker: -// umoci consumes whiteouts while this store must retain them for composition. -// Paths are confined to dest. -func unpackLayerBlob(ctx context.Context, blobPath, mediaType, dest string) (*unpackStats, error) { +// unpackLayerBlob extracts one compressed layer blob into dest with umoci, +// which confines every entry to dest and interprets whiteouts per onDisk. +// The decompressed stream is hashed for the diff ID and capped in size. +func unpackLayerBlob(ctx context.Context, blobPath, mediaType, dest string, onDisk layer.OnDiskFormat) (*unpackStats, error) { if err := os.MkdirAll(dest, 0755); err != nil { return nil, fmt.Errorf("create extraction root: %w", err) } @@ -282,133 +288,23 @@ func unpackLayerBlob(ctx context.Context, blobPath, mediaType, dest string) (*un return nil, err } defer closer.Close() - reader = contextReader{ctx: ctx, reader: reader} hash := sha256.New() - stats := &unpackStats{explicitDirs: make(map[string]struct{})} - // Directory metadata is re-applied after extraction, once children - // exist, so tar directory mtimes are not overwritten by later writes. - pendingDirs := make([]pendingDir, 0) - pendingHardlinks := make([]pendingHardlink, 0) - limitedReader := &io.LimitedReader{R: reader, N: maxLayerUnpackedBytes + 1} - hashedReader := io.TeeReader(limitedReader, hash) - tr := tar.NewReader(hashedReader) - for { - header, err := tr.Next() - if err == io.EOF { - break - } - if err != nil { - return nil, fmt.Errorf("read tar entry: %w", err) - } - - target, err := safeJoin(dest, header.Name) - if err != nil { - return nil, err - } - if stats.entries >= maxLayerEntries { - return nil, fmt.Errorf("layer exceeds maximum entry count of %d", maxLayerEntries) - } - if header.Size > maxLayerUnpackedBytes-stats.unpackedBytes { - return nil, fmt.Errorf("layer exceeds maximum unpacked size of %d bytes", maxLayerUnpackedBytes) - } - stats.entries++ - - base := filepath.Base(header.Name) - if strings.HasPrefix(base, whiteoutPrefix) && base != opaqueWhiteout { - targetName := strings.TrimPrefix(base, whiteoutPrefix) - if targetName == "" || targetName == "." || targetName == ".." { - return nil, fmt.Errorf("invalid whiteout entry: %s", header.Name) - } - } - - if header.Typeflag == tar.TypeDir { - stats.explicitDirs[filepath.Clean(header.Name)] = struct{}{} - pendingDirs = append(pendingDirs, pendingDir{target: target, header: header}) - } - if header.Typeflag == tar.TypeLink { - // Hardlinks may reference entries that appear later in the tar, so - // resolve them after extracting all non-link entries. - pendingHardlinks = append(pendingHardlinks, pendingHardlink{target: target, linkname: header.Linkname}) - } else if err := extractTarEntry(tr, header, dest, target); err != nil { - return nil, fmt.Errorf("extract %s: %w", header.Name, err) - } - if header.Typeflag == tar.TypeReg || header.Typeflag == tar.TypeGNUSparse { - stats.unpackedBytes += header.Size - } + limited := &io.LimitedReader{R: contextReader{ctx: ctx, reader: reader}, N: maxLayerUnpackedBytes + 1} + hashed := io.TeeReader(limited, hash) + if err := layer.UnpackLayer(dest, hashed, &layer.UnpackOptions{OnDiskFormat: onDisk}); err != nil { + return nil, err } - if _, err := io.Copy(io.Discard, hashedReader); err != nil { + if _, err := io.Copy(io.Discard, hashed); err != nil { return nil, fmt.Errorf("drain layer: %w", err) } - if limitedReader.N == 0 { + if limited.N == 0 { return nil, fmt.Errorf("layer exceeds maximum unpacked size of %d bytes", maxLayerUnpackedBytes) } - if err := resolveHardlinks(dest, pendingHardlinks); err != nil { - return nil, err - } - for i := len(pendingDirs) - 1; i >= 0; i-- { - dir := pendingDirs[i] - if err := applyTarMetadata(dir.target, dir.header); err != nil { - return nil, fmt.Errorf("restore dir metadata %s: %w", dir.target, err) - } - } - stats.diffID = fmt.Sprintf("sha256:%x", hash.Sum(nil)) - return stats, nil -} - -type pendingDir struct { - target string - header *tar.Header -} - -type pendingHardlink struct { - target string - linkname string - linkTarget string -} - -func resolveHardlinks(root string, pending []pendingHardlink) error { - waiting := make(map[string][]pendingHardlink) - ready := make([]pendingHardlink, 0, len(pending)) - for _, link := range pending { - linkname := filepath.Clean(link.linkname) - if filepath.IsAbs(linkname) { - linkname = strings.TrimPrefix(linkname, string(filepath.Separator)) - } - linkTarget, err := safeJoin(root, linkname) - if err != nil { - return err - } - link.linkTarget = linkTarget - if _, err := os.Lstat(linkTarget); err == nil { - ready = append(ready, link) - } else if os.IsNotExist(err) { - waiting[linkTarget] = append(waiting[linkTarget], link) - } else { - return err - } - } - - resolved := 0 - for len(ready) > 0 { - link := ready[0] - ready = ready[1:] - if err := prepareTarTarget(link.target); err != nil { - return err - } - if err := os.Link(link.linkTarget, link.target); err != nil { - return fmt.Errorf("create hardlink %s -> %s: %w", link.target, link.linkname, err) - } - resolved++ - ready = append(ready, waiting[link.target]...) - delete(waiting, link.target) - } - if resolved != len(pending) { - for _, links := range waiting { - return fmt.Errorf("hardlink target not found for %s", links[0].linkname) - } - } - return nil + return &unpackStats{ + unpackedBytes: maxLayerUnpackedBytes + 1 - limited.N, + diffID: fmt.Sprintf("sha256:%x", hash.Sum(nil)), + }, nil } // decompressLayer wraps the blob in the reader for its layer media type. Both @@ -445,260 +341,40 @@ func (c multiCloser) Close() error { return firstErr } -// confineToRoot lexically resolves an entry name inside root, rejecting -// absolute names and parent traversal. root must be an existing directory -// that is not itself a symlink. -func confineToRoot(root, name string) (string, error) { - root = filepath.Clean(root) - rootInfo, err := os.Lstat(root) - if err != nil { - return "", fmt.Errorf("inspect extraction root: %w", err) - } - if !rootInfo.IsDir() || rootInfo.Mode()&os.ModeSymlink != 0 { - return "", fmt.Errorf("extraction root is not a directory: %s", root) - } - if filepath.IsAbs(name) { - return "", fmt.Errorf("tar entry escapes root: %s", name) - } - target := filepath.Join(root, name) - if !pathWithinRoot(root, target) { - return "", fmt.Errorf("tar entry escapes root: %s", name) - } - return target, nil -} - -// safeJoin resolves a tar entry inside root and rejects symlinked parents -// while extracting a layer. This prevents one tar entry from changing where -// a later entry is written. -func safeJoin(root, name string) (string, error) { - target, err := confineToRoot(root, name) - if err != nil { - return "", err - } - root = filepath.Clean(root) - if target == root { - return root, nil +// removePath removes a tree that may contain read-only directories restored +// from layer metadata. +func removePath(path string) error { + if err := makeTreeWritable(path); err != nil && !os.IsNotExist(err) { + return err } - for parent := filepath.Dir(target); parent != root; parent = filepath.Dir(parent) { - info, err := os.Lstat(parent) - if err != nil { - if os.IsNotExist(err) { - continue - } - return "", fmt.Errorf("inspect tar entry parent: %w", err) - } - if info.Mode()&os.ModeSymlink != 0 { - return "", fmt.Errorf("tar entry traverses symlink: %s", name) - } - if !info.IsDir() { - return "", fmt.Errorf("tar entry parent is not a directory: %s", parent) - } + if err := os.RemoveAll(path); err != nil && !os.IsNotExist(err) { + return err } - return target, nil + return nil } -// safeJoinForComposition resolves an entry through existing parent symlinks, -// interpreting absolute link targets relative to the image root. -func safeJoinForComposition(root, name string) (string, error) { - target, err := confineToRoot(root, name) - if err != nil { - return "", err - } - root = filepath.Clean(root) - if target == root { - return root, nil - } - resolvedParent, err := resolveLayerPath(root, filepath.Dir(target), 0) +func makeTreeWritable(path string) error { + info, err := os.Lstat(path) if err != nil { - return "", fmt.Errorf("inspect tar entry parent: %w", err) - } - return filepath.Join(resolvedParent, filepath.Base(target)), nil -} - -func resolveLayerPath(root, path string, depth int) (string, error) { - if depth > 40 { - return "", fmt.Errorf("too many symlinks") + return err } - root = filepath.Clean(root) - path = filepath.Clean(path) - rel, err := filepath.Rel(root, path) - if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { - return "", fmt.Errorf("path escapes root") + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return nil } - if rel == "." { - return root, nil + if err := os.Chmod(path, info.Mode().Perm()|0700); err != nil { + return err } - parts := strings.Split(rel, string(filepath.Separator)) - current := root - for i, part := range parts { - candidate := filepath.Join(current, part) - info, err := os.Lstat(candidate) + return filepath.WalkDir(path, func(p string, entry fs.DirEntry, err error) error { if err != nil { - if os.IsNotExist(err) { - return filepath.Join(candidate, filepath.Join(parts[i+1:]...)), nil - } - return "", err + return err } - if info.Mode()&os.ModeSymlink != 0 { - link, err := os.Readlink(candidate) + if entry.IsDir() && entry.Type()&os.ModeSymlink == 0 { + info, err := entry.Info() if err != nil { - return "", err - } - var linkTarget string - if filepath.IsAbs(link) { - linkTarget = filepath.Join(root, strings.TrimPrefix(link, string(filepath.Separator))) - } else { - linkTarget = filepath.Join(filepath.Dir(candidate), link) + return err } - linkTarget = filepath.Clean(linkTarget) - if !pathWithinRoot(root, linkTarget) { - return "", fmt.Errorf("symlink target escapes root") - } - return resolveLayerPath(root, filepath.Join(linkTarget, filepath.Join(parts[i+1:]...)), depth+1) - } - if i < len(parts)-1 && !info.IsDir() { - return "", fmt.Errorf("path parent is not a directory: %s", candidate) - } - current = candidate - } - return current, nil -} - -func pathWithinRoot(root, path string) bool { - rel, err := filepath.Rel(filepath.Clean(root), filepath.Clean(path)) - return err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) -} - -func resolveCompositionDirTarget(root, target string) (string, error) { - resolved, err := resolveLayerPath(root, target, 0) - if err == nil { - return resolved, nil - } - info, statErr := os.Lstat(target) - if statErr == nil && info.Mode()&os.ModeSymlink != 0 { - if removeErr := removePath(target); removeErr != nil { - return "", removeErr + return os.Chmod(p, info.Mode().Perm()|0700) } - return target, nil - } - return "", err -} - -// validateSymlinkTarget permits absolute targets because OCI images may use -// them and the image filesystem must preserve their link text. Extraction -// still rejects symlink traversal for later entries, while composition copies -// symlink text without following it. -func validateSymlinkTarget(root, target, linkname string) error { - if filepath.IsAbs(linkname) { - return nil - } - if !pathWithinRoot(root, filepath.Join(filepath.Dir(target), linkname)) { - return fmt.Errorf("symlink target escapes root: %s", linkname) - } - return nil -} - -func extractTarEntry(tr *tar.Reader, header *tar.Header, root, target string) error { - switch header.Typeflag { - case tar.TypeDir: - return extractTarDir(target) - case tar.TypeReg, tar.TypeGNUSparse: - return extractTarFile(tr, target, header) - case tar.TypeSymlink: - return extractTarSymlink(root, target, header) - case tar.TypeChar, tar.TypeBlock: - return extractTarDevice(target, header) - case tar.TypeFifo: - return extractTarFIFO(target, header) - default: return nil - } -} - -func prepareTarTarget(target string) error { - if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { - return err - } - return removePath(target) -} - -func extractTarDir(target string) error { - if info, err := os.Lstat(target); err == nil && !info.IsDir() { - if err := removePath(target); err != nil { - return err - } - } - return os.MkdirAll(target, 0755) -} - -func extractTarFile(tr *tar.Reader, target string, header *tar.Header) error { - if err := prepareTarTarget(target); err != nil { - return err - } - file, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC|syscall.O_NOFOLLOW, 0644) - if err != nil { - return err - } - if _, err := io.Copy(file, tr); err != nil { - _ = file.Close() - return err - } - if err := file.Close(); err != nil { - return err - } - return applyTarMetadata(target, header) -} - -func extractTarSymlink(root, target string, header *tar.Header) error { - if err := validateSymlinkTarget(root, target, header.Linkname); err != nil { - return err - } - if err := prepareTarTarget(target); err != nil { - return err - } - if err := os.Symlink(header.Linkname, target); err != nil { - return err - } - return applyTarMetadata(target, header) -} - -func extractTarDevice(target string, header *tar.Header) error { - if err := prepareTarTarget(target); err != nil { - return err - } - mode := uint32(syscall.S_IFCHR) - if header.Typeflag == tar.TypeBlock { - mode = uint32(syscall.S_IFBLK) - } - dev := int(unix.Mkdev(uint32(header.Devmajor), uint32(header.Devminor))) - if err := mknodWithRootlessFallback(target, mode|uint32(header.FileInfo().Mode().Perm()), dev); err != nil { - return err - } - return applyTarMetadata(target, header) -} - -func extractTarFIFO(target string, header *tar.Header) error { - if err := prepareTarTarget(target); err != nil { - return err - } - if err := syscall.Mkfifo(target, uint32(header.FileInfo().Mode().Perm())); err != nil { - return err - } - return applyTarMetadata(target, header) -} - -func applyTarMetadata(path string, header *tar.Header) error { - mtime := header.ModTime - if mtime.IsZero() { - mtime = time.Now() - } - return applyEntryMetadata(path, entryMetadata{ - uid: header.Uid, - gid: header.Gid, - hasOwner: true, - mode: header.FileInfo().Mode(), - symlink: header.Typeflag == tar.TypeSymlink, - mtime: mtime, - xattrs: tarXattrs(header), }) } diff --git a/lib/images/layer_artifact_test.go b/lib/images/layer_artifact_test.go index eeff21fa5..528c91abd 100644 --- a/lib/images/layer_artifact_test.go +++ b/lib/images/layer_artifact_test.go @@ -7,29 +7,23 @@ import ( "context" "crypto/sha256" "encoding/json" - "errors" "fmt" - "io" - "io/fs" "os" "os/exec" "path/filepath" - "strings" - "syscall" "testing" - "time" - - "golang.org/x/sys/unix" gcr "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/layout" "github.com/google/go-containerregistry/pkg/v1/mutate" - "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" ) +const testTarGzMediaType = "application/vnd.oci.image.layer.v1.tar+gzip" + // writeLayerTestLayout writes img into the shared OCI cache of p tagged with // the image's digest, mirroring pullToOCILayout. func writeLayerTestLayout(t *testing.T, p *paths.Paths, img gcr.Image) { @@ -58,6 +52,84 @@ func layerDescFromImage(t *testing.T, img gcr.Image, index int) layerDescriptor } } +// tarGz builds a gzipped tar from the entries written by fn. +func tarGz(t *testing.T, fn func(tw *tar.Writer)) []byte { + t.Helper() + var buf bytes.Buffer + gzw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gzw) + fn(tw) + require.NoError(t, tw.Close()) + require.NoError(t, gzw.Close()) + return buf.Bytes() +} + +func writeTarEntry(t *testing.T, tw *tar.Writer, header *tar.Header, content string) { + t.Helper() + if header.Typeflag == tar.TypeReg { + header.Size = int64(len(content)) + } + require.NoError(t, tw.WriteHeader(header)) + if content != "" { + _, err := tw.Write([]byte(content)) + require.NoError(t, err) + } +} + +func fileEntry(name, content string) func(*testing.T, *tar.Writer) { + return func(t *testing.T, tw *tar.Writer) { + writeTarEntry(t, tw, &tar.Header{Name: name, Typeflag: tar.TypeReg, Mode: 0644}, content) + } +} + +func dirEntry(name string) func(*testing.T, *tar.Writer) { + return func(t *testing.T, tw *tar.Writer) { + writeTarEntry(t, tw, &tar.Header{Name: name, Typeflag: tar.TypeDir, Mode: 0755}, "") + } +} + +func symlinkEntry(name, target string) func(*testing.T, *tar.Writer) { + return func(t *testing.T, tw *tar.Writer) { + writeTarEntry(t, tw, &tar.Header{Name: name, Typeflag: tar.TypeSymlink, Linkname: target}, "") + } +} + +func hardlinkEntry(name, target string) func(*testing.T, *tar.Writer) { + return func(t *testing.T, tw *tar.Writer) { + writeTarEntry(t, tw, &tar.Header{Name: name, Typeflag: tar.TypeLink, Linkname: target}, "") + } +} + +// writeLayerBlob writes a gzipped tar layer to a file and returns its path. +func writeLayerBlob(t *testing.T, dir, name string, entries ...func(*testing.T, *tar.Writer)) string { + t.Helper() + data := tarGz(t, func(tw *tar.Writer) { + for _, entry := range entries { + entry(t, tw) + } + }) + path := filepath.Join(dir, name) + require.NoError(t, os.WriteFile(path, data, 0644)) + return path +} + +// unpackInto applies a layer blob onto dest with compose semantics. +func unpackInto(t *testing.T, blobPath, dest string) *unpackStats { + t.Helper() + stats, err := unpackLayerBlob(context.Background(), blobPath, testTarGzMediaType, dest, composeOnDiskFormat()) + require.NoError(t, err) + return stats +} + +func requireNoWhiteoutMarkers(t *testing.T, root string) { + t.Helper() + require.NoError(t, filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { + require.NoError(t, err) + require.NotContains(t, entry.Name(), whiteoutPrefix, "whiteout marker leaked into tree") + return nil + })) +} + func TestMaterializeLayerArtifact(t *testing.T) { if _, err := exec.LookPath("mkfs.erofs"); err != nil { t.Skip("mkfs.erofs not available") @@ -78,15 +150,12 @@ func TestMaterializeLayerArtifact(t *testing.T) { require.Equal(t, layerArtifactFormat(), record.Format) require.Greater(t, record.SizeBytes, int64(0)) require.Greater(t, record.UnpackedBytes, int64(0)) - require.Greater(t, record.Entries, 0) layerHex := desc.Digest[len("sha256:"):] - _, err = os.Stat(p.ImageLayerArtifactForFormat(layerHex, layerArtifactFormat())) + artifactInfo, err := os.Stat(p.ImageLayerArtifactForFormat(layerHex, layerArtifactFormat())) require.NoError(t, err, "layer.erofs must be installed") // A second materialization reuses the existing artifact. - artifactInfo, err := os.Stat(p.ImageLayerArtifactForFormat(layerHex, layerArtifactFormat())) - require.NoError(t, err) reused, err := m.materializeLayerArtifact(context.Background(), desc) require.NoError(t, err) require.True(t, record.CreatedAt.Equal(reused.CreatedAt), "reuse must return the stored record") @@ -138,263 +207,155 @@ func TestMaterializeLayerArtifactMissingBlob(t *testing.T) { _, err := m.materializeLayerArtifact(context.Background(), layerDescriptor{ Digest: "sha256:abababababababababababababababababababababababababababababababab", - MediaType: "application/vnd.oci.image.layer.v1.tar+gzip", + MediaType: testTarGzMediaType, }) require.ErrorContains(t, err, "missing from oci cache") } -// whiteoutLayer builds a gzipped tar layer exercising whiteouts: a plain file, -// a whiteout marker, an opaque directory marker, and a whiteout+recreate pair. -func whiteoutLayer(t *testing.T) gcr.Layer { - t.Helper() - - var buf bytes.Buffer - gzw := gzip.NewWriter(&buf) - tw := tar.NewWriter(gzw) - - writeEntry := func(header *tar.Header, content string) { - require.NoError(t, tw.WriteHeader(header)) - if content != "" { - _, err := tw.Write([]byte(content)) - require.NoError(t, err) - } - } - writeEntry(&tar.Header{Name: "keep.txt", Typeflag: tar.TypeReg, Mode: 0644, Size: 4}, "keep") - writeEntry(&tar.Header{Name: "gone/", Typeflag: tar.TypeDir, Mode: 0755}, "") - writeEntry(&tar.Header{Name: "gone/.wh.deleted.txt", Typeflag: tar.TypeReg, Mode: 0644, Size: 0}, "") - writeEntry(&tar.Header{Name: "opq/", Typeflag: tar.TypeDir, Mode: 0755}, "") - writeEntry(&tar.Header{Name: "opq/.wh..wh..opq", Typeflag: tar.TypeReg, Mode: 0644, Size: 0}, "") - writeEntry(&tar.Header{Name: "opq/fresh.txt", Typeflag: tar.TypeReg, Mode: 0644, Size: 5}, "fresh") - writeEntry(&tar.Header{Name: "added/", Typeflag: tar.TypeDir, Mode: 0755}, "") - writeEntry(&tar.Header{Name: "added/.wh.foo", Typeflag: tar.TypeReg, Mode: 0644, Size: 0}, "") - writeEntry(&tar.Header{Name: "added/foo", Typeflag: tar.TypeReg, Mode: 0644, Size: 3}, "new") - - require.NoError(t, tw.Close()) - require.NoError(t, gzw.Close()) - - data := buf.Bytes() - layer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { - return io.NopCloser(bytes.NewReader(data)), nil - }) - require.NoError(t, err) - return layer -} - -func TestMaterializeLayerDoesNotPersistWhiteoutInventory(t *testing.T) { - if _, err := exec.LookPath("mkfs.erofs"); err != nil { - t.Skip("mkfs.erofs not available") +// TestUnpackLayerBlobArtifactFormatKeepsWhiteouts checks that the artifact +// extraction format records deletions in overlayfs form: a 0:0 character +// device for a whiteout and the opaque xattr for an opaque directory. +func TestUnpackLayerBlobArtifactFormatKeepsWhiteouts(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("overlayfs whiteouts need mknod and trusted xattrs") } - - p := paths.New(t.TempDir()) - img, err := mutate.AppendLayers(empty.Image, whiteoutLayer(t)) - require.NoError(t, err) - writeLayerTestLayout(t, p, img) - - desc := layerDescFromImage(t, img, 0) - m := &manager{paths: p} - - _, err = m.materializeLayerArtifact(context.Background(), desc) - require.NoError(t, err) - - data, err := os.ReadFile(p.ImageLayerRecordForFormat(desc.Digest[len("sha256:"):], layerArtifactFormat())) - require.NoError(t, err) - require.NotContains(t, string(data), "whiteouts") -} - -func TestApplyLayerTreeCopiesRestrictiveFiles(t *testing.T) { root := t.TempDir() - targetDir := filepath.Join(root, "target") - layerDir := filepath.Join(root, "layer") - file := filepath.Join(layerDir, "secret.txt") - - require.NoError(t, os.MkdirAll(layerDir, 0755)) - require.NoError(t, os.WriteFile(file, []byte("secret"), 0644)) - require.NoError(t, os.Chmod(file, 0000)) - - require.NoError(t, applyLayerTree(layerDir, targetDir, nil)) - - target := filepath.Join(targetDir, "secret.txt") - info, err := os.Stat(target) - require.NoError(t, err) - require.Equal(t, os.FileMode(0000), info.Mode().Perm()) - require.NoError(t, os.Chmod(target, 0644)) - data, err := os.ReadFile(target) + blob := writeLayerBlob(t, root, "layer.tar.gz", + fileEntry("keep.txt", "keep"), + dirEntry("gone/"), + fileEntry("gone/.wh.deleted.txt", ""), + dirEntry("opq/"), + fileEntry("opq/.wh..wh..opq", ""), + fileEntry("opq/fresh.txt", "fresh"), + ) + dest := filepath.Join(root, "dest") + _, err := unpackLayerBlob(context.Background(), blob, testTarGzMediaType, dest, layerArtifactOnDiskFormat()) require.NoError(t, err) - require.Equal(t, "secret", string(data)) -} - -func TestRestoreDirectoryModesSkipsReplacedSymlink(t *testing.T) { - root := t.TempDir() - dir := filepath.Join(root, "dir") - target := filepath.Join(root, "target") - require.NoError(t, os.Mkdir(dir, 0750)) - require.NoError(t, os.WriteFile(target, []byte("target"), 0644)) - require.NoError(t, os.Remove(dir)) - require.NoError(t, os.Symlink("target", dir)) - require.NoError(t, restoreDirectoryModes(map[string]fs.FileMode{dir: 0750})) + var stat unix.Stat_t + require.NoError(t, unix.Lstat(filepath.Join(dest, "gone", "deleted.txt"), &stat)) + require.Equal(t, uint32(unix.S_IFCHR), stat.Mode&unix.S_IFMT, "whiteout must be a character device") + require.Equal(t, uint64(0), uint64(stat.Rdev), "whiteout device must be 0:0") - info, err := os.Stat(target) + value := make([]byte, 8) + n, err := unix.Lgetxattr(filepath.Join(dest, "opq"), "trusted.overlay.opaque", value) require.NoError(t, err) - require.Equal(t, os.FileMode(0644), info.Mode().Perm()) + require.Equal(t, "y", string(value[:n])) + requireNoWhiteoutMarkers(t, dest) } -func TestApplyLayerTreeWhiteoutSemantics(t *testing.T) { +func TestUnpackLayerBlobAppliesWhiteoutsAcrossLayers(t *testing.T) { root := t.TempDir() - targetDir := filepath.Join(root, "target") - layerDir := filepath.Join(root, "layer") - - // Lower state contributed by earlier layers. - require.NoError(t, os.MkdirAll(filepath.Join(targetDir, "opqdir"), 0755)) - require.NoError(t, os.MkdirAll(filepath.Join(targetDir, "swapdir"), 0755)) - require.NoError(t, os.WriteFile(filepath.Join(targetDir, "victim.txt"), []byte("old"), 0644)) - require.NoError(t, os.MkdirAll(filepath.Join(targetDir, "removedir"), 0755)) - require.NoError(t, os.WriteFile(filepath.Join(targetDir, "removedir", "inner.txt"), []byte("old"), 0644)) - require.NoError(t, os.WriteFile(filepath.Join(targetDir, "keep.txt"), []byte("old"), 0644)) - require.NoError(t, os.WriteFile(filepath.Join(targetDir, "opqdir", "stale.txt"), []byte("stale"), 0644)) - require.NoError(t, os.WriteFile(filepath.Join(targetDir, "swapdir", "inner.txt"), []byte("inner"), 0644)) - - // Layer: whiteout victim.txt, opaque opqdir, replace swapdir with a file, - // and whiteout-then-recreate added/foo within the same layer. - require.NoError(t, os.MkdirAll(filepath.Join(layerDir, "opqdir"), 0755)) - require.NoError(t, os.MkdirAll(filepath.Join(layerDir, "added"), 0755)) - require.NoError(t, os.WriteFile(filepath.Join(layerDir, ".wh.victim.txt"), nil, 0644)) - require.NoError(t, os.WriteFile(filepath.Join(layerDir, ".wh.removedir"), nil, 0644)) - require.NoError(t, os.WriteFile(filepath.Join(layerDir, "keep.txt"), []byte("new"), 0644)) - require.NoError(t, os.WriteFile(filepath.Join(layerDir, "opqdir", ".wh..wh..opq"), nil, 0644)) - require.NoError(t, os.WriteFile(filepath.Join(layerDir, "opqdir", "fresh.txt"), []byte("fresh"), 0644)) - require.NoError(t, os.WriteFile(filepath.Join(layerDir, "swapdir"), []byte("now a file"), 0644)) - require.NoError(t, os.WriteFile(filepath.Join(layerDir, "added", ".wh.foo"), nil, 0644)) - require.NoError(t, os.WriteFile(filepath.Join(layerDir, "added", "foo"), []byte("new"), 0644)) - - require.NoError(t, applyLayerTree(layerDir, targetDir, nil)) - - // Whiteout removed the lower entry. - _, err := os.Lstat(filepath.Join(targetDir, "victim.txt")) - require.True(t, os.IsNotExist(err), "whiteout must delete the lower entry") - _, err = os.Lstat(filepath.Join(targetDir, "removedir")) - require.True(t, os.IsNotExist(err), "directory whiteout must delete the complete lower directory") + base := writeLayerBlob(t, root, "base.tar.gz", + dirEntry("etc/"), + fileEntry("etc/config.txt", "original"), + dirEntry("data/"), + fileEntry("data/old.txt", "stale"), + dirEntry("replacedir/"), + fileEntry("replacedir/inner.txt", "inner"), + fileEntry("added/foo", "old"), + ) + top := writeLayerBlob(t, root, "top.tar.gz", + fileEntry("etc/.wh.config.txt", ""), + fileEntry("data/.wh..wh..opq", ""), + fileEntry("data/new.txt", "new"), + fileEntry("replacedir", "now a file"), + fileEntry("added/.wh.foo", ""), + fileEntry("added/foo", "new"), + ) + dest := filepath.Join(root, "dest") + unpackInto(t, base, dest) + unpackInto(t, top, dest) - // Regular file replacement. - data, err := os.ReadFile(filepath.Join(targetDir, "keep.txt")) + _, err := os.Lstat(filepath.Join(dest, "etc", "config.txt")) + require.True(t, os.IsNotExist(err), "whiteout must delete the lower entry") + _, err = os.Lstat(filepath.Join(dest, "data", "old.txt")) + require.True(t, os.IsNotExist(err), "opaque marker must mask lower contents") + data, err := os.ReadFile(filepath.Join(dest, "data", "new.txt")) require.NoError(t, err) require.Equal(t, "new", string(data)) - - // Opaque directory: stale content gone, layer content present. - _, err = os.Lstat(filepath.Join(targetDir, "opqdir", "stale.txt")) - require.True(t, os.IsNotExist(err), "opaque dir must hide lower contents") - data, err = os.ReadFile(filepath.Join(targetDir, "opqdir", "fresh.txt")) + info, err := os.Lstat(filepath.Join(dest, "replacedir")) require.NoError(t, err) - require.Equal(t, "fresh", string(data)) - - // Directory replaced by a file. - info, err := os.Lstat(filepath.Join(targetDir, "swapdir")) - require.NoError(t, err) - require.False(t, info.IsDir()) - - // Whiteout followed by recreate in the same layer keeps the new entry. - data, err = os.ReadFile(filepath.Join(targetDir, "added", "foo")) + require.False(t, info.IsDir(), "file must replace lower directory") + data, err = os.ReadFile(filepath.Join(dest, "added", "foo")) require.NoError(t, err) - require.Equal(t, "new", string(data)) - - // Whiteout marker files never leak into the composed tree. - leaked := make([]string, 0) - require.NoError(t, filepath.WalkDir(targetDir, func(path string, entry fs.DirEntry, err error) error { - if err != nil { - return err - } - if strings.HasPrefix(entry.Name(), whiteoutPrefix) { - leaked = append(leaked, path) - } - return nil - })) - require.Empty(t, leaked) + require.Equal(t, "new", string(data), "whiteout then recreate in one layer keeps the new file") + requireNoWhiteoutMarkers(t, dest) } -func TestApplyLayerTreeReplacesSymlinkBeforeNestedWhiteout(t *testing.T) { +// TestUnpackLayerBlobReplacesDirectorySymlink checks the standard runtime +// behavior: a directory in an upper layer replaces a symlink from a lower one +// rather than writing through it. +func TestUnpackLayerBlobReplacesDirectorySymlink(t *testing.T) { root := t.TempDir() - targetDir := filepath.Join(root, "target") - layerDir := filepath.Join(root, "layer") - lowerDir := filepath.Join(root, "lower") - - require.NoError(t, os.MkdirAll(targetDir, 0755)) - require.NoError(t, os.MkdirAll(lowerDir, 0755)) - require.NoError(t, os.WriteFile(filepath.Join(lowerDir, "old.txt"), []byte("old"), 0644)) - require.NoError(t, os.Symlink("../lower", filepath.Join(targetDir, "replaced"))) - require.NoError(t, os.MkdirAll(filepath.Join(layerDir, "replaced"), 0755)) - require.NoError(t, os.WriteFile(filepath.Join(layerDir, "replaced", ".wh.old.txt"), nil, 0644)) - require.NoError(t, os.WriteFile(filepath.Join(layerDir, "replaced", "new.txt"), []byte("new"), 0644)) - - require.NoError(t, applyLayerTree(layerDir, targetDir, nil)) + base := writeLayerBlob(t, root, "base.tar.gz", + dirEntry("usr/"), + dirEntry("usr/bin/"), + fileEntry("usr/bin/sh", "sh"), + symlinkEntry("bin", "usr/bin"), + ) + top := writeLayerBlob(t, root, "top.tar.gz", + dirEntry("bin/"), + fileEntry("bin/tool", "tool"), + ) + dest := filepath.Join(root, "dest") + unpackInto(t, base, dest) + unpackInto(t, top, dest) - info, err := os.Lstat(filepath.Join(targetDir, "replaced")) + info, err := os.Lstat(filepath.Join(dest, "bin")) require.NoError(t, err) - require.True(t, info.IsDir()) - require.FileExists(t, filepath.Join(targetDir, "replaced", "new.txt")) - _, err = os.Lstat(filepath.Join(targetDir, "replaced", "old.txt")) - require.ErrorIs(t, err, os.ErrNotExist) + require.True(t, info.IsDir(), "upper directory must replace the lower symlink") + require.FileExists(t, filepath.Join(dest, "bin", "tool")) + require.FileExists(t, filepath.Join(dest, "usr", "bin", "sh")) + require.NoFileExists(t, filepath.Join(dest, "usr", "bin", "tool")) } -func TestApplyLayerTreePreservesUpperDirectoryMode(t *testing.T) { +func TestUnpackLayerBlobConfinesSymlinkTraversal(t *testing.T) { root := t.TempDir() - targetDir := filepath.Join(root, "target") - layerDir := filepath.Join(root, "layer") - - require.NoError(t, os.MkdirAll(filepath.Join(targetDir, "etc"), 0755)) - require.NoError(t, os.MkdirAll(filepath.Join(layerDir, "etc"), 0750)) - require.NoError(t, os.WriteFile(filepath.Join(layerDir, "etc", "new.txt"), []byte("new"), 0644)) - - require.NoError(t, applyLayerTree(layerDir, targetDir, nil)) + blob := writeLayerBlob(t, root, "layer.tar.gz", + symlinkEntry("link", "../../escape"), + fileEntry("link/pwned", "x"), + ) + dest := filepath.Join(root, "dest") + unpackInto(t, blob, dest) - info, err := os.Stat(filepath.Join(targetDir, "etc")) - require.NoError(t, err) - require.Equal(t, os.FileMode(0750), info.Mode().Perm()) + require.NoFileExists(t, filepath.Join(root, "escape", "pwned")) + require.FileExists(t, filepath.Join(dest, "escape", "pwned"), "escaping link must be resolved inside the root") } -func TestApplyLayerTreeWhiteoutRemovesReadOnlyDirectory(t *testing.T) { +func TestUnpackLayerBlobPreservesHardlinks(t *testing.T) { root := t.TempDir() - targetDir := filepath.Join(root, "target") - layerDir := filepath.Join(root, "layer") - - require.NoError(t, os.MkdirAll(filepath.Join(targetDir, "gone"), 0755)) - require.NoError(t, os.WriteFile(filepath.Join(targetDir, "gone", "inner.txt"), []byte("old"), 0644)) - require.NoError(t, os.Chmod(filepath.Join(targetDir, "gone"), 0555)) - require.NoError(t, os.MkdirAll(layerDir, 0755)) - require.NoError(t, os.WriteFile(filepath.Join(layerDir, ".wh.gone"), nil, 0644)) - - require.NoError(t, applyLayerTree(layerDir, targetDir, nil)) - _, err := os.Lstat(filepath.Join(targetDir, "gone")) - require.ErrorIs(t, err, os.ErrNotExist) + blob := writeLayerBlob(t, root, "layer.tar.gz", + fileEntry("a", "shared"), + hardlinkEntry("b", "a"), + ) + dest := filepath.Join(root, "dest") + unpackInto(t, blob, dest) + + a, err := os.Stat(filepath.Join(dest, "a")) + require.NoError(t, err) + b, err := os.Stat(filepath.Join(dest, "b")) + require.NoError(t, err) + require.True(t, os.SameFile(a, b)) } func TestUnpackLayerBlobContextHonorsCancellation(t *testing.T) { root := t.TempDir() - blobPath := filepath.Join(root, "layer.tar") - var buf bytes.Buffer - tw := tar.NewWriter(&buf) - require.NoError(t, tw.WriteHeader(&tar.Header{Name: "file", Typeflag: tar.TypeReg, Mode: 0644, Size: 1})) - _, err := tw.Write([]byte("x")) - require.NoError(t, err) - require.NoError(t, tw.Close()) - require.NoError(t, os.WriteFile(blobPath, buf.Bytes(), 0644)) + blob := writeLayerBlob(t, root, "layer.tar.gz", fileEntry("file", "x")) ctx, cancel := context.WithCancel(context.Background()) cancel() - _, err = unpackLayerBlob(ctx, blobPath, "application/vnd.oci.image.layer.v1.tar", filepath.Join(root, "dest")) + _, err := unpackLayerBlob(ctx, blob, testTarGzMediaType, filepath.Join(root, "dest"), composeOnDiskFormat()) require.ErrorIs(t, err, context.Canceled) } func TestUnpackLayerBlobIncludesTrailingTarPaddingInDiffID(t *testing.T) { root := t.TempDir() - blobPath := filepath.Join(root, "layer.tar.gz") - var tarData bytes.Buffer tw := tar.NewWriter(&tarData) - require.NoError(t, tw.WriteHeader(&tar.Header{Name: "file", Typeflag: tar.TypeReg, Mode: 0644, Size: 1})) - _, err := tw.Write([]byte("x")) - require.NoError(t, err) + writeTarEntry(t, tw, &tar.Header{Name: "file", Typeflag: tar.TypeReg, Mode: 0644}, "x") require.NoError(t, tw.Close()) - _, err = tarData.Write(make([]byte, 512)) + _, err := tarData.Write(make([]byte, 512)) require.NoError(t, err) var compressed bytes.Buffer @@ -402,311 +363,11 @@ func TestUnpackLayerBlobIncludesTrailingTarPaddingInDiffID(t *testing.T) { _, err = gzw.Write(tarData.Bytes()) require.NoError(t, err) require.NoError(t, gzw.Close()) - require.NoError(t, os.WriteFile(blobPath, compressed.Bytes(), 0644)) + blob := filepath.Join(root, "layer.tar.gz") + require.NoError(t, os.WriteFile(blob, compressed.Bytes(), 0644)) - stats, err := unpackLayerBlob(context.Background(), blobPath, "application/vnd.oci.image.layer.v1.tar+gzip", filepath.Join(root, "dest")) - require.NoError(t, err) + stats := unpackInto(t, blob, filepath.Join(root, "dest")) want := sha256.Sum256(tarData.Bytes()) require.Equal(t, "sha256:"+fmt.Sprintf("%x", want), stats.diffID) -} - -func TestResolveHardlinksResolvesDependencyChain(t *testing.T) { - root := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(root, "source"), []byte("content"), 0644)) - - pending := []pendingHardlink{ - {target: filepath.Join(root, "third"), linkname: "second"}, - {target: filepath.Join(root, "second"), linkname: "first"}, - {target: filepath.Join(root, "first"), linkname: "source"}, - } - require.NoError(t, resolveHardlinks(root, pending)) - - source, err := os.Stat(filepath.Join(root, "source")) - require.NoError(t, err) - third, err := os.Stat(filepath.Join(root, "third")) - require.NoError(t, err) - require.True(t, os.SameFile(source, third)) -} - -func TestUnpackLayerBlobRejectsSymlinkTraversal(t *testing.T) { - root := t.TempDir() - blobPath := filepath.Join(root, "layer.tar.gz") - var buf bytes.Buffer - gzw := gzip.NewWriter(&buf) - tw := tar.NewWriter(gzw) - require.NoError(t, tw.WriteHeader(&tar.Header{Name: "link", Typeflag: tar.TypeSymlink, Linkname: "outside"})) - require.NoError(t, tw.WriteHeader(&tar.Header{Name: "link/escape", Typeflag: tar.TypeReg, Mode: 0644, Size: 1})) - _, err := tw.Write([]byte("x")) - require.NoError(t, err) - require.NoError(t, tw.Close()) - require.NoError(t, gzw.Close()) - require.NoError(t, os.WriteFile(blobPath, buf.Bytes(), 0644)) - - _, err = unpackLayerBlob(context.Background(), blobPath, "application/vnd.oci.image.layer.v1.tar+gzip", filepath.Join(root, "dest")) - require.ErrorContains(t, err, "symlink") -} - -func TestValidateSymlinkTargetAllowsAbsoluteTargets(t *testing.T) { - root := t.TempDir() - target := filepath.Join(root, "link") - require.NoError(t, validateSymlinkTarget(root, target, "/etc/resolv.conf")) -} - -func TestApplyLayerTreePreservesImplicitDirectoryMetadata(t *testing.T) { - root := t.TempDir() - targetDir := filepath.Join(root, "target") - layerDir := filepath.Join(root, "layer") - require.NoError(t, os.MkdirAll(filepath.Join(targetDir, "etc"), 0700)) - require.NoError(t, os.MkdirAll(filepath.Join(layerDir, "etc"), 0755)) - require.NoError(t, os.WriteFile(filepath.Join(layerDir, "etc", "app.conf"), []byte("config"), 0644)) - - require.NoError(t, applyLayerTree(layerDir, targetDir, map[string]struct{}{})) - - info, err := os.Stat(filepath.Join(targetDir, "etc")) - require.NoError(t, err) - require.Equal(t, os.FileMode(0700), info.Mode().Perm()) -} - -func TestApplyLayerTreeResolvesInRootSymlinkParents(t *testing.T) { - root := t.TempDir() - targetDir := filepath.Join(root, "target") - layerDir := filepath.Join(root, "layer") - require.NoError(t, os.MkdirAll(filepath.Join(targetDir, "usr", "bin"), 0755)) - require.NoError(t, os.Symlink("usr/bin", filepath.Join(targetDir, "bin"))) - require.NoError(t, os.MkdirAll(filepath.Join(layerDir, "bin"), 0755)) - require.NoError(t, os.WriteFile(filepath.Join(layerDir, "bin", "tool"), []byte("tool"), 0644)) - - require.NoError(t, applyLayerTree(layerDir, targetDir, nil)) - - data, err := os.ReadFile(filepath.Join(targetDir, "usr", "bin", "tool")) - require.NoError(t, err) - require.Equal(t, "tool", string(data)) - info, err := os.Lstat(filepath.Join(targetDir, "bin")) - require.NoError(t, err) - require.Equal(t, os.ModeSymlink, info.Mode()&os.ModeSymlink) -} - -func TestApplyLayerTreeSymlinksAndHardlinks(t *testing.T) { - root := t.TempDir() - targetDir := filepath.Join(root, "target") - layerDir := filepath.Join(root, "layer") - require.NoError(t, os.MkdirAll(targetDir, 0755)) - require.NoError(t, os.MkdirAll(layerDir, 0755)) - - // A symlink in the lower tree pointing at a file the new layer deletes: - // the symlink itself must be removed, never followed. - require.NoError(t, os.WriteFile(filepath.Join(targetDir, "real.txt"), []byte("real"), 0644)) - require.NoError(t, os.Symlink("real.txt", filepath.Join(targetDir, "alias"))) - - require.NoError(t, os.WriteFile(filepath.Join(layerDir, ".wh.alias"), nil, 0644)) - require.NoError(t, os.WriteFile(filepath.Join(layerDir, "a.txt"), []byte("content"), 0644)) - require.NoError(t, os.Link(filepath.Join(layerDir, "a.txt"), filepath.Join(layerDir, "b.txt"))) - require.NoError(t, os.Symlink("a.txt", filepath.Join(layerDir, "link-to-a"))) - - require.NoError(t, applyLayerTree(layerDir, targetDir, nil)) - - _, err := os.Lstat(filepath.Join(targetDir, "alias")) - require.True(t, os.IsNotExist(err), "symlink whiteout must remove the link itself") - _, err = os.Lstat(filepath.Join(targetDir, "real.txt")) - require.NoError(t, err, "symlink target must survive an unrelated whiteout") - - infoA, err := os.Stat(filepath.Join(targetDir, "a.txt")) - require.NoError(t, err) - infoB, err := os.Stat(filepath.Join(targetDir, "b.txt")) - require.NoError(t, err) - require.Equal(t, int64(7), infoA.Size()) - require.True(t, os.SameFile(infoA, infoB), "hardlinks within the layer must stay linked") - - linkTarget, err := os.Readlink(filepath.Join(targetDir, "link-to-a")) - require.NoError(t, err) - require.Equal(t, "a.txt", linkTarget) -} - -func TestCopyXattrs(t *testing.T) { - root := t.TempDir() - src := filepath.Join(root, "src") - dst := filepath.Join(root, "dst") - require.NoError(t, os.WriteFile(src, []byte("payload"), 0644)) - require.NoError(t, os.WriteFile(dst, []byte("payload"), 0644)) - - err := unix.Lsetxattr(src, "user.one", []byte("1"), 0) - if errors.Is(err, unix.ENOTSUP) || errors.Is(err, unix.EPERM) { - t.Skip("filesystem does not support user xattrs") - } - require.NoError(t, err) - require.NoError(t, unix.Lsetxattr(src, "user.two", []byte("22"), 0)) - - xattrs, err := readXattrs(src) - require.NoError(t, err) - require.NoError(t, applyXattrs(dst, xattrs)) - - for name, want := range map[string]string{"user.one": "1", "user.two": "22"} { - size, err := unix.Lgetxattr(dst, name, nil) - require.NoError(t, err, "xattr %s must be copied", name) - value := make([]byte, size) - n, err := unix.Lgetxattr(dst, name, value) - require.NoError(t, err) - require.Equal(t, want, string(value[:n])) - } -} - -func TestUnpackLayerBlobPreservesDirMtime(t *testing.T) { - root := t.TempDir() - blobPath := filepath.Join(root, "layer.tar") - dirTime := time.Now().Add(-time.Hour).Truncate(time.Second) - - var buf bytes.Buffer - tw := tar.NewWriter(&buf) - require.NoError(t, tw.WriteHeader(&tar.Header{Name: "d/", Typeflag: tar.TypeDir, Mode: 0755, ModTime: dirTime})) - require.NoError(t, tw.WriteHeader(&tar.Header{Name: "d/file.txt", Typeflag: tar.TypeReg, Mode: 0644, Size: 1})) - _, err := tw.Write([]byte("x")) - require.NoError(t, err) - require.NoError(t, tw.Close()) - require.NoError(t, os.WriteFile(blobPath, buf.Bytes(), 0644)) - - dest := filepath.Join(root, "dest") - _, err = unpackLayerBlob(context.Background(), blobPath, "application/vnd.oci.image.layer.v1.tar", dest) - require.NoError(t, err) - - info, err := os.Stat(filepath.Join(dest, "d")) - require.NoError(t, err) - require.True(t, info.ModTime().Equal(dirTime), "dir mtime must come from the tar header") -} - -func TestSpecialFileModeSupportsSockets(t *testing.T) { - mode, err := specialFileMode(fs.ModeSocket) - require.NoError(t, err) - require.Equal(t, uint32(syscall.S_IFSOCK), mode) -} - -func TestApplyLayerTreePreservesDirMtime(t *testing.T) { - root := t.TempDir() - targetDir := filepath.Join(root, "target") - layerDir := filepath.Join(root, "layer") - - old := time.Now().Add(-time.Hour).Truncate(time.Second) - require.NoError(t, os.MkdirAll(filepath.Join(layerDir, "d"), 0700)) - require.NoError(t, os.WriteFile(filepath.Join(layerDir, "d", "file.txt"), []byte("x"), 0644)) - require.NoError(t, os.Chtimes(filepath.Join(layerDir, "d"), old, old)) - - require.NoError(t, applyLayerTree(layerDir, targetDir, nil)) - - info, err := os.Stat(filepath.Join(targetDir, "d")) - require.NoError(t, err) - require.True(t, info.ModTime().Equal(old), "dir mtime must survive the merge") -} - -func TestSpecialFileModeSupportsCharDevice(t *testing.T) { - mode, err := specialFileMode(fs.ModeCharDevice | fs.ModeDevice) - require.NoError(t, err) - require.Equal(t, uint32(syscall.S_IFCHR), mode) -} - -func TestApplyTarMetadataPreservesSpecialBits(t *testing.T) { - root := t.TempDir() - target := filepath.Join(root, "suid-bin") - require.NoError(t, os.WriteFile(target, []byte("bin"), 0755)) - - header := &tar.Header{ - Name: "suid-bin", - Typeflag: tar.TypeReg, - Mode: 04755, - } - require.NoError(t, applyTarMetadata(target, header)) - - info, err := os.Stat(target) - require.NoError(t, err) - require.Equal(t, os.ModeSetuid, info.Mode()&os.ModeSetuid, "setuid bit must be preserved") -} - -func TestApplyLayerTreeRejectsMalformedWhiteouts(t *testing.T) { - root := t.TempDir() - targetDir := filepath.Join(root, "target") - layerDir := filepath.Join(root, "layer") - - require.NoError(t, os.MkdirAll(filepath.Join(targetDir, "parent", "child"), 0755)) - require.NoError(t, os.WriteFile(filepath.Join(targetDir, "parent", "victim.txt"), []byte("v"), 0644)) - require.NoError(t, os.MkdirAll(filepath.Join(layerDir, "parent"), 0755)) - // Malformed whiteouts must be rejected without deleting lower-layer entries. - require.NoError(t, os.WriteFile(filepath.Join(layerDir, "parent", ".wh."), nil, 0644)) - require.NoError(t, os.WriteFile(filepath.Join(layerDir, "parent", ".wh.."), nil, 0644)) - require.NoError(t, os.WriteFile(filepath.Join(layerDir, "parent", ".wh..."), nil, 0644)) - - err := applyLayerTree(layerDir, targetDir, nil) - require.ErrorContains(t, err, "invalid whiteout entry") - - // Parent directory must not be deleted by malformed whiteout paths. - info, err := os.Stat(filepath.Join(targetDir, "parent")) - require.NoError(t, err) - require.True(t, info.IsDir()) - require.FileExists(t, filepath.Join(targetDir, "parent", "victim.txt")) -} - -func TestApplyLayerTreeRejectsEscapingSymlink(t *testing.T) { - root := t.TempDir() - targetDir := filepath.Join(root, "target") - layerDir := filepath.Join(root, "layer") - require.NoError(t, os.MkdirAll(targetDir, 0755)) - require.NoError(t, os.MkdirAll(layerDir, 0755)) - require.NoError(t, os.Symlink("../../outside", filepath.Join(layerDir, "link"))) - - err := applyLayerTree(layerDir, targetDir, nil) - require.ErrorContains(t, err, "symlink target escapes root") -} - -func TestApplyLayerTreeRestoresRestrictiveNestedDirModes(t *testing.T) { - root := t.TempDir() - t.Cleanup(func() { _ = makeTreeWritable(root) }) - targetDir := filepath.Join(root, "target") - layerDir := filepath.Join(root, "layer") - - require.NoError(t, os.MkdirAll(filepath.Join(layerDir, "outer", "inner"), 0755)) - require.NoError(t, os.WriteFile(filepath.Join(layerDir, "outer", "inner", "file.txt"), []byte("data"), 0644)) - require.NoError(t, os.Chmod(filepath.Join(layerDir, "outer", "inner"), 0500)) - require.NoError(t, os.Chmod(filepath.Join(layerDir, "outer"), 0500)) - - require.NoError(t, applyLayerTree(layerDir, targetDir, nil)) - - infoOuter, err := os.Stat(filepath.Join(targetDir, "outer")) - require.NoError(t, err) - require.Equal(t, os.FileMode(0500), infoOuter.Mode().Perm()) - - infoInner, err := os.Stat(filepath.Join(targetDir, "outer", "inner")) - require.NoError(t, err) - require.Equal(t, os.FileMode(0500), infoInner.Mode().Perm()) -} - -func TestMakePathWritableTopDown(t *testing.T) { - root := t.TempDir() - t.Cleanup(func() { _ = makeTreeWritable(root) }) - parent := filepath.Join(root, "restricted") - child := filepath.Join(parent, "child") - require.NoError(t, os.MkdirAll(child, 0755)) - require.NoError(t, os.Chmod(parent, 0400)) // read-only, no execute - - originalModes := make(map[string]fs.FileMode) - err := makePathWritable(root, child, originalModes) - require.NoError(t, err) - - info, err := os.Stat(parent) - require.NoError(t, err) - require.True(t, info.Mode().Perm()&0700 == 0700) - - require.NoError(t, restoreDirectoryModes(originalModes)) - infoAfter, err := os.Stat(parent) - require.NoError(t, err) - require.Equal(t, os.FileMode(0400), infoAfter.Mode().Perm()) -} - -func TestSafeJoinCleanRootSymlink(t *testing.T) { - root := t.TempDir() - realDir := filepath.Join(root, "real") - symDir := filepath.Join(root, "sym") - require.NoError(t, os.Mkdir(realDir, 0755)) - require.NoError(t, os.Symlink("real", symDir)) - - // Calling safeJoin with trailing slash on symlink root must be rejected as symlink. - _, err := safeJoin(symDir+"/", "file.txt") - require.Error(t, err) - require.Contains(t, err.Error(), "extraction root is not a directory") + require.Equal(t, int64(tarData.Len()), stats.unpackedBytes) } diff --git a/lib/images/layer_tree.go b/lib/images/layer_tree.go deleted file mode 100644 index 634a015b3..000000000 --- a/lib/images/layer_tree.go +++ /dev/null @@ -1,563 +0,0 @@ -package images - -import ( - "archive/tar" - "bytes" - "errors" - "fmt" - "io" - "io/fs" - "os" - "path/filepath" - "sort" - "strings" - "syscall" - "time" - - "golang.org/x/sys/unix" -) - -func removePath(path string) error { - if err := makeTreeWritable(path); err != nil && !os.IsNotExist(err) { - return err - } - if err := os.RemoveAll(path); err != nil && !os.IsNotExist(err) { - return err - } - return nil -} - -func makeTreeWritable(path string) error { - info, err := os.Lstat(path) - if err != nil { - return err - } - if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { - return nil - } - if err := os.Chmod(path, info.Mode().Perm()|0700); err != nil { - return err - } - return filepath.WalkDir(path, func(p string, entry fs.DirEntry, err error) error { - if err != nil { - return err - } - if entry.IsDir() && entry.Type()&os.ModeSymlink == 0 { - info, err := entry.Info() - if err != nil { - return err - } - return os.Chmod(p, info.Mode().Perm()|0700) - } - return nil - }) -} - -// applyLayerTree merges one unpacked layer directory into targetDir following -// OCI whiteout semantics: whiteouts and opaque markers remove what lower layers -// contributed, then the layer's own entries are copied on top. Raw tar -// whiteout files are interpreted here rather than passed through, because -// overlayfs does not understand them. No production caller yet; composition -// lands in a later change. -// -// explicitDirs, when non-nil, names the directories the layer tar listed -// explicitly (relative, cleaned); only those have their metadata copied. -// A nil map treats every directory as explicit. -func applyLayerTree(layerDir, targetDir string, explicitDirs map[string]struct{}) (err error) { - if err := os.MkdirAll(targetDir, 0755); err != nil { - return fmt.Errorf("create target directory: %w", err) - } - originalModes := make(map[string]fs.FileMode) - defer func() { - if restoreErr := restoreDirectoryModes(originalModes); restoreErr != nil { - err = errors.Join(err, fmt.Errorf("restore directory modes: %w", restoreErr)) - } - }() - - // Prepare directories before whiteouts so nested markers are applied after - // the layer has replaced any conflicting lower-layer entry. - if err := prepareLayerDirectories(layerDir, targetDir, originalModes); err != nil { - return fmt.Errorf("prepare layer directories: %w", err) - } - - // Phase 1: apply whiteouts against what is already in the target. - err = filepath.WalkDir(layerDir, func(path string, entry fs.DirEntry, err error) error { - if err != nil { - return err - } - base := entry.Name() - if path == layerDir || !strings.HasPrefix(base, whiteoutPrefix) { - return nil - } - rel, err := filepath.Rel(layerDir, path) - if err != nil { - return err - } - targetParent, err := safeJoinForComposition(targetDir, filepath.Dir(rel)) - if err != nil { - return err - } - if err := makePathWritable(targetDir, targetParent, originalModes); err != nil { - return err - } - if base == opaqueWhiteout { - return clearDirContents(targetParent) - } - hidden := strings.TrimPrefix(base, whiteoutPrefix) - if hidden == "" || hidden == "." || hidden == ".." { - return fmt.Errorf("invalid whiteout entry: %s", rel) - } - target, err := safeJoinForComposition(targetDir, filepath.Join(filepath.Dir(rel), hidden)) - if err != nil { - return err - } - return removePath(target) - }) - if err != nil { - return fmt.Errorf("apply whiteouts: %w", err) - } - - // Phase 2: copy the layer's own entries, skipping whiteout markers. - // Directory metadata is deferred until all children are copied, so tar - // directory mtimes survive the merge. - pendingDirs := make([]dirMeta, 0) - hardlinks := make(map[hardlinkIdentity]string) - err = filepath.WalkDir(layerDir, func(path string, entry fs.DirEntry, err error) error { - if err != nil { - return err - } - if path == layerDir { - return nil - } - if strings.HasPrefix(entry.Name(), whiteoutPrefix) { - if entry.IsDir() { - return fs.SkipDir - } - return nil - } - rel, err := filepath.Rel(layerDir, path) - if err != nil { - return err - } - target, err := safeJoinForComposition(targetDir, rel) - if err != nil { - return err - } - if entry.IsDir() { - target, err = resolveCompositionDirTarget(targetDir, target) - if err != nil { - return err - } - } - if err := makePathWritable(targetDir, filepath.Dir(target), originalModes); err != nil { - return err - } - info, err := entry.Info() - if err != nil { - return err - } - if entry.IsDir() { - if _, explicit := explicitDirs[filepath.Clean(rel)]; explicitDirs == nil || explicit { - pendingDirs = append(pendingDirs, dirMeta{src: path, dst: target, info: info}) - } - } - if err := copyEntryInto(path, target, info, hardlinks, targetDir); err != nil { - return err - } - if entry.IsDir() { - return makePathWritable(targetDir, target, originalModes) - } - return nil - }) - if err != nil { - return fmt.Errorf("copy layer tree: %w", err) - } - for i := len(pendingDirs) - 1; i >= 0; i-- { - dir := pendingDirs[i] - if err := copyEntryMetadata(dir.src, dir.dst, dir.info); err != nil { - return fmt.Errorf("restore dir metadata %s: %w", dir.dst, err) - } - delete(originalModes, dir.dst) - } - return nil -} - -func prepareLayerDirectories(layerDir, targetDir string, originalModes map[string]fs.FileMode) error { - return filepath.WalkDir(layerDir, func(path string, entry fs.DirEntry, err error) error { - if err != nil { - return err - } - if path == layerDir || !entry.IsDir() { - return nil - } - if strings.HasPrefix(entry.Name(), whiteoutPrefix) { - return filepath.SkipDir - } - rel, err := filepath.Rel(layerDir, path) - if err != nil { - return err - } - target, err := safeJoinForComposition(targetDir, rel) - if err != nil { - return err - } - target, err = resolveCompositionDirTarget(targetDir, target) - if err != nil { - return err - } - if err := makePathWritable(targetDir, filepath.Dir(target), originalModes); err != nil { - return err - } - info, err := entry.Info() - if err != nil { - return err - } - if err := copyDirectoryEntry(target, info); err != nil { - return err - } - return makePathWritable(targetDir, target, originalModes) - }) -} - -func makePathWritable(root, path string, originalModes map[string]fs.FileMode) error { - root = filepath.Clean(root) - path = filepath.Clean(path) - if !pathWithinRoot(root, path) { - return fmt.Errorf("path is outside target root: %s", path) - } - rel, err := filepath.Rel(root, path) - if err != nil { - return err - } - if err := makeDirWritable(root, originalModes); err != nil { - return err - } - if rel == "." { - return nil - } - current := root - for _, part := range strings.Split(rel, string(filepath.Separator)) { - current = filepath.Join(current, part) - if err := makeDirWritable(current, originalModes); err != nil { - return err - } - } - return nil -} - -func makeDirWritable(dir string, originalModes map[string]fs.FileMode) error { - info, err := os.Lstat(dir) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { - return fmt.Errorf("target parent is not a directory: %s", dir) - } - if _, recorded := originalModes[dir]; !recorded { - originalModes[dir] = info.Mode() - if err := os.Chmod(dir, info.Mode().Perm()|0700); err != nil { - return err - } - } - return nil -} - -func restoreDirectoryModes(originalModes map[string]fs.FileMode) error { - paths := make([]string, 0, len(originalModes)) - for path := range originalModes { - paths = append(paths, path) - } - sort.Slice(paths, func(i, j int) bool { - return len(paths[i]) > len(paths[j]) - }) - var restoreErr error - for _, path := range paths { - mode := originalModes[path] - info, err := os.Lstat(path) - if os.IsNotExist(err) || (err == nil && (info.Mode()&os.ModeSymlink != 0 || !info.IsDir())) { - continue - } else if err != nil { - restoreErr = errors.Join(restoreErr, err) - continue - } - if err := os.Chmod(path, mode.Perm()|mode&(os.ModeSetuid|os.ModeSetgid|os.ModeSticky)); err != nil { - restoreErr = errors.Join(restoreErr, err) - } - } - return restoreErr -} - -type dirMeta struct { - src, dst string - info os.FileInfo -} - -// clearDirContents removes everything inside dir without removing dir itself, -// and without following symlinks. -func clearDirContents(dir string) error { - info, err := os.Lstat(dir) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - if !info.IsDir() { - return removePath(dir) - } - entries, err := os.ReadDir(dir) - if err != nil { - return err - } - for _, entry := range entries { - if err := removePath(filepath.Join(dir, entry.Name())); err != nil { - return err - } - } - return nil -} - -type hardlinkIdentity struct { - dev uint64 - ino uint64 -} - -// copyEntryInto copies one filesystem entry from src to dst, replacing any -// conflicting entry and preserving hardlinks within the layer. -func copyEntryInto(src, dst string, info os.FileInfo, hardlinks map[hardlinkIdentity]string, root string) error { - switch info.Mode() & fs.ModeType { - case 0: - return copyRegularEntry(src, dst, info, hardlinks) - case fs.ModeDir: - return copyDirectoryEntry(dst, info) - case fs.ModeSymlink: - return copySymlinkEntry(src, dst, info, root) - default: - return copySpecialEntry(src, dst, info) - } -} - -func copyRegularEntry(src, dst string, info os.FileInfo, hardlinks map[hardlinkIdentity]string) error { - if err := removePath(dst); err != nil { - return err - } - if stat, ok := info.Sys().(*syscall.Stat_t); ok && stat.Nlink > 1 { - identity := hardlinkIdentity{dev: uint64(stat.Dev), ino: uint64(stat.Ino)} - if first, seen := hardlinks[identity]; seen { - return os.Link(first, dst) - } - hardlinks[identity] = dst - } - if err := copyFileContents(src, dst, info); err != nil { - return err - } - return copyEntryMetadata(src, dst, info) -} - -// copyDirectoryEntry creates dst as a directory, replacing a conflicting -// non-directory. Callers resolve dst through resolveCompositionDirTarget -// first, so dst is never an existing symlink here. -func copyDirectoryEntry(dst string, info os.FileInfo) error { - if existing, err := os.Lstat(dst); err == nil && !existing.IsDir() { - if err := removePath(dst); err != nil { - return err - } - } - return os.MkdirAll(dst, info.Mode().Perm()) -} - -func copySymlinkEntry(src, dst string, info os.FileInfo, root string) error { - linkTarget, err := os.Readlink(src) - if err != nil { - return err - } - if err := validateSymlinkTarget(root, dst, linkTarget); err != nil { - return err - } - if err := removePath(dst); err != nil { - return err - } - if err := os.Symlink(linkTarget, dst); err != nil { - return err - } - return copyEntryMetadata(src, dst, info) -} - -func copySpecialEntry(src, dst string, info os.FileInfo) error { - if err := removePath(dst); err != nil { - return err - } - stat, ok := info.Sys().(*syscall.Stat_t) - if !ok { - return fmt.Errorf("unsupported entry type for %s", src) - } - mode, err := specialFileMode(info.Mode() & fs.ModeType) - if err != nil { - return fmt.Errorf("unsupported entry type for %s: %w", src, err) - } - if err := mknodWithRootlessFallback(dst, mode|uint32(info.Mode().Perm()), int(stat.Rdev)); err != nil { - return err - } - return copyEntryMetadata(src, dst, info) -} - -func specialFileMode(mode fs.FileMode) (uint32, error) { - switch { - case mode&fs.ModeCharDevice != 0: - return syscall.S_IFCHR, nil - case mode&fs.ModeDevice != 0: - return syscall.S_IFBLK, nil - case mode&fs.ModeNamedPipe != 0: - return syscall.S_IFIFO, nil - case mode&fs.ModeSocket != 0: - return syscall.S_IFSOCK, nil - default: - return 0, fmt.Errorf("unsupported file mode") - } -} - -type entryMetadata struct { - uid, gid int - hasOwner bool - mode fs.FileMode - symlink bool - mtime time.Time - xattrs map[string][]byte -} - -func applyEntryMetadata(path string, metadata entryMetadata) error { - if metadata.hasOwner { - if err := os.Lchown(path, metadata.uid, metadata.gid); err != nil && !errors.Is(err, os.ErrPermission) && !errors.Is(err, unix.EPERM) { - return err - } - } - if metadata.symlink { - return nil - } - if err := applyXattrs(path, metadata.xattrs); err != nil { - return err - } - mode := metadata.mode.Perm() | metadata.mode&(os.ModeSetuid|os.ModeSetgid|os.ModeSticky) - if err := os.Chmod(path, mode); err != nil { - return err - } - return os.Chtimes(path, metadata.mtime, metadata.mtime) -} - -func copyEntryMetadata(src, dst string, info os.FileInfo) error { - metadata := entryMetadata{ - mode: info.Mode(), - symlink: info.Mode()&os.ModeSymlink != 0, - mtime: info.ModTime(), - } - if stat, ok := info.Sys().(*syscall.Stat_t); ok { - metadata.uid = int(stat.Uid) - metadata.gid = int(stat.Gid) - metadata.hasOwner = true - } - if !metadata.symlink { - xattrs, err := readXattrs(src) - if err != nil { - return err - } - metadata.xattrs = xattrs - } - return applyEntryMetadata(dst, metadata) -} - -func readXattrs(src string) (map[string][]byte, error) { - size, err := unix.Llistxattr(src, nil) - if err != nil { - if errors.Is(err, unix.ENOTSUP) || errors.Is(err, unix.EPERM) { - return nil, nil - } - return nil, err - } - names := make([]byte, size) - if size > 0 { - n, err := unix.Llistxattr(src, names) - if err != nil { - return nil, err - } - names = names[:n] - } - xattrs := make(map[string][]byte) - for _, attr := range bytes.Split(bytes.TrimSuffix(names, []byte{0}), []byte{0}) { - if len(attr) == 0 { - continue - } - name := string(attr) - size, err := unix.Lgetxattr(src, name, nil) - if err != nil { - if errors.Is(err, unix.ENOTSUP) || errors.Is(err, unix.EPERM) || errors.Is(err, unix.ENODATA) { - continue - } - return nil, err - } - value := make([]byte, size) - n, err := unix.Lgetxattr(src, name, value) - if err != nil { - return nil, err - } - xattrs[name] = value[:n] - } - return xattrs, nil -} - -func tarXattrs(header *tar.Header) map[string][]byte { - const paxXattrPrefix = "SCHILY.xattr." - xattrs := make(map[string][]byte, len(header.Xattrs)) - for name, value := range header.Xattrs { - xattrs[name] = []byte(value) - } - for key, value := range header.PAXRecords { - if strings.HasPrefix(key, paxXattrPrefix) { - name := strings.TrimPrefix(key, paxXattrPrefix) - if _, exists := xattrs[name]; !exists { - xattrs[name] = []byte(value) - } - } - } - return xattrs -} - -func applyXattrs(path string, xattrs map[string][]byte) error { - for name, value := range xattrs { - if err := unix.Lsetxattr(path, name, value, 0); err != nil && !errors.Is(err, unix.ENOTSUP) && !errors.Is(err, unix.EPERM) { - return fmt.Errorf("restore xattr %s: %w", name, err) - } - } - return nil -} - -func copyFileContents(src, dst string, info os.FileInfo) (err error) { - sourceMode := info.Mode().Perm() | info.Mode()&(os.ModeSetuid|os.ModeSetgid|os.ModeSticky) - if sourceMode&0400 == 0 { - if err := os.Chmod(src, sourceMode|0400); err != nil { - return err - } - defer func() { - if restoreErr := os.Chmod(src, sourceMode); err == nil { - err = restoreErr - } - }() - } - - in, err := os.Open(src) - if err != nil { - return err - } - defer in.Close() - out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC|syscall.O_NOFOLLOW, 0644) - if err != nil { - return err - } - if _, err := io.Copy(out, in); err != nil { - out.Close() - return err - } - return out.Close() -} From d3cb0cfcdd8c0dfffef81c4315fd18a77dafde7b Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Thu, 3 Sep 2026 15:16:22 +0000 Subject: [PATCH 17/41] Detach shared layer builds from the initiating context The singleflight body ran with the first caller's context, so cancelling one pull failed materialization for every concurrent pull waiting on the same layer. Run the build under context.WithoutCancel; callers still return on their own cancellation. --- lib/images/layer_artifact.go | 8 +++++++- lib/images/layer_artifact_test.go | 32 +++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/lib/images/layer_artifact.go b/lib/images/layer_artifact.go index 6d67a9c56..2b93c1106 100644 --- a/lib/images/layer_artifact.go +++ b/lib/images/layer_artifact.go @@ -147,10 +147,16 @@ func discardLayerCache(p *paths.Paths, layerHex string) error { // temp directory; lifecycle reconciliation removes stale temp directories after // an interrupted build. No production caller yet: pull integration and // composition land in later changes. +// +// Concurrent callers share one build. The build itself is detached from the +// initiating caller's cancellation so one cancelled pull cannot fail every +// other pull waiting on the same layer; each caller still returns as soon as +// its own context is done. func (m *manager) materializeLayerArtifact(ctx context.Context, desc layerDescriptor) (*layerArtifact, error) { key := desc.Digest + "\x00" + layerArtifactFormat() + buildCtx := context.WithoutCancel(ctx) result := m.layerFlights.DoChan(key, func() (any, error) { - return m.materializeLayerArtifactOnce(ctx, desc) + return m.materializeLayerArtifactOnce(buildCtx, desc) }) select { case <-ctx.Done(): diff --git a/lib/images/layer_artifact_test.go b/lib/images/layer_artifact_test.go index 528c91abd..5e81a4624 100644 --- a/lib/images/layer_artifact_test.go +++ b/lib/images/layer_artifact_test.go @@ -12,6 +12,7 @@ import ( "os/exec" "path/filepath" "testing" + "time" gcr "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" @@ -201,6 +202,37 @@ func TestMaterializeLayerArtifactRecoversCorruptRecord(t *testing.T) { require.NoError(t, record.validate()) } +// TestMaterializeLayerArtifactOutlivesCancelledCaller checks that a caller +// abandoning a shared build does not abort the build for everyone else: the +// artifact still lands even though the initiating context was cancelled. +func TestMaterializeLayerArtifactOutlivesCancelledCaller(t *testing.T) { + if _, err := exec.LookPath("mkfs.ext4"); err != nil { + t.Skip("mkfs.ext4 not available") + } + originalFormat := DefaultImageFormat + DefaultImageFormat = FormatExt4 + t.Cleanup(func() { DefaultImageFormat = originalFormat }) + + p := paths.New(t.TempDir()) + img, err := mutate.AppendLayers(empty.Image, syntheticLayer(t, "base.txt", "base layer content")) + require.NoError(t, err) + writeLayerTestLayout(t, p, img) + desc := layerDescFromImage(t, img, 0) + m := &manager{paths: p} + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err = m.materializeLayerArtifact(ctx, desc) + require.ErrorIs(t, err, context.Canceled) + + layerHex := desc.Digest[len("sha256:"):] + require.Eventually(t, func() bool { + record, err := readLayerRecord(p, layerHex) + return err == nil && record != nil + }, 30*time.Second, 50*time.Millisecond, "detached build must still install the artifact") + require.FileExists(t, p.ImageLayerArtifactForFormat(layerHex, layerArtifactFormat())) +} + func TestMaterializeLayerArtifactMissingBlob(t *testing.T) { p := paths.New(t.TempDir()) m := &manager{paths: p} From 6681c4178dbca71dc23b30d3040fa06691933d17 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Thu, 3 Sep 2026 15:30:58 +0000 Subject: [PATCH 18/41] Trim layer artifact plumbing Fold contextReader into layer_artifact.go, its only user. Drop the context-aware mkfs.erofs wrapper: the shared build now runs detached from caller cancellation, so the context could never fire during conversion, and ExportRootfs already dispatches on the format. --- lib/images/context_reader.go | 18 ------------------ lib/images/disk.go | 7 +------ lib/images/layer_artifact.go | 30 +++++++++++++++++++----------- 3 files changed, 20 insertions(+), 35 deletions(-) delete mode 100644 lib/images/context_reader.go diff --git a/lib/images/context_reader.go b/lib/images/context_reader.go deleted file mode 100644 index 8aec8b2e4..000000000 --- a/lib/images/context_reader.go +++ /dev/null @@ -1,18 +0,0 @@ -package images - -import ( - "context" - "io" -) - -type contextReader struct { - ctx context.Context - reader io.Reader -} - -func (r contextReader) Read(p []byte) (int, error) { - if err := r.ctx.Err(); err != nil { - return 0, err - } - return r.reader.Read(p) -} diff --git a/lib/images/disk.go b/lib/images/disk.go index 97e77db53..896d10f2a 100644 --- a/lib/images/disk.go +++ b/lib/images/disk.go @@ -1,7 +1,6 @@ package images import ( - "context" "fmt" "os" "os/exec" @@ -195,10 +194,6 @@ func convertToExt4(rootfsDir, diskPath string) (int64, error) { // convertToErofs converts a rootfs directory to an erofs disk image using mkfs.erofs func convertToErofs(rootfsDir, diskPath string) (int64, error) { - return convertToErofsContext(context.Background(), rootfsDir, diskPath) -} - -func convertToErofsContext(ctx context.Context, rootfsDir, diskPath string) (int64, error) { // Ensure parent directory exists if err := os.MkdirAll(filepath.Dir(diskPath), 0755); err != nil { return 0, fmt.Errorf("create disk parent dir: %w", err) @@ -207,7 +202,7 @@ func convertToErofsContext(ctx context.Context, rootfsDir, diskPath string) (int // Create erofs image with LZ4 fast compression // -zlz4: LZ4 fast compression (~20-25% space savings, faster builds) // erofs doesn't need pre-allocation, creates file directly - cmd := exec.CommandContext(ctx, "mkfs.erofs", "-zlz4", diskPath, rootfsDir) + cmd := exec.Command("mkfs.erofs", "-zlz4", diskPath, rootfsDir) output, err := cmd.CombinedOutput() if err != nil { return 0, fmt.Errorf("mkfs.erofs failed: %w, output: %s", err, output) diff --git a/lib/images/layer_artifact.go b/lib/images/layer_artifact.go index 2b93c1106..0f40a0960 100644 --- a/lib/images/layer_artifact.go +++ b/lib/images/layer_artifact.go @@ -206,10 +206,10 @@ func (m *manager) materializeLayerArtifactOnce(ctx context.Context, desc layerDe if err != nil { return nil, err } - return m.installLayerArtifact(ctx, desc, layerHex, unpackDir, stats) + return m.installLayerArtifact(desc, layerHex, unpackDir, stats) } -func (m *manager) installLayerArtifact(ctx context.Context, desc layerDescriptor, layerHex, unpackDir string, stats *unpackStats) (*layerArtifact, error) { +func (m *manager) installLayerArtifact(desc layerDescriptor, layerHex, unpackDir string, stats *unpackStats) (*layerArtifact, error) { record := &layerArtifact{ SchemaVersion: layerRecordSchemaVersion, Digest: desc.Digest, @@ -220,15 +220,9 @@ func (m *manager) installLayerArtifact(ctx context.Context, desc layerDescriptor } if err := installAtomically(layerArtifactPath(m.paths, layerHex), func(path string) error { - var size int64 - var convErr error - if DefaultImageFormat == FormatErofs { - size, convErr = convertToErofsContext(ctx, unpackDir, path) - } else { - size, convErr = ExportRootfs(unpackDir, path, DefaultImageFormat) - } - if convErr != nil { - return convErr + size, err := ExportRootfs(unpackDir, path, DefaultImageFormat) + if err != nil { + return err } record.SizeBytes = size return nil @@ -252,6 +246,20 @@ type unpackStats struct { diffID string } +// contextReader fails reads once ctx is done so a cancelled caller stops a +// long extraction instead of running it to completion. +type contextReader struct { + ctx context.Context + reader io.Reader +} + +func (r contextReader) Read(p []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.reader.Read(p) +} + // unpackCachedLayer locates desc's blob in the shared OCI cache, unpacks it // into dest, and verifies the diff ID when the descriptor carries one. func unpackCachedLayer(ctx context.Context, cacheDir string, desc layerDescriptor, dest string, onDisk layer.OnDiskFormat) (*unpackStats, error) { From 57d56ae3ef76fd1587f492608ee84e75292660bd Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Thu, 3 Sep 2026 16:06:03 +0000 Subject: [PATCH 19/41] Harden layer artifact builds and drop unused plumbing - Bound detached layer builds with a deadline so a hung cache-blob read cannot wedge the singleflight key forever - Verify the compressed blob digest during unpack; the pull path does not re-verify blobs after they land in the OCI cache - Resolve blob paths through paths.OCICacheBlob instead of hand-joining the cache layout, and drop the duplicated digest validation - Simplify decompressLayer and move composeOnDiskFormat/whiteoutPrefix to the tests that use them - Log failures to remove the unpack directory instead of discarding the error - Make the cancelled-caller test tolerant of a build that finishes before the caller's select runs --- lib/images/layer_artifact.go | 92 ++++++++++++++++--------------- lib/images/layer_artifact_test.go | 21 ++++++- 2 files changed, 68 insertions(+), 45 deletions(-) diff --git a/lib/images/layer_artifact.go b/lib/images/layer_artifact.go index 0f40a0960..d7b79fe19 100644 --- a/lib/images/layer_artifact.go +++ b/lib/images/layer_artifact.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "io/fs" + "log/slog" "os" "path/filepath" "strings" @@ -19,15 +20,14 @@ import ( "github.com/opencontainers/umoci/oci/layer" ) -// whiteoutPrefix marks OCI whiteout entries (".wh." and ".wh..wh..opq"). -// umoci interprets them during extraction: DirRootfs applies them against the -// tree being composed, OverlayfsRootfs converts them to overlayfs whiteout -// inodes and opaque xattrs so per-layer artifacts can later be stacked. -const whiteoutPrefix = ".wh." - const ( layerRecordSchemaVersion = 1 maxLayerUnpackedBytes = 100 << 30 + + // layerBuildTimeout bounds a shared layer build end to end. Builds are + // detached from the initiating request's context, so this deadline is the + // only thing that can abort a hung unpack and free the singleflight key. + layerBuildTimeout = time.Hour ) var errCorruptLayerRecord = errors.New("corrupt layer record") @@ -64,6 +64,10 @@ func (a *layerArtifact) validate() error { return nil } +// matches decides whether the stored record satisfies a lookup. A descriptor +// without a DiffID (a manifest lookup that did not consult the image config) +// matches any record: the compressed digest content-addresses the blob, and +// unpackCachedLayer re-verifies the diff ID whenever one is supplied. func (a *layerArtifact) matches(desc layerDescriptor) bool { if a.Digest != desc.Digest || a.Format != layerArtifactFormat() { return false @@ -90,6 +94,9 @@ func layerArtifactRecordPath(p *paths.Paths, layerHex string) string { // layerMapOptions preserves tar ownership when running as root. Otherwise // umoci's rootless mode skips chown and stands in empty files for device nodes. +// Unlike unpackLayers in oci.go, which maps container root to the current +// user, this deliberately leaves ownership untouched as root: artifacts must +// keep the layer's on-disk ownership for later stacking. func layerMapOptions() layer.MapOptions { return layer.MapOptions{Rootless: os.Geteuid() != 0} } @@ -102,11 +109,6 @@ func layerArtifactOnDiskFormat() layer.OnDiskFormat { return layer.OverlayfsRootfs{MapOptions: layerMapOptions()} } -// composeOnDiskFormat applies whiteouts against the tree being composed. -func composeOnDiskFormat() layer.OnDiskFormat { - return layer.DirRootfs{MapOptions: layerMapOptions()} -} - // readLayerRecord loads the artifact record for a layer digest, if present. // A missing record returns (nil, nil): the layer simply was never // materialized. @@ -144,8 +146,9 @@ func discardLayerCache(p *paths.Paths, layerHex string) error { // by its blob digest, building it from the shared OCI cache blob when absent. // The layer is unpacked into an isolated temp directory, converted to the // default image format, and installed atomically. Normal failures remove the -// temp directory; lifecycle reconciliation removes stale temp directories after -// an interrupted build. No production caller yet: pull integration and +// temp directory; a crash mid-build can leave a stale .unpack-* directory +// behind, which reconciliation landing with the pull integration is expected +// to sweep. No production caller yet: pull integration and // composition land in later changes. // // Concurrent callers share one build. The build itself is detached from the @@ -154,8 +157,14 @@ func discardLayerCache(p *paths.Paths, layerHex string) error { // its own context is done. func (m *manager) materializeLayerArtifact(ctx context.Context, desc layerDescriptor) (*layerArtifact, error) { key := desc.Digest + "\x00" + layerArtifactFormat() - buildCtx := context.WithoutCancel(ctx) + // The build outlives the initiating request, so the deadline below is its + // only bound: without it a hung cache-blob read would wedge the + // singleflight key, and every future caller for the layer, forever. + buildCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), layerBuildTimeout) result := m.layerFlights.DoChan(key, func() (any, error) { + // Cancel here, in the flight's frame: the initiating caller may return + // long before the detached build finishes. + defer cancel() return m.materializeLayerArtifactOnce(buildCtx, desc) }) select { @@ -200,9 +209,13 @@ func (m *manager) materializeLayerArtifactOnce(ctx context.Context, desc layerDe if err != nil { return nil, fmt.Errorf("create unpack directory: %w", err) } - defer removePath(unpackDir) + defer func() { + if err := removePath(unpackDir); err != nil { + slog.Warn("failed to remove layer unpack directory", "dir", unpackDir, "error", err) + } + }() - stats, err := unpackCachedLayer(ctx, m.paths.SystemOCICache(), desc, unpackDir, layerArtifactOnDiskFormat()) + stats, err := unpackCachedLayer(ctx, m.paths, desc, unpackDir, layerArtifactOnDiskFormat()) if err != nil { return nil, err } @@ -244,6 +257,7 @@ func (m *manager) installLayerArtifact(desc layerDescriptor, layerHex, unpackDir type unpackStats struct { unpackedBytes int64 diffID string + blobDigest string // sha256 of the compressed bytes as read } // contextReader fails reads once ctx is done so a cancelled caller stops a @@ -261,13 +275,10 @@ func (r contextReader) Read(p []byte) (int, error) { } // unpackCachedLayer locates desc's blob in the shared OCI cache, unpacks it -// into dest, and verifies the diff ID when the descriptor carries one. -func unpackCachedLayer(ctx context.Context, cacheDir string, desc layerDescriptor, dest string, onDisk layer.OnDiskFormat) (*unpackStats, error) { - layerHex := strings.TrimPrefix(desc.Digest, "sha256:") - if err := paths.ValidatePathComponent(layerHex); err != nil { - return nil, fmt.Errorf("invalid layer digest: %s", desc.Digest) - } - blobPath := filepath.Join(cacheDir, "blobs", "sha256", layerHex) +// into dest, and verifies both the blob digest and the diff ID when the +// descriptor carries one. The caller must have validated desc.Digest. +func unpackCachedLayer(ctx context.Context, p *paths.Paths, desc layerDescriptor, dest string, onDisk layer.OnDiskFormat) (*unpackStats, error) { + blobPath := p.OCICacheBlob(strings.TrimPrefix(desc.Digest, "sha256:")) if _, err := os.Stat(blobPath); err != nil { if os.IsNotExist(err) { return nil, fmt.Errorf("layer blob missing from oci cache: %s", desc.Digest) @@ -278,6 +289,9 @@ func unpackCachedLayer(ctx context.Context, cacheDir string, desc layerDescripto if err != nil { return nil, fmt.Errorf("unpack layer %s: %w", desc.Digest, err) } + if desc.Digest != "" && stats.blobDigest != desc.Digest { + return nil, fmt.Errorf("layer blob digest mismatch: got %s, want %s", stats.blobDigest, desc.Digest) + } if desc.DiffID != "" && stats.diffID != desc.DiffID { return nil, fmt.Errorf("layer %s diff id mismatch: got %s, want %s", desc.Digest, stats.diffID, desc.DiffID) } @@ -286,7 +300,8 @@ func unpackCachedLayer(ctx context.Context, cacheDir string, desc layerDescripto // unpackLayerBlob extracts one compressed layer blob into dest with umoci, // which confines every entry to dest and interprets whiteouts per onDisk. -// The decompressed stream is hashed for the diff ID and capped in size. +// The compressed stream is hashed to verify the blob digest, the decompressed +// stream for the diff ID, and capped in size. func unpackLayerBlob(ctx context.Context, blobPath, mediaType, dest string, onDisk layer.OnDiskFormat) (*unpackStats, error) { if err := os.MkdirAll(dest, 0755); err != nil { return nil, fmt.Errorf("create extraction root: %w", err) @@ -297,7 +312,8 @@ func unpackLayerBlob(ctx context.Context, blobPath, mediaType, dest string, onDi } defer blob.Close() - reader, closer, err := decompressLayer(blob, mediaType) + blobHash := sha256.New() + reader, closer, err := decompressLayer(io.TeeReader(blob, blobHash), mediaType) if err != nil { return nil, err } @@ -318,41 +334,31 @@ func unpackLayerBlob(ctx context.Context, blobPath, mediaType, dest string, onDi return &unpackStats{ unpackedBytes: maxLayerUnpackedBytes + 1 - limited.N, diffID: fmt.Sprintf("sha256:%x", hash.Sum(nil)), + blobDigest: fmt.Sprintf("sha256:%x", blobHash.Sum(nil)), }, nil } // decompressLayer wraps the blob in the reader for its layer media type. Both // OCI-style suffixes (+gzip, +zstd) and docker-style media types (tar.gzip, // tar.zstd) are matched so neither encoding falls through to the raw path. -func decompressLayer(blob *os.File, mediaType string) (io.Reader, io.Closer, error) { +func decompressLayer(r io.Reader, mediaType string) (io.Reader, io.Closer, error) { switch { case strings.HasSuffix(mediaType, "+zstd"), strings.Contains(mediaType, "tar.zstd"): - decoder, err := zstd.NewReader(blob) + decoder, err := zstd.NewReader(r) if err != nil { return nil, nil, fmt.Errorf("zstd reader: %w", err) } - return decoder, multiCloser{decoder.IOReadCloser(), blob}, nil + decodeCloser := decoder.IOReadCloser() + return decodeCloser, decodeCloser, nil case strings.HasSuffix(mediaType, "+gzip"), strings.Contains(mediaType, "tar.gzip"): - gz, err := gzip.NewReader(blob) + gz, err := gzip.NewReader(r) if err != nil { return nil, nil, fmt.Errorf("gzip reader: %w", err) } - return gz, multiCloser{gz, blob}, nil + return gz, gz, nil default: - return blob, blob, nil - } -} - -type multiCloser []io.Closer - -func (c multiCloser) Close() error { - var firstErr error - for _, closer := range c { - if err := closer.Close(); err != nil && firstErr == nil { - firstErr = err - } + return r, io.NopCloser(r), nil } - return firstErr } // removePath removes a tree that may contain read-only directories restored diff --git a/lib/images/layer_artifact_test.go b/lib/images/layer_artifact_test.go index 5e81a4624..2ba416031 100644 --- a/lib/images/layer_artifact_test.go +++ b/lib/images/layer_artifact_test.go @@ -19,10 +19,24 @@ import ( "github.com/google/go-containerregistry/pkg/v1/layout" "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/kernel/hypeman/lib/paths" + "github.com/opencontainers/umoci/oci/layer" "github.com/stretchr/testify/require" "golang.org/x/sys/unix" ) +// whiteoutPrefix marks OCI whiteout entries (".wh." and ".wh..wh..opq"). +// umoci interprets them during extraction: composeOnDiskFormat applies them +// against the tree being composed, layerArtifactOnDiskFormat converts them to +// overlayfs whiteout inodes and opaque xattrs so per-layer artifacts can +// later be stacked. +const whiteoutPrefix = ".wh." + +// composeOnDiskFormat applies whiteouts against the tree being composed. It +// belongs to the composition flow and moves to production with that change. +func composeOnDiskFormat() layer.OnDiskFormat { + return layer.DirRootfs{MapOptions: layerMapOptions()} +} + const testTarGzMediaType = "application/vnd.oci.image.layer.v1.tar+gzip" // writeLayerTestLayout writes img into the shared OCI cache of p tagged with @@ -222,8 +236,11 @@ func TestMaterializeLayerArtifactOutlivesCancelledCaller(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - _, err = m.materializeLayerArtifact(ctx, desc) - require.ErrorIs(t, err, context.Canceled) + // The build usually loses the race against the cancelled context, but a + // fast build can finish first; either outcome is valid when detached. + if _, buildErr := m.materializeLayerArtifact(ctx, desc); buildErr != nil { + require.ErrorIs(t, buildErr, context.Canceled) + } layerHex := desc.Digest[len("sha256:"):] require.Eventually(t, func() bool { From ae7ca838f1f48d3558b3601e0846194aab17bd07 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Thu, 3 Sep 2026 16:06:03 +0000 Subject: [PATCH 20/41] Add layer artifact edge-case smoke tests Covers zstd and docker-style media types, traversal in layer digest hex, rebuild when the artifact file is missing, concurrent shared materialization, device and fifo entries (root only), and layer artifact disk-usage accounting. --- lib/images/layer_artifact_smoke_test.go | 239 ++++++++++++++++++++++++ 1 file changed, 239 insertions(+) create mode 100644 lib/images/layer_artifact_smoke_test.go diff --git a/lib/images/layer_artifact_smoke_test.go b/lib/images/layer_artifact_smoke_test.go new file mode 100644 index 000000000..9f0d30b5c --- /dev/null +++ b/lib/images/layer_artifact_smoke_test.go @@ -0,0 +1,239 @@ +package images + +import ( + "archive/tar" + "bytes" + "context" + "os" + "os/exec" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/kernel/hypeman/lib/paths" + "github.com/klauspost/compress/zstd" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +// Smoke tests for PR #456 edge cases beyond the standard suite. +// Media types, path validation, cache-miss rebuilds, singleflight sharing, +// device/fifo entries (root only), and disk-usage accounting. + +func writeRawTarLayer(t *testing.T, dir, name string, entries ...func(*testing.T, *tar.Writer)) string { + t.Helper() + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + for _, entry := range entries { + entry(t, tw) + } + require.NoError(t, tw.Close()) + path := filepath.Join(dir, name) + require.NoError(t, os.WriteFile(path, buf.Bytes(), 0644)) + return path +} + +func writeZstdTarLayer(t *testing.T, dir, name string, entries ...func(*testing.T, *tar.Writer)) string { + t.Helper() + var tarBuf bytes.Buffer + tw := tar.NewWriter(&tarBuf) + for _, entry := range entries { + entry(t, tw) + } + require.NoError(t, tw.Close()) + + var zstBuf bytes.Buffer + zw, err := zstd.NewWriter(&zstBuf) + require.NoError(t, err) + _, err = zw.Write(tarBuf.Bytes()) + require.NoError(t, err) + require.NoError(t, zw.Close()) + path := filepath.Join(dir, name) + require.NoError(t, os.WriteFile(path, zstBuf.Bytes(), 0644)) + return path +} + +const ( + ociZstdMediaType = "application/vnd.oci.image.layer.v1.tar+zstd" + dockerZstdMediaType = "application/vnd.docker.image.rootfs.diff.tar.zstd" + dockerGzipMediaType = "application/vnd.docker.image.rootfs.diff.tar.gzip" + ociPlainTarMediaType = "application/vnd.oci.image.layer.v1.tar" +) + +func TestSmokeUnpackZstdMediaTypes(t *testing.T) { + root := t.TempDir() + for name, tc := range map[string]struct { + mediaType string + blob string + }{ + "oci_suffix_zstd": {ociZstdMediaType, writeZstdTarLayer(t, root, "oci.tar.zst", fileEntry("f.txt", "zstd"))}, + "docker_tar_zstd": {dockerZstdMediaType, writeZstdTarLayer(t, root, "docker.tar.zstd", fileEntry("f.txt", "zstd"))}, + "docker_tar_gzip": {dockerGzipMediaType, writeLayerBlob(t, root, "docker.tar.gz", fileEntry("f.txt", "gzip"))}, + "plain_tar": {ociPlainTarMediaType, writeRawTarLayer(t, root, "plain.tar", fileEntry("f.txt", "raw"))}, + "unknown_media_raw": {"application/vnd.custom.tar", writeRawTarLayer(t, root, "custom.tar", fileEntry("f.txt", "raw"))}, + } { + t.Run(name, func(t *testing.T) { + dest := filepath.Join(root, t.Name()) + stats, err := unpackLayerBlob(context.Background(), tc.blob, tc.mediaType, dest, composeOnDiskFormat()) + require.NoError(t, err) + require.Greater(t, stats.unpackedBytes, int64(0)) + require.FileExists(t, filepath.Join(dest, "f.txt")) + }) + } +} + +func TestSmokeRejectsBadLayerHex(t *testing.T) { + p := paths.New(t.TempDir()) + m := &manager{paths: p} + for _, digest := range []string{ + "sha256:..", + "sha256:../escape", + "sha256:a/b", + "sha256:", + } { + _, err := m.materializeLayerArtifact(context.Background(), layerDescriptor{ + Digest: digest, + MediaType: testTarGzMediaType, + }) + require.ErrorContains(t, err, "invalid layer digest", "digest %q must be rejected", digest) + } + require.NoDirExists(t, filepath.Join(p.ImageLayersDir(), "..", "layers-escape")) + + // A non-digest string without the sha256: prefix passes path validation + // and fails at blob lookup instead — acceptable, since real descriptors + // always carry manifest-parsed digests. + _, err := m.materializeLayerArtifact(context.Background(), layerDescriptor{ + Digest: "notadigest", + MediaType: testTarGzMediaType, + }) + require.ErrorContains(t, err, "missing from oci cache") +} + +func TestSmokeRebuildWhenArtifactFileMissing(t *testing.T) { + if _, err := exec.LookPath("mkfs.ext4"); err != nil { + t.Skip("mkfs.ext4 not available") + } + originalFormat := DefaultImageFormat + DefaultImageFormat = FormatExt4 + t.Cleanup(func() { DefaultImageFormat = originalFormat }) + + p := paths.New(t.TempDir()) + img, err := mutate.AppendLayers(empty.Image, syntheticLayer(t, "f.txt", "content")) + require.NoError(t, err) + writeLayerTestLayout(t, p, img) + desc := layerDescFromImage(t, img, 0) + m := &manager{paths: p} + + first, err := m.materializeLayerArtifact(context.Background(), desc) + require.NoError(t, err) + layerHex := desc.Digest[len("sha256:"):] + + // Record survives but the artifact file is gone: must rebuild, not reuse. + require.NoError(t, os.Remove(p.ImageLayerArtifactForFormat(layerHex, layerArtifactFormat()))) + second, err := m.materializeLayerArtifact(context.Background(), desc) + require.NoError(t, err) + require.True(t, second.CreatedAt.After(first.CreatedAt), "missing artifact must trigger a rebuild") + require.FileExists(t, p.ImageLayerArtifactForFormat(layerHex, layerArtifactFormat())) + + // Truncated artifact with a valid record: reuse only checks existence, so + // this documents current behavior (record says nothing about integrity). + require.NoError(t, os.Truncate(p.ImageLayerArtifactForFormat(layerHex, layerArtifactFormat()), 10)) + third, err := m.materializeLayerArtifact(context.Background(), desc) + require.NoError(t, err) + require.True(t, third.CreatedAt.Equal(second.CreatedAt), "truncated artifact is currently reused; integrity is trusted from install-time atomicity") +} + +func TestSmokeConcurrentMaterializeSharesFlight(t *testing.T) { + if _, err := exec.LookPath("mkfs.ext4"); err != nil { + t.Skip("mkfs.ext4 not available") + } + originalFormat := DefaultImageFormat + DefaultImageFormat = FormatExt4 + t.Cleanup(func() { DefaultImageFormat = originalFormat }) + + p := paths.New(t.TempDir()) + img, err := mutate.AppendLayers(empty.Image, syntheticLayer(t, "f.txt", "shared")) + require.NoError(t, err) + writeLayerTestLayout(t, p, img) + desc := layerDescFromImage(t, img, 0) + m := &manager{paths: p} + + const callers = 8 + var wg sync.WaitGroup + records := make([]*layerArtifact, callers) + errs := make([]error, callers) + start := time.Now() + for i := 0; i < callers; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + records[i], errs[i] = m.materializeLayerArtifact(context.Background(), desc) + }(i) + } + wg.Wait() + require.NotEmpty(t, errs) + for i := range errs { + require.NoError(t, errs[i]) + require.Equal(t, records[0], records[i], "all callers must share one build result") + } + t.Logf("8 callers finished in %s sharing one flight", time.Since(start)) + layerHex := desc.Digest[len("sha256:"):] + require.FileExists(t, p.ImageLayerArtifactForFormat(layerHex, layerArtifactFormat())) +} + +func TestSmokeDeviceAndFifoEntries(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("device and fifo tar entries need root to mknod") + } + root := t.TempDir() + blob := writeRawTarLayer(t, root, "devices.tar", + fileEntry("regular.txt", "x"), + func(t *testing.T, tw *tar.Writer) { + require.NoError(t, tw.WriteHeader(&tar.Header{ + Name: "dev/null", Typeflag: tar.TypeChar, + Devmajor: 1, Devminor: 3, Mode: 0666, + })) + }, + func(t *testing.T, tw *tar.Writer) { + require.NoError(t, tw.WriteHeader(&tar.Header{ + Name: "pipes/fifo", Typeflag: tar.TypeFifo, Mode: 0644, + })) + }, + ) + dest := filepath.Join(root, "dest") + _, err := unpackLayerBlob(context.Background(), blob, ociPlainTarMediaType, dest, composeOnDiskFormat()) + require.NoError(t, err) + + require.FileExists(t, filepath.Join(dest, "regular.txt")) + var stat unix.Stat_t + require.NoError(t, unix.Lstat(filepath.Join(dest, "dev", "null"), &stat)) + require.Equal(t, uint32(unix.S_IFCHR), stat.Mode&unix.S_IFMT) + require.Equal(t, uint64(0x103), uint64(stat.Rdev), "char device 1:3") + require.NoError(t, unix.Lstat(filepath.Join(dest, "pipes", "fifo"), &stat)) + require.Equal(t, uint32(unix.S_IFIFO), stat.Mode&unix.S_IFMT) +} + +func TestSmokeDiskUsageCountsArtifactsOnly(t *testing.T) { + p := paths.New(t.TempDir()) + layersDir := p.ImageLayersDir() + require.NoError(t, os.MkdirAll(filepath.Join(layersDir, "abab", ".unpack-123"), 0755)) + write := func(rel string, size int) { + require.NoError(t, os.WriteFile(filepath.Join(layersDir, rel), make([]byte, size), 0644)) + } + write(filepath.Join("abab", "layer.erofs"), 100) + write(filepath.Join("abab", "artifact.erofs.json"), 50) + write(filepath.Join("abab", ".unpack-123", "layer.erofs"), 999) + write("stray.txt", 500) + + total, err := totalLayerArtifactBytesFromFilesystem(layersDir) + require.NoError(t, err) + require.Equal(t, int64(100), total, "only materialized artifacts count; temp dirs and records are excluded") + + // Missing directory is not an error. + total, err = totalLayerArtifactBytesFromFilesystem(filepath.Join(p.DataDir(), "nope")) + require.NoError(t, err) + require.Equal(t, int64(0), total) +} From f18a718d8a457a12799d201db3680cfe6cda3176 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Thu, 3 Sep 2026 17:18:07 +0000 Subject: [PATCH 21/41] Trim redundant smoke tests Drop the disk-usage accounting case (covered by TestTotalLayerArtifactBytesFromFilesystem) and the truncated-artifact documentation case. --- lib/images/layer_artifact_smoke_test.go | 29 ------------------------- 1 file changed, 29 deletions(-) diff --git a/lib/images/layer_artifact_smoke_test.go b/lib/images/layer_artifact_smoke_test.go index 9f0d30b5c..ebbee631a 100644 --- a/lib/images/layer_artifact_smoke_test.go +++ b/lib/images/layer_artifact_smoke_test.go @@ -137,13 +137,6 @@ func TestSmokeRebuildWhenArtifactFileMissing(t *testing.T) { require.NoError(t, err) require.True(t, second.CreatedAt.After(first.CreatedAt), "missing artifact must trigger a rebuild") require.FileExists(t, p.ImageLayerArtifactForFormat(layerHex, layerArtifactFormat())) - - // Truncated artifact with a valid record: reuse only checks existence, so - // this documents current behavior (record says nothing about integrity). - require.NoError(t, os.Truncate(p.ImageLayerArtifactForFormat(layerHex, layerArtifactFormat()), 10)) - third, err := m.materializeLayerArtifact(context.Background(), desc) - require.NoError(t, err) - require.True(t, third.CreatedAt.Equal(second.CreatedAt), "truncated artifact is currently reused; integrity is trusted from install-time atomicity") } func TestSmokeConcurrentMaterializeSharesFlight(t *testing.T) { @@ -215,25 +208,3 @@ func TestSmokeDeviceAndFifoEntries(t *testing.T) { require.NoError(t, unix.Lstat(filepath.Join(dest, "pipes", "fifo"), &stat)) require.Equal(t, uint32(unix.S_IFIFO), stat.Mode&unix.S_IFMT) } - -func TestSmokeDiskUsageCountsArtifactsOnly(t *testing.T) { - p := paths.New(t.TempDir()) - layersDir := p.ImageLayersDir() - require.NoError(t, os.MkdirAll(filepath.Join(layersDir, "abab", ".unpack-123"), 0755)) - write := func(rel string, size int) { - require.NoError(t, os.WriteFile(filepath.Join(layersDir, rel), make([]byte, size), 0644)) - } - write(filepath.Join("abab", "layer.erofs"), 100) - write(filepath.Join("abab", "artifact.erofs.json"), 50) - write(filepath.Join("abab", ".unpack-123", "layer.erofs"), 999) - write("stray.txt", 500) - - total, err := totalLayerArtifactBytesFromFilesystem(layersDir) - require.NoError(t, err) - require.Equal(t, int64(100), total, "only materialized artifacts count; temp dirs and records are excluded") - - // Missing directory is not an error. - total, err = totalLayerArtifactBytesFromFilesystem(filepath.Join(p.DataDir(), "nope")) - require.NoError(t, err) - require.Equal(t, int64(0), total) -} From cda396560df11be20dc6f3e13fb90e56e432e2d0 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 8 Sep 2026 15:51:09 +0000 Subject: [PATCH 22/41] Harden shared layer materialization --- lib/images/disk.go | 20 +++++++++++++------- lib/images/disk_usage.go | 6 ++++++ lib/images/layer_artifact.go | 21 ++++++++++++--------- lib/resources/resource.go | 2 +- 4 files changed, 32 insertions(+), 17 deletions(-) diff --git a/lib/images/disk.go b/lib/images/disk.go index 896d10f2a..f0f90dc1b 100644 --- a/lib/images/disk.go +++ b/lib/images/disk.go @@ -1,6 +1,7 @@ package images import ( + "context" "fmt" "os" "os/exec" @@ -29,13 +30,18 @@ var DefaultImageFormat = func() ExportFormat { return FormatErofs }() -// ExportRootfs exports rootfs directory in specified format (public for system manager) +// ExportRootfs exports rootfs directory in specified format (public for system manager). func ExportRootfs(rootfsDir, outputPath string, format ExportFormat) (int64, error) { + return ExportRootfsWithContext(context.Background(), rootfsDir, outputPath, format) +} + +// ExportRootfsWithContext exports rootfs directory and cancels external formatters with ctx. +func ExportRootfsWithContext(ctx context.Context, rootfsDir, outputPath string, format ExportFormat) (int64, error) { switch format { case FormatExt4: - return convertToExt4(rootfsDir, outputPath) + return convertToExt4(ctx, rootfsDir, outputPath) case FormatErofs: - return convertToErofs(rootfsDir, outputPath) + return convertToErofs(ctx, rootfsDir, outputPath) case FormatCpio: return convertToCpio(rootfsDir, outputPath) default: @@ -128,7 +134,7 @@ func alignToSector(size int64) int64 { } // convertToExt4 converts a rootfs directory to an ext4 disk image using mkfs.ext4 -func convertToExt4(rootfsDir, diskPath string) (int64, error) { +func convertToExt4(ctx context.Context, rootfsDir, diskPath string) (int64, error) { // Calculate size of rootfs directory sizeBytes, err := dirSize(rootfsDir) if err != nil { @@ -168,7 +174,7 @@ func convertToExt4(rootfsDir, diskPath string) (int64, error) { // -O ^has_journal: Disable journal (not needed for read-only VM mounts) // -d: Copy directory contents into filesystem // -F: Force creation (file not block device) - cmd := exec.Command(mkfsExt4Binary(), "-b", "4096", "-O", "^has_journal", "-d", rootfsDir, "-F", diskPath) + cmd := exec.CommandContext(ctx, mkfsExt4Binary(), "-b", "4096", "-O", "^has_journal", "-d", rootfsDir, "-F", diskPath) output, err := cmd.CombinedOutput() if err != nil { return 0, fmt.Errorf("mkfs.ext4 failed: %w, output: %s", err, output) @@ -193,7 +199,7 @@ func convertToExt4(rootfsDir, diskPath string) (int64, error) { } // convertToErofs converts a rootfs directory to an erofs disk image using mkfs.erofs -func convertToErofs(rootfsDir, diskPath string) (int64, error) { +func convertToErofs(ctx context.Context, rootfsDir, diskPath string) (int64, error) { // Ensure parent directory exists if err := os.MkdirAll(filepath.Dir(diskPath), 0755); err != nil { return 0, fmt.Errorf("create disk parent dir: %w", err) @@ -202,7 +208,7 @@ func convertToErofs(rootfsDir, diskPath string) (int64, error) { // Create erofs image with LZ4 fast compression // -zlz4: LZ4 fast compression (~20-25% space savings, faster builds) // erofs doesn't need pre-allocation, creates file directly - cmd := exec.Command("mkfs.erofs", "-zlz4", diskPath, rootfsDir) + cmd := exec.CommandContext(ctx, "mkfs.erofs", "-zlz4", diskPath, rootfsDir) output, err := cmd.CombinedOutput() if err != nil { return 0, fmt.Errorf("mkfs.erofs failed: %w, output: %s", err, output) diff --git a/lib/images/disk_usage.go b/lib/images/disk_usage.go index 0cb6a74d6..90c49d1e0 100644 --- a/lib/images/disk_usage.go +++ b/lib/images/disk_usage.go @@ -170,6 +170,12 @@ func (m *manager) getDiskUsageTotals() (int64, int64, error) { return readyImageBytes, ociCacheBytes, nil } +func (m *manager) invalidateDiskUsageTotals() { + m.diskUsageMu.Lock() + m.diskUsageLoaded = false + m.diskUsageMu.Unlock() +} + func (m *manager) refreshDiskUsageTotals() { readyImageBytes, ociCacheBytes, err := m.computeDiskUsageTotals() if err != nil { diff --git a/lib/images/layer_artifact.go b/lib/images/layer_artifact.go index d7b79fe19..70ae5860b 100644 --- a/lib/images/layer_artifact.go +++ b/lib/images/layer_artifact.go @@ -157,13 +157,11 @@ func discardLayerCache(p *paths.Paths, layerHex string) error { // its own context is done. func (m *manager) materializeLayerArtifact(ctx context.Context, desc layerDescriptor) (*layerArtifact, error) { key := desc.Digest + "\x00" + layerArtifactFormat() - // The build outlives the initiating request, so the deadline below is its - // only bound: without it a hung cache-blob read would wedge the - // singleflight key, and every future caller for the layer, forever. - buildCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), layerBuildTimeout) result := m.layerFlights.DoChan(key, func() (any, error) { - // Cancel here, in the flight's frame: the initiating caller may return - // long before the detached build finishes. + // The build outlives the initiating request, so this deadline is its + // only bound: without it a hung cache-blob read would wedge the + // singleflight key, and every future caller for the layer, forever. + buildCtx, cancel := context.WithTimeout(context.Background(), layerBuildTimeout) defer cancel() return m.materializeLayerArtifactOnce(buildCtx, desc) }) @@ -219,10 +217,15 @@ func (m *manager) materializeLayerArtifactOnce(ctx context.Context, desc layerDe if err != nil { return nil, err } - return m.installLayerArtifact(desc, layerHex, unpackDir, stats) + record, err := m.installLayerArtifact(ctx, desc, layerHex, unpackDir, stats) + if err != nil { + return nil, err + } + m.invalidateDiskUsageTotals() + return record, nil } -func (m *manager) installLayerArtifact(desc layerDescriptor, layerHex, unpackDir string, stats *unpackStats) (*layerArtifact, error) { +func (m *manager) installLayerArtifact(ctx context.Context, desc layerDescriptor, layerHex, unpackDir string, stats *unpackStats) (*layerArtifact, error) { record := &layerArtifact{ SchemaVersion: layerRecordSchemaVersion, Digest: desc.Digest, @@ -233,7 +236,7 @@ func (m *manager) installLayerArtifact(desc layerDescriptor, layerHex, unpackDir } if err := installAtomically(layerArtifactPath(m.paths, layerHex), func(path string) error { - size, err := ExportRootfs(unpackDir, path, DefaultImageFormat) + size, err := ExportRootfsWithContext(ctx, unpackDir, path, DefaultImageFormat) if err != nil { return err } diff --git a/lib/resources/resource.go b/lib/resources/resource.go index 5943cb0bc..f8f624e93 100644 --- a/lib/resources/resource.go +++ b/lib/resources/resource.go @@ -90,7 +90,7 @@ type AllocationBreakdown struct { // DiskBreakdown shows disk usage by category. type DiskBreakdown struct { Images int64 `json:"images_bytes"` // Exported rootfs disk files - OCICache int64 `json:"oci_cache_bytes"` // OCI layer cache (shared blobs) + OCICache int64 `json:"oci_cache_bytes"` // OCI blobs and materialized shared layers Volumes int64 `json:"volumes_bytes"` Overlays int64 `json:"overlays_bytes"` // Rootfs overlays + volume overlays } From bd685142353c96c5b09c181005e94c9c36deb4d5 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 8 Sep 2026 18:07:51 +0000 Subject: [PATCH 23/41] Harden layer cache validation --- lib/images/disk_usage.go | 16 ++++++++++---- lib/images/layer_artifact.go | 29 +++++++++++++++++-------- lib/images/layer_artifact_smoke_test.go | 5 +---- lib/images/layer_artifact_test.go | 12 +++++++++- lib/images/manager.go | 1 + 5 files changed, 45 insertions(+), 18 deletions(-) diff --git a/lib/images/disk_usage.go b/lib/images/disk_usage.go index 90c49d1e0..b7df8f238 100644 --- a/lib/images/disk_usage.go +++ b/lib/images/disk_usage.go @@ -144,6 +144,7 @@ func totalOCICacheBlobBytesFromFilesystem(blobDir string) (int64, error) { func (m *manager) getDiskUsageTotals() (int64, int64, error) { m.diskUsageMu.RLock() + generation := m.diskUsageGeneration if m.diskUsageLoaded { readyImageBytes := m.readyImageBytes ociCacheBytes := m.ociCacheBytes @@ -158,7 +159,7 @@ func (m *manager) getDiskUsageTotals() (int64, int64, error) { } m.diskUsageMu.Lock() - if !m.diskUsageLoaded { + if m.diskUsageGeneration == generation && !m.diskUsageLoaded { m.readyImageBytes = readyImageBytes m.ociCacheBytes = ociCacheBytes m.diskUsageLoaded = true @@ -172,20 +173,27 @@ func (m *manager) getDiskUsageTotals() (int64, int64, error) { func (m *manager) invalidateDiskUsageTotals() { m.diskUsageMu.Lock() + m.diskUsageGeneration++ m.diskUsageLoaded = false m.diskUsageMu.Unlock() } func (m *manager) refreshDiskUsageTotals() { + m.diskUsageMu.RLock() + generation := m.diskUsageGeneration + m.diskUsageMu.RUnlock() + readyImageBytes, ociCacheBytes, err := m.computeDiskUsageTotals() if err != nil { return } m.diskUsageMu.Lock() - m.readyImageBytes = readyImageBytes - m.ociCacheBytes = ociCacheBytes - m.diskUsageLoaded = true + if m.diskUsageGeneration == generation { + m.readyImageBytes = readyImageBytes + m.ociCacheBytes = ociCacheBytes + m.diskUsageLoaded = true + } m.diskUsageMu.Unlock() } diff --git a/lib/images/layer_artifact.go b/lib/images/layer_artifact.go index 70ae5860b..d384f7f32 100644 --- a/lib/images/layer_artifact.go +++ b/lib/images/layer_artifact.go @@ -92,6 +92,14 @@ func layerArtifactRecordPath(p *paths.Paths, layerHex string) string { return p.ImageLayerRecordForFormat(layerHex, layerArtifactFormat()) } +func layerDigestHex(value string) (string, error) { + digest, err := parseSHA256Digest(value) + if err != nil || digest.String() != value { + return "", fmt.Errorf("invalid layer digest %q", value) + } + return digest.Encoded(), nil +} + // layerMapOptions preserves tar ownership when running as root. Otherwise // umoci's rootless mode skips chown and stands in empty files for device nodes. // Unlike unpackLayers in oci.go, which maps container root to the current @@ -135,7 +143,7 @@ func discardLayerCache(p *paths.Paths, layerHex string) error { layerArtifactRecordPath(p, layerHex), layerArtifactPath(p, layerHex), } { - if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + if err := removePath(path); err != nil && !os.IsNotExist(err) { return err } } @@ -156,6 +164,10 @@ func discardLayerCache(p *paths.Paths, layerHex string) error { // other pull waiting on the same layer; each caller still returns as soon as // its own context is done. func (m *manager) materializeLayerArtifact(ctx context.Context, desc layerDescriptor) (*layerArtifact, error) { + layerHex, err := layerDigestHex(desc.Digest) + if err != nil { + return nil, err + } key := desc.Digest + "\x00" + layerArtifactFormat() result := m.layerFlights.DoChan(key, func() (any, error) { // The build outlives the initiating request, so this deadline is its @@ -163,7 +175,7 @@ func (m *manager) materializeLayerArtifact(ctx context.Context, desc layerDescri // singleflight key, and every future caller for the layer, forever. buildCtx, cancel := context.WithTimeout(context.Background(), layerBuildTimeout) defer cancel() - return m.materializeLayerArtifactOnce(buildCtx, desc) + return m.materializeLayerArtifactOnce(buildCtx, desc, layerHex) }) select { case <-ctx.Done(): @@ -176,14 +188,10 @@ func (m *manager) materializeLayerArtifact(ctx context.Context, desc layerDescri } } -func (m *manager) materializeLayerArtifactOnce(ctx context.Context, desc layerDescriptor) (*layerArtifact, error) { +func (m *manager) materializeLayerArtifactOnce(ctx context.Context, desc layerDescriptor, layerHex string) (*layerArtifact, error) { if layerArtifactFormat() == "" { return nil, fmt.Errorf("unsupported layer artifact format: %s", DefaultImageFormat) } - layerHex := strings.TrimPrefix(desc.Digest, "sha256:") - if err := paths.ValidatePathComponent(layerHex); err != nil { - return nil, fmt.Errorf("invalid layer digest %s: %w", desc.Digest, err) - } if record, err := readLayerRecord(m.paths, layerHex); err != nil { if !errors.Is(err, errCorruptLayerRecord) { @@ -193,10 +201,13 @@ func (m *manager) materializeLayerArtifactOnce(ctx context.Context, desc layerDe return nil, fmt.Errorf("discard corrupt layer cache: %w", discardErr) } } else if record != nil && record.matches(desc) { - if _, statErr := os.Stat(layerArtifactPath(m.paths, layerHex)); statErr == nil { + artifactInfo, statErr := os.Lstat(layerArtifactPath(m.paths, layerHex)) + if statErr == nil && artifactInfo.Mode().IsRegular() && record.SizeBytes > 0 && artifactInfo.Size() == record.SizeBytes { return record, nil } - // Record without artifact: rebuild below. + if discardErr := discardLayerCache(m.paths, layerHex); discardErr != nil { + return nil, fmt.Errorf("discard invalid layer cache: %w", discardErr) + } } layerDir := m.paths.ImageLayerDir(layerHex) diff --git a/lib/images/layer_artifact_smoke_test.go b/lib/images/layer_artifact_smoke_test.go index ebbee631a..f4d6987d3 100644 --- a/lib/images/layer_artifact_smoke_test.go +++ b/lib/images/layer_artifact_smoke_test.go @@ -102,14 +102,11 @@ func TestSmokeRejectsBadLayerHex(t *testing.T) { } require.NoDirExists(t, filepath.Join(p.ImageLayersDir(), "..", "layers-escape")) - // A non-digest string without the sha256: prefix passes path validation - // and fails at blob lookup instead — acceptable, since real descriptors - // always carry manifest-parsed digests. _, err := m.materializeLayerArtifact(context.Background(), layerDescriptor{ Digest: "notadigest", MediaType: testTarGzMediaType, }) - require.ErrorContains(t, err, "missing from oci cache") + require.ErrorContains(t, err, "invalid layer digest") } func TestSmokeRebuildWhenArtifactFileMissing(t *testing.T) { diff --git a/lib/images/layer_artifact_test.go b/lib/images/layer_artifact_test.go index 2ba416031..555fab776 100644 --- a/lib/images/layer_artifact_test.go +++ b/lib/images/layer_artifact_test.go @@ -167,7 +167,8 @@ func TestMaterializeLayerArtifact(t *testing.T) { require.Greater(t, record.UnpackedBytes, int64(0)) layerHex := desc.Digest[len("sha256:"):] - artifactInfo, err := os.Stat(p.ImageLayerArtifactForFormat(layerHex, layerArtifactFormat())) + artifactPath := p.ImageLayerArtifactForFormat(layerHex, layerArtifactFormat()) + artifactInfo, err := os.Stat(artifactPath) require.NoError(t, err, "layer.erofs must be installed") // A second materialization reuses the existing artifact. @@ -177,6 +178,15 @@ func TestMaterializeLayerArtifact(t *testing.T) { artifactInfoAfter, err := os.Stat(p.ImageLayerArtifactForFormat(layerHex, layerArtifactFormat())) require.NoError(t, err) require.Equal(t, artifactInfo.ModTime(), artifactInfoAfter.ModTime(), "reuse must not rebuild") + + // A truncated artifact must not be reused just because its record is valid. + require.NoError(t, os.Truncate(artifactPath, 0)) + rebuilt, err := m.materializeLayerArtifact(context.Background(), desc) + require.NoError(t, err) + require.NotEqual(t, reused.CreatedAt, rebuilt.CreatedAt, "invalid artifact must rebuild") + rebuiltInfo, err := os.Stat(artifactPath) + require.NoError(t, err) + require.Equal(t, rebuilt.SizeBytes, rebuiltInfo.Size()) } func TestMaterializeLayerArtifactRecoversCorruptRecord(t *testing.T) { diff --git a/lib/images/manager.go b/lib/images/manager.go index bceeb0f05..a0fa01736 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -78,6 +78,7 @@ type manager struct { createMu sync.Mutex layerFlights singleflight.Group diskUsageMu sync.RWMutex + diskUsageGeneration uint64 tagGenerations map[string]uint64 requestedTags map[string]string // newest pull's digest per requested tag diskUsageLoaded bool From 9e817ef926ebda761b9f787d987bc257959bb848 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 8 Sep 2026 18:22:09 +0000 Subject: [PATCH 24/41] Retry disk usage scans after invalidation --- lib/images/disk_usage.go | 50 ++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/lib/images/disk_usage.go b/lib/images/disk_usage.go index b7df8f238..07cafd92a 100644 --- a/lib/images/disk_usage.go +++ b/lib/images/disk_usage.go @@ -143,32 +143,38 @@ func totalOCICacheBlobBytesFromFilesystem(blobDir string) (int64, error) { } func (m *manager) getDiskUsageTotals() (int64, int64, error) { - m.diskUsageMu.RLock() - generation := m.diskUsageGeneration - if m.diskUsageLoaded { - readyImageBytes := m.readyImageBytes - ociCacheBytes := m.ociCacheBytes + for { + m.diskUsageMu.RLock() + generation := m.diskUsageGeneration + if m.diskUsageLoaded { + readyImageBytes := m.readyImageBytes + ociCacheBytes := m.ociCacheBytes + m.diskUsageMu.RUnlock() + return readyImageBytes, ociCacheBytes, nil + } m.diskUsageMu.RUnlock() - return readyImageBytes, ociCacheBytes, nil - } - m.diskUsageMu.RUnlock() - readyImageBytes, ociCacheBytes, err := m.computeDiskUsageTotals() - if err != nil { - return 0, 0, err - } + readyImageBytes, ociCacheBytes, err := m.computeDiskUsageTotals() + if err != nil { + return 0, 0, err + } - m.diskUsageMu.Lock() - if m.diskUsageGeneration == generation && !m.diskUsageLoaded { - m.readyImageBytes = readyImageBytes - m.ociCacheBytes = ociCacheBytes - m.diskUsageLoaded = true - } - readyImageBytes = m.readyImageBytes - ociCacheBytes = m.ociCacheBytes - m.diskUsageMu.Unlock() + m.diskUsageMu.Lock() + if m.diskUsageGeneration != generation { + m.diskUsageMu.Unlock() + continue + } + if !m.diskUsageLoaded { + m.readyImageBytes = readyImageBytes + m.ociCacheBytes = ociCacheBytes + m.diskUsageLoaded = true + } + readyImageBytes = m.readyImageBytes + ociCacheBytes = m.ociCacheBytes + m.diskUsageMu.Unlock() - return readyImageBytes, ociCacheBytes, nil + return readyImageBytes, ociCacheBytes, nil + } } func (m *manager) invalidateDiskUsageTotals() { From 10197537688044de8eb499625e5252521223e588 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 8 Sep 2026 18:34:28 +0000 Subject: [PATCH 25/41] Detect hosts that support layer artifacts --- lib/images/layer_artifact.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/images/layer_artifact.go b/lib/images/layer_artifact.go index d384f7f32..48b547451 100644 --- a/lib/images/layer_artifact.go +++ b/lib/images/layer_artifact.go @@ -12,6 +12,7 @@ import ( "log/slog" "os" "path/filepath" + "runtime" "strings" "time" @@ -109,10 +110,18 @@ func layerMapOptions() layer.MapOptions { return layer.MapOptions{Rootless: os.Geteuid() != 0} } +// supportsLayerArtifacts reports whether the host can create the native +// overlayfs representation used by per-layer artifacts. Other platforms and +// rootless processes must compose the OCI layers into one rootfs instead. +func supportsLayerArtifacts() bool { + return runtime.GOOS == "linux" && os.Geteuid() == 0 +} + // layerArtifactOnDiskFormat is the extraction format for per-layer artifacts. // Whiteouts become overlayfs whiteout inodes and opaque xattrs, the form an // overlayfs mount of stacked layers understands, so the artifact retains the -// layer's deletions without a private marker format. +// layer's deletions without a private marker format. Callers must check +// supportsLayerArtifacts before materializing this format. func layerArtifactOnDiskFormat() layer.OnDiskFormat { return layer.OverlayfsRootfs{MapOptions: layerMapOptions()} } From f99495aefd3986994fc4253d050c6be1b48b4894 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 8 Sep 2026 19:06:50 +0000 Subject: [PATCH 26/41] Harden layer artifact accounting and cancellation --- lib/images/disk.go | 11 +++++++++-- lib/images/disk_usage.go | 3 +++ lib/images/layer_artifact.go | 3 +++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/lib/images/disk.go b/lib/images/disk.go index f0f90dc1b..f092f0e8d 100644 --- a/lib/images/disk.go +++ b/lib/images/disk.go @@ -136,7 +136,7 @@ func alignToSector(size int64) int64 { // convertToExt4 converts a rootfs directory to an ext4 disk image using mkfs.ext4 func convertToExt4(ctx context.Context, rootfsDir, diskPath string) (int64, error) { // Calculate size of rootfs directory - sizeBytes, err := dirSize(rootfsDir) + sizeBytes, err := dirSizeWithContext(ctx, rootfsDir) if err != nil { return 0, fmt.Errorf("calculate dir size: %w", err) } @@ -232,13 +232,20 @@ func convertToErofs(ctx context.Context, rootfsDir, diskPath string) (int64, err return stat.Size(), nil } -// dirSize calculates the total size of a directory +// dirSize calculates the total size of a directory. func dirSize(path string) (int64, error) { + return dirSizeWithContext(context.Background(), path) +} + +func dirSizeWithContext(ctx context.Context, path string) (int64, error) { var size int64 err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error { if err != nil { return err } + if err := ctx.Err(); err != nil { + return err + } if !info.IsDir() { size += info.Size() } diff --git a/lib/images/disk_usage.go b/lib/images/disk_usage.go index 07cafd92a..14eb58545 100644 --- a/lib/images/disk_usage.go +++ b/lib/images/disk_usage.go @@ -105,6 +105,9 @@ func totalLayerArtifactBytesFromFilesystem(layersDir string) (int64, error) { } info, err := d.Info() if err != nil { + if os.IsNotExist(err) { + return nil + } return err } total += info.Size() diff --git a/lib/images/layer_artifact.go b/lib/images/layer_artifact.go index 48b547451..64f8a4ed7 100644 --- a/lib/images/layer_artifact.go +++ b/lib/images/layer_artifact.go @@ -206,6 +206,7 @@ func (m *manager) materializeLayerArtifactOnce(ctx context.Context, desc layerDe if !errors.Is(err, errCorruptLayerRecord) { return nil, err } + m.invalidateDiskUsageTotals() if discardErr := discardLayerCache(m.paths, layerHex); discardErr != nil { return nil, fmt.Errorf("discard corrupt layer cache: %w", discardErr) } @@ -214,6 +215,7 @@ func (m *manager) materializeLayerArtifactOnce(ctx context.Context, desc layerDe if statErr == nil && artifactInfo.Mode().IsRegular() && record.SizeBytes > 0 && artifactInfo.Size() == record.SizeBytes { return record, nil } + m.invalidateDiskUsageTotals() if discardErr := discardLayerCache(m.paths, layerHex); discardErr != nil { return nil, fmt.Errorf("discard invalid layer cache: %w", discardErr) } @@ -239,6 +241,7 @@ func (m *manager) materializeLayerArtifactOnce(ctx context.Context, desc layerDe } record, err := m.installLayerArtifact(ctx, desc, layerHex, unpackDir, stats) if err != nil { + m.invalidateDiskUsageTotals() return nil, err } m.invalidateDiskUsageTotals() From 85afb94306fc55e9e274fde2f97d7beaeb34ce39 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 8 Sep 2026 20:00:37 +0000 Subject: [PATCH 27/41] Harden layer artifact capabilities and integrity --- lib/images/disk_usage.go | 42 +++++++++------ lib/images/disk_usage_test.go | 4 +- lib/images/layer_artifact.go | 89 ++++++++++++++++++++++++++----- lib/images/layer_artifact_test.go | 53 ++++++++++++++++++ lib/images/manager.go | 10 ++++ 5 files changed, 169 insertions(+), 29 deletions(-) diff --git a/lib/images/disk_usage.go b/lib/images/disk_usage.go index 14eb58545..67ef07a5e 100644 --- a/lib/images/disk_usage.go +++ b/lib/images/disk_usage.go @@ -6,7 +6,6 @@ import ( "io/fs" "os" "path/filepath" - "strings" "syscall" ) @@ -84,7 +83,8 @@ func totalReadyImageBytesFromMetadata(imagesDir string) (int64, error) { return total, nil } -// totalLayerArtifactBytesFromFilesystem sums materialized layer artifacts. +// totalLayerArtifactBytesFromFilesystem sums the layer store, including +// records and in-progress extraction/install trees. func totalLayerArtifactBytesFromFilesystem(layersDir string) (int64, error) { var total int64 err := filepath.WalkDir(layersDir, func(path string, d fs.DirEntry, err error) error { @@ -95,12 +95,6 @@ func totalLayerArtifactBytesFromFilesystem(layersDir string) (int64, error) { return err } if d.IsDir() { - if strings.HasPrefix(d.Name(), ".") && path != layersDir { - return filepath.SkipDir - } - return nil - } - if !strings.HasPrefix(d.Name(), "layer.") { return nil } info, err := d.Info() @@ -167,13 +161,15 @@ func (m *manager) getDiskUsageTotals() (int64, int64, error) { m.diskUsageMu.Unlock() continue } - if !m.diskUsageLoaded { - m.readyImageBytes = readyImageBytes - m.ociCacheBytes = ociCacheBytes - m.diskUsageLoaded = true + if m.activeLayerBuilds == 0 { + if !m.diskUsageLoaded { + m.readyImageBytes = readyImageBytes + m.ociCacheBytes = ociCacheBytes + m.diskUsageLoaded = true + } + readyImageBytes = m.readyImageBytes + ociCacheBytes = m.ociCacheBytes } - readyImageBytes = m.readyImageBytes - ociCacheBytes = m.ociCacheBytes m.diskUsageMu.Unlock() return readyImageBytes, ociCacheBytes, nil @@ -187,6 +183,22 @@ func (m *manager) invalidateDiskUsageTotals() { m.diskUsageMu.Unlock() } +func (m *manager) beginLayerBuild() { + m.diskUsageMu.Lock() + m.activeLayerBuilds++ + m.diskUsageGeneration++ + m.diskUsageLoaded = false + m.diskUsageMu.Unlock() +} + +func (m *manager) endLayerBuild() { + m.diskUsageMu.Lock() + m.activeLayerBuilds-- + m.diskUsageGeneration++ + m.diskUsageLoaded = false + m.diskUsageMu.Unlock() +} + func (m *manager) refreshDiskUsageTotals() { m.diskUsageMu.RLock() generation := m.diskUsageGeneration @@ -198,7 +210,7 @@ func (m *manager) refreshDiskUsageTotals() { } m.diskUsageMu.Lock() - if m.diskUsageGeneration == generation { + if m.diskUsageGeneration == generation && m.activeLayerBuilds == 0 { m.readyImageBytes = readyImageBytes m.ociCacheBytes = ociCacheBytes m.diskUsageLoaded = true diff --git a/lib/images/disk_usage_test.go b/lib/images/disk_usage_test.go index 7de35871b..c5d0871cd 100644 --- a/lib/images/disk_usage_test.go +++ b/lib/images/disk_usage_test.go @@ -74,14 +74,14 @@ func TestTotalLayerArtifactBytesFromFilesystem(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(digestDir, "layer.ext4"), []byte("ext4"), 0o644)) require.NoError(t, os.WriteFile(filepath.Join(digestDir, "artifact.erofs.json"), []byte("record"), 0o644)) - // In-progress temp dirs should be skipped. + // In-progress temp dirs and records count toward actual layer-store usage. unpackDir := filepath.Join(digestDir, ".unpack-tmp") require.NoError(t, os.MkdirAll(unpackDir, 0o755)) require.NoError(t, os.WriteFile(filepath.Join(unpackDir, "layer.bin"), []byte("unpacked-content"), 0o644)) total, err := totalLayerArtifactBytesFromFilesystem(layersDir) require.NoError(t, err) - require.Equal(t, int64(len("erofs")+len("ext4")), total) + require.Equal(t, int64(len("erofs")+len("ext4")+len("record")+len("unpacked-content")), total) } func TestTotalReadyImageBytesFromMetadata_UsesRootfsFallbackForReadyImageWithoutSize(t *testing.T) { diff --git a/lib/images/layer_artifact.go b/lib/images/layer_artifact.go index 64f8a4ed7..e47e23685 100644 --- a/lib/images/layer_artifact.go +++ b/lib/images/layer_artifact.go @@ -19,10 +19,11 @@ import ( "github.com/kernel/hypeman/lib/paths" "github.com/klauspost/compress/zstd" "github.com/opencontainers/umoci/oci/layer" + "golang.org/x/sys/unix" ) const ( - layerRecordSchemaVersion = 1 + layerRecordSchemaVersion = 2 maxLayerUnpackedBytes = 100 << 30 // layerBuildTimeout bounds a shared layer build end to end. Builds are @@ -37,13 +38,14 @@ var errCorruptLayerRecord = errors.New("corrupt layer record") // The key is the compressed layer blob digest plus the artifact format, so the // same layer can coexist in several materializations. type layerArtifact struct { - SchemaVersion int `json:"schema_version"` - Digest string `json:"digest"` // compressed layer blob digest, sha256:... - DiffID string `json:"diff_id,omitempty"` - Format string `json:"format"` - SizeBytes int64 `json:"size_bytes"` // artifact bytes on disk - UnpackedBytes int64 `json:"unpacked_bytes"` // decompressed tar stream bytes - CreatedAt time.Time `json:"created_at"` + SchemaVersion int `json:"schema_version"` + Digest string `json:"digest"` // compressed layer blob digest, sha256:... + DiffID string `json:"diff_id,omitempty"` + Format string `json:"format"` + SizeBytes int64 `json:"size_bytes"` // artifact bytes on disk + ArtifactDigest string `json:"artifact_digest"` // sha256 of artifact bytes + UnpackedBytes int64 `json:"unpacked_bytes"` // decompressed tar stream bytes + CreatedAt time.Time `json:"created_at"` } // validate checks a record read back from disk. The format fully determines @@ -53,8 +55,11 @@ func (a *layerArtifact) validate() error { if a.SchemaVersion != layerRecordSchemaVersion { return fmt.Errorf("unsupported schema version: %d", a.SchemaVersion) } - if a.Digest == "" { - return fmt.Errorf("missing digest") + if _, err := parseSHA256Digest(a.Digest); err != nil { + return fmt.Errorf("invalid digest: %w", err) + } + if _, err := parseSHA256Digest(a.ArtifactDigest); err != nil { + return fmt.Errorf("invalid artifact digest: %w", err) } if a.Format != string(FormatErofs) && a.Format != string(FormatExt4) { return fmt.Errorf("invalid format: %s", a.Format) @@ -117,6 +122,35 @@ func supportsLayerArtifacts() bool { return runtime.GOOS == "linux" && os.Geteuid() == 0 } +func probeLayerArtifactSupport(layersDir string) bool { + if !supportsLayerArtifacts() || os.MkdirAll(layersDir, 0755) != nil { + return false + } + probeDir, err := os.MkdirTemp(layersDir, ".probe-*") + if err != nil { + return false + } + defer os.RemoveAll(probeDir) + + whiteout := filepath.Join(probeDir, "whiteout") + if err := unix.Mknod(whiteout, unix.S_IFCHR|0600, int(unix.Mkdev(0, 0))); err != nil { + return false + } + if err := unix.Lsetxattr(probeDir, "trusted.overlay.opaque", []byte("y"), 0); err != nil { + return false + } + value := make([]byte, 1) + n, err := unix.Lgetxattr(probeDir, "trusted.overlay.opaque", value) + return err == nil && n == 1 && value[0] == 'y' +} + +func (m *manager) layerArtifactSupport() bool { + if m.layerArtifactSupportKnown { + return m.layerArtifactsSupported + } + return supportsLayerArtifacts() +} + // layerArtifactOnDiskFormat is the extraction format for per-layer artifacts. // Whiteouts become overlayfs whiteout inodes and opaque xattrs, the form an // overlayfs mount of stacked layers understands, so the artifact retains the @@ -184,6 +218,14 @@ func (m *manager) materializeLayerArtifact(ctx context.Context, desc layerDescri // singleflight key, and every future caller for the layer, forever. buildCtx, cancel := context.WithTimeout(context.Background(), layerBuildTimeout) defer cancel() + if m.layerBuilds != nil { + select { + case m.layerBuilds <- struct{}{}: + defer func() { <-m.layerBuilds }() + case <-buildCtx.Done(): + return nil, buildCtx.Err() + } + } return m.materializeLayerArtifactOnce(buildCtx, desc, layerHex) }) select { @@ -201,6 +243,9 @@ func (m *manager) materializeLayerArtifactOnce(ctx context.Context, desc layerDe if layerArtifactFormat() == "" { return nil, fmt.Errorf("unsupported layer artifact format: %s", DefaultImageFormat) } + if m.layerArtifactSupportKnown && !m.layerArtifactSupport() { + return nil, fmt.Errorf("layer artifacts are unsupported on this host") + } if record, err := readLayerRecord(m.paths, layerHex); err != nil { if !errors.Is(err, errCorruptLayerRecord) { @@ -213,7 +258,10 @@ func (m *manager) materializeLayerArtifactOnce(ctx context.Context, desc layerDe } else if record != nil && record.matches(desc) { artifactInfo, statErr := os.Lstat(layerArtifactPath(m.paths, layerHex)) if statErr == nil && artifactInfo.Mode().IsRegular() && record.SizeBytes > 0 && artifactInfo.Size() == record.SizeBytes { - return record, nil + artifactDigest, digestErr := fileSHA256(layerArtifactPath(m.paths, layerHex)) + if digestErr == nil && artifactDigest == record.ArtifactDigest { + return record, nil + } } m.invalidateDiskUsageTotals() if discardErr := discardLayerCache(m.paths, layerHex); discardErr != nil { @@ -229,7 +277,9 @@ func (m *manager) materializeLayerArtifactOnce(ctx context.Context, desc layerDe if err != nil { return nil, fmt.Errorf("create unpack directory: %w", err) } + m.beginLayerBuild() defer func() { + m.endLayerBuild() if err := removePath(unpackDir); err != nil { slog.Warn("failed to remove layer unpack directory", "dir", unpackDir, "error", err) } @@ -264,7 +314,8 @@ func (m *manager) installLayerArtifact(ctx context.Context, desc layerDescriptor return err } record.SizeBytes = size - return nil + record.ArtifactDigest, err = fileSHA256(path) + return err }); err != nil { return nil, fmt.Errorf("install layer artifact %s: %w", desc.Digest, err) } @@ -286,6 +337,20 @@ type unpackStats struct { blobDigest string // sha256 of the compressed bytes as read } +func fileSHA256(path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", err + } + defer file.Close() + + hash := sha256.New() + if _, err := io.Copy(hash, file); err != nil { + return "", err + } + return fmt.Sprintf("sha256:%x", hash.Sum(nil)), nil +} + // contextReader fails reads once ctx is done so a cancelled caller stops a // long extraction instead of running it to completion. type contextReader struct { diff --git a/lib/images/layer_artifact_test.go b/lib/images/layer_artifact_test.go index 555fab776..cab726cf4 100644 --- a/lib/images/layer_artifact_test.go +++ b/lib/images/layer_artifact_test.go @@ -187,6 +187,16 @@ func TestMaterializeLayerArtifact(t *testing.T) { rebuiltInfo, err := os.Stat(artifactPath) require.NoError(t, err) require.Equal(t, rebuilt.SizeBytes, rebuiltInfo.Size()) + + // A same-size mutation must not be reused. + file, err := os.OpenFile(artifactPath, os.O_WRONLY, 0) + require.NoError(t, err) + _, err = file.WriteAt([]byte{0}, 0) + require.NoError(t, err) + require.NoError(t, file.Close()) + rebuiltAgain, err := m.materializeLayerArtifact(context.Background(), desc) + require.NoError(t, err) + require.NotEqual(t, rebuilt.CreatedAt, rebuiltAgain.CreatedAt, "checksum mismatch must rebuild") } func TestMaterializeLayerArtifactRecoversCorruptRecord(t *testing.T) { @@ -303,6 +313,49 @@ func TestUnpackLayerBlobArtifactFormatKeepsWhiteouts(t *testing.T) { requireNoWhiteoutMarkers(t, dest) } +func TestExportedLayerArtifactPreservesWhiteouts(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("exported whiteouts need root to extract device nodes") + } + if DefaultImageFormat != FormatErofs { + t.Skip("round-trip test requires erofs") + } + if _, err := exec.LookPath("mkfs.erofs"); err != nil { + t.Skip("mkfs.erofs not available") + } + if _, err := exec.LookPath("fsck.erofs"); err != nil { + t.Skip("fsck.erofs not available") + } + + root := t.TempDir() + blob := writeLayerBlob(t, root, "layer.tar.gz", + dirEntry("gone/"), + fileEntry("gone/.wh.deleted.txt", ""), + dirEntry("opq/"), + fileEntry("opq/.wh..wh..opq", ""), + ) + dest := filepath.Join(root, "dest") + _, err := unpackLayerBlob(context.Background(), blob, testTarGzMediaType, dest, layerArtifactOnDiskFormat()) + require.NoError(t, err) + + artifact := filepath.Join(root, "layer.erofs") + _, err = ExportRootfs(dest, artifact, FormatErofs) + require.NoError(t, err) + + extracted := filepath.Join(root, "extracted") + output, err := exec.Command("fsck.erofs", "--xattrs", "--extract="+extracted, artifact).CombinedOutput() + require.NoError(t, err, string(output)) + + var stat unix.Stat_t + require.NoError(t, unix.Lstat(filepath.Join(extracted, "gone", "deleted.txt"), &stat)) + require.Equal(t, uint32(unix.S_IFCHR), stat.Mode&unix.S_IFMT) + require.Equal(t, uint64(0), uint64(stat.Rdev)) + value := make([]byte, 8) + n, err := unix.Lgetxattr(filepath.Join(extracted, "opq"), "trusted.overlay.opaque", value) + require.NoError(t, err) + require.Equal(t, "y", string(value[:n])) +} + func TestUnpackLayerBlobAppliesWhiteoutsAcrossLayers(t *testing.T) { root := t.TempDir() base := writeLayerBlob(t, root, "base.tar.gz", diff --git a/lib/images/manager.go b/lib/images/manager.go index a0fa01736..8f5276496 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -77,8 +77,12 @@ type manager struct { queue *queue.Queue createMu sync.Mutex layerFlights singleflight.Group + layerBuilds chan struct{} + layerArtifactSupportKnown bool + layerArtifactsSupported bool diskUsageMu sync.RWMutex diskUsageGeneration uint64 + activeLayerBuilds int tagGenerations map[string]uint64 requestedTags map[string]string // newest pull's digest per requested tag diskUsageLoaded bool @@ -101,16 +105,22 @@ func NewManager(p *paths.Paths, maxConcurrentBuilds int, meter metric.Meter) (Ma return nil, fmt.Errorf("create oci client: %w", err) } + if maxConcurrentBuilds < 1 { + maxConcurrentBuilds = 1 + } m := &manager{ paths: p, ociClient: ociClient, queue: queue.New(maxConcurrentBuilds), + layerBuilds: make(chan struct{}, maxConcurrentBuilds), inflightPulls: make(map[string]*inflightImagePull), borrowedCredentialsTimeout: DefaultBorrowedCredentialsTimeout, readySubscribers: make(map[string][]chan StatusEvent), tagGenerations: make(map[string]uint64), requestedTags: make(map[string]string), } + m.layerArtifactSupportKnown = true + m.layerArtifactsSupported = probeLayerArtifactSupport(p.ImageLayersDir()) // Initialize metrics if meter is provided if meter != nil { From 6247ad305a7ce5a0227c6d01565ce856126e1098 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 8 Sep 2026 20:03:15 +0000 Subject: [PATCH 28/41] Use capability probes for whiteout tests --- lib/images/layer_artifact_smoke_test.go | 4 ++-- lib/images/layer_artifact_test.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/images/layer_artifact_smoke_test.go b/lib/images/layer_artifact_smoke_test.go index f4d6987d3..790041ef8 100644 --- a/lib/images/layer_artifact_smoke_test.go +++ b/lib/images/layer_artifact_smoke_test.go @@ -175,8 +175,8 @@ func TestSmokeConcurrentMaterializeSharesFlight(t *testing.T) { } func TestSmokeDeviceAndFifoEntries(t *testing.T) { - if os.Geteuid() != 0 { - t.Skip("device and fifo tar entries need root to mknod") + if !probeLayerArtifactSupport(t.TempDir()) { + t.Skip("device and fifo tar entries need mknod") } root := t.TempDir() blob := writeRawTarLayer(t, root, "devices.tar", diff --git a/lib/images/layer_artifact_test.go b/lib/images/layer_artifact_test.go index cab726cf4..cd2d9d60d 100644 --- a/lib/images/layer_artifact_test.go +++ b/lib/images/layer_artifact_test.go @@ -285,7 +285,7 @@ func TestMaterializeLayerArtifactMissingBlob(t *testing.T) { // extraction format records deletions in overlayfs form: a 0:0 character // device for a whiteout and the opaque xattr for an opaque directory. func TestUnpackLayerBlobArtifactFormatKeepsWhiteouts(t *testing.T) { - if os.Geteuid() != 0 { + if !probeLayerArtifactSupport(t.TempDir()) { t.Skip("overlayfs whiteouts need mknod and trusted xattrs") } root := t.TempDir() @@ -314,8 +314,8 @@ func TestUnpackLayerBlobArtifactFormatKeepsWhiteouts(t *testing.T) { } func TestExportedLayerArtifactPreservesWhiteouts(t *testing.T) { - if os.Geteuid() != 0 { - t.Skip("exported whiteouts need root to extract device nodes") + if !probeLayerArtifactSupport(t.TempDir()) { + t.Skip("exported whiteouts need mknod and trusted xattrs") } if DefaultImageFormat != FormatErofs { t.Skip("round-trip test requires erofs") From f9fdb37d01741c0d0bb37f211921f014a9ceaeaa Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 8 Sep 2026 20:18:33 +0000 Subject: [PATCH 29/41] Tighten layer build cleanup and verification --- lib/images/disk_usage.go | 5 ++- lib/images/layer_artifact.go | 38 +++++++++++------ lib/images/layer_artifact_smoke_test.go | 41 ++---------------- lib/images/layer_artifact_test.go | 56 +++++++++++++++++++++++-- lib/images/manager.go | 2 - 5 files changed, 85 insertions(+), 57 deletions(-) diff --git a/lib/images/disk_usage.go b/lib/images/disk_usage.go index 67ef07a5e..3e191d0d0 100644 --- a/lib/images/disk_usage.go +++ b/lib/images/disk_usage.go @@ -140,7 +140,7 @@ func totalOCICacheBlobBytesFromFilesystem(blobDir string) (int64, error) { } func (m *manager) getDiskUsageTotals() (int64, int64, error) { - for { + for attempt := 0; ; attempt++ { m.diskUsageMu.RLock() generation := m.diskUsageGeneration if m.diskUsageLoaded { @@ -159,6 +159,9 @@ func (m *manager) getDiskUsageTotals() (int64, int64, error) { m.diskUsageMu.Lock() if m.diskUsageGeneration != generation { m.diskUsageMu.Unlock() + if attempt >= 2 { + return readyImageBytes, ociCacheBytes, nil + } continue } if m.activeLayerBuilds == 0 { diff --git a/lib/images/layer_artifact.go b/lib/images/layer_artifact.go index e47e23685..033ab3357 100644 --- a/lib/images/layer_artifact.go +++ b/lib/images/layer_artifact.go @@ -145,10 +145,7 @@ func probeLayerArtifactSupport(layersDir string) bool { } func (m *manager) layerArtifactSupport() bool { - if m.layerArtifactSupportKnown { - return m.layerArtifactsSupported - } - return supportsLayerArtifacts() + return m.layerArtifactsSupported } // layerArtifactOnDiskFormat is the extraction format for per-layer artifacts. @@ -164,7 +161,18 @@ func layerArtifactOnDiskFormat() layer.OnDiskFormat { // A missing record returns (nil, nil): the layer simply was never // materialized. func readLayerRecord(p *paths.Paths, layerHex string) (*layerArtifact, error) { - data, err := os.ReadFile(layerArtifactRecordPath(p, layerHex)) + recordPath := layerArtifactRecordPath(p, layerHex) + info, err := os.Lstat(recordPath) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("stat layer record: %w", err) + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("%w: layer record is not a regular file", errCorruptLayerRecord) + } + data, err := os.ReadFile(recordPath) if err != nil { if os.IsNotExist(err) { return nil, nil @@ -243,7 +251,7 @@ func (m *manager) materializeLayerArtifactOnce(ctx context.Context, desc layerDe if layerArtifactFormat() == "" { return nil, fmt.Errorf("unsupported layer artifact format: %s", DefaultImageFormat) } - if m.layerArtifactSupportKnown && !m.layerArtifactSupport() { + if !m.layerArtifactSupport() { return nil, fmt.Errorf("layer artifacts are unsupported on this host") } @@ -258,7 +266,7 @@ func (m *manager) materializeLayerArtifactOnce(ctx context.Context, desc layerDe } else if record != nil && record.matches(desc) { artifactInfo, statErr := os.Lstat(layerArtifactPath(m.paths, layerHex)) if statErr == nil && artifactInfo.Mode().IsRegular() && record.SizeBytes > 0 && artifactInfo.Size() == record.SizeBytes { - artifactDigest, digestErr := fileSHA256(layerArtifactPath(m.paths, layerHex)) + artifactDigest, digestErr := fileSHA256(ctx, layerArtifactPath(m.paths, layerHex)) if digestErr == nil && artifactDigest == record.ArtifactDigest { return record, nil } @@ -279,10 +287,10 @@ func (m *manager) materializeLayerArtifactOnce(ctx context.Context, desc layerDe } m.beginLayerBuild() defer func() { - m.endLayerBuild() if err := removePath(unpackDir); err != nil { slog.Warn("failed to remove layer unpack directory", "dir", unpackDir, "error", err) } + m.endLayerBuild() }() stats, err := unpackCachedLayer(ctx, m.paths, desc, unpackDir, layerArtifactOnDiskFormat()) @@ -314,7 +322,7 @@ func (m *manager) installLayerArtifact(ctx context.Context, desc layerDescriptor return err } record.SizeBytes = size - record.ArtifactDigest, err = fileSHA256(path) + record.ArtifactDigest, err = fileSHA256(ctx, path) return err }); err != nil { return nil, fmt.Errorf("install layer artifact %s: %w", desc.Digest, err) @@ -337,7 +345,7 @@ type unpackStats struct { blobDigest string // sha256 of the compressed bytes as read } -func fileSHA256(path string) (string, error) { +func fileSHA256(ctx context.Context, path string) (string, error) { file, err := os.Open(path) if err != nil { return "", err @@ -345,7 +353,7 @@ func fileSHA256(path string) (string, error) { defer file.Close() hash := sha256.New() - if _, err := io.Copy(hash, file); err != nil { + if _, err := io.Copy(hash, contextReader{ctx: ctx, reader: file}); err != nil { return "", err } return fmt.Sprintf("sha256:%x", hash.Sum(nil)), nil @@ -404,7 +412,13 @@ func unpackLayerBlob(ctx context.Context, blobPath, mediaType, dest string, onDi defer blob.Close() blobHash := sha256.New() - reader, closer, err := decompressLayer(io.TeeReader(blob, blobHash), mediaType) + if _, err := io.Copy(blobHash, contextReader{ctx: ctx, reader: blob}); err != nil { + return nil, fmt.Errorf("hash blob: %w", err) + } + if _, err := blob.Seek(0, io.SeekStart); err != nil { + return nil, fmt.Errorf("rewind blob: %w", err) + } + reader, closer, err := decompressLayer(blob, mediaType) if err != nil { return nil, err } diff --git a/lib/images/layer_artifact_smoke_test.go b/lib/images/layer_artifact_smoke_test.go index 790041ef8..79e2c6c98 100644 --- a/lib/images/layer_artifact_smoke_test.go +++ b/lib/images/layer_artifact_smoke_test.go @@ -2,7 +2,6 @@ package images import ( "archive/tar" - "bytes" "context" "os" "os/exec" @@ -14,7 +13,6 @@ import ( "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/kernel/hypeman/lib/paths" - "github.com/klauspost/compress/zstd" "github.com/stretchr/testify/require" "golang.org/x/sys/unix" ) @@ -23,39 +21,6 @@ import ( // Media types, path validation, cache-miss rebuilds, singleflight sharing, // device/fifo entries (root only), and disk-usage accounting. -func writeRawTarLayer(t *testing.T, dir, name string, entries ...func(*testing.T, *tar.Writer)) string { - t.Helper() - var buf bytes.Buffer - tw := tar.NewWriter(&buf) - for _, entry := range entries { - entry(t, tw) - } - require.NoError(t, tw.Close()) - path := filepath.Join(dir, name) - require.NoError(t, os.WriteFile(path, buf.Bytes(), 0644)) - return path -} - -func writeZstdTarLayer(t *testing.T, dir, name string, entries ...func(*testing.T, *tar.Writer)) string { - t.Helper() - var tarBuf bytes.Buffer - tw := tar.NewWriter(&tarBuf) - for _, entry := range entries { - entry(t, tw) - } - require.NoError(t, tw.Close()) - - var zstBuf bytes.Buffer - zw, err := zstd.NewWriter(&zstBuf) - require.NoError(t, err) - _, err = zw.Write(tarBuf.Bytes()) - require.NoError(t, err) - require.NoError(t, zw.Close()) - path := filepath.Join(dir, name) - require.NoError(t, os.WriteFile(path, zstBuf.Bytes(), 0644)) - return path -} - const ( ociZstdMediaType = "application/vnd.oci.image.layer.v1.tar+zstd" dockerZstdMediaType = "application/vnd.docker.image.rootfs.diff.tar.zstd" @@ -87,7 +52,7 @@ func TestSmokeUnpackZstdMediaTypes(t *testing.T) { func TestSmokeRejectsBadLayerHex(t *testing.T) { p := paths.New(t.TempDir()) - m := &manager{paths: p} + m := testLayerArtifactManager(p) for _, digest := range []string{ "sha256:..", "sha256:../escape", @@ -122,7 +87,7 @@ func TestSmokeRebuildWhenArtifactFileMissing(t *testing.T) { require.NoError(t, err) writeLayerTestLayout(t, p, img) desc := layerDescFromImage(t, img, 0) - m := &manager{paths: p} + m := testLayerArtifactManager(p) first, err := m.materializeLayerArtifact(context.Background(), desc) require.NoError(t, err) @@ -149,7 +114,7 @@ func TestSmokeConcurrentMaterializeSharesFlight(t *testing.T) { require.NoError(t, err) writeLayerTestLayout(t, p, img) desc := layerDescFromImage(t, img, 0) - m := &manager{paths: p} + m := testLayerArtifactManager(p) const callers = 8 var wg sync.WaitGroup diff --git a/lib/images/layer_artifact_test.go b/lib/images/layer_artifact_test.go index cd2d9d60d..9711a682a 100644 --- a/lib/images/layer_artifact_test.go +++ b/lib/images/layer_artifact_test.go @@ -11,6 +11,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "testing" "time" @@ -19,6 +20,7 @@ import ( "github.com/google/go-containerregistry/pkg/v1/layout" "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/kernel/hypeman/lib/paths" + "github.com/klauspost/compress/zstd" "github.com/opencontainers/umoci/oci/layer" "github.com/stretchr/testify/require" "golang.org/x/sys/unix" @@ -41,6 +43,10 @@ const testTarGzMediaType = "application/vnd.oci.image.layer.v1.tar+gzip" // writeLayerTestLayout writes img into the shared OCI cache of p tagged with // the image's digest, mirroring pullToOCILayout. +func testLayerArtifactManager(p *paths.Paths) *manager { + return &manager{paths: p, layerArtifactsSupported: true, layerBuilds: make(chan struct{}, 1)} +} + func writeLayerTestLayout(t *testing.T, p *paths.Paths, img gcr.Image) { t.Helper() digest, err := img.Digest() @@ -128,6 +134,38 @@ func writeLayerBlob(t *testing.T, dir, name string, entries ...func(*testing.T, return path } +func writeRawTarLayer(t *testing.T, dir, name string, entries ...func(*testing.T, *tar.Writer)) string { + t.Helper() + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + for _, entry := range entries { + entry(t, tw) + } + require.NoError(t, tw.Close()) + path := filepath.Join(dir, name) + require.NoError(t, os.WriteFile(path, buf.Bytes(), 0644)) + return path +} + +func writeZstdTarLayer(t *testing.T, dir, name string, entries ...func(*testing.T, *tar.Writer)) string { + t.Helper() + var tarBuf bytes.Buffer + tw := tar.NewWriter(&tarBuf) + for _, entry := range entries { + entry(t, tw) + } + require.NoError(t, tw.Close()) + var zstBuf bytes.Buffer + zw, err := zstd.NewWriter(&zstBuf) + require.NoError(t, err) + _, err = zw.Write(tarBuf.Bytes()) + require.NoError(t, err) + require.NoError(t, zw.Close()) + path := filepath.Join(dir, name) + require.NoError(t, os.WriteFile(path, zstBuf.Bytes(), 0644)) + return path +} + // unpackInto applies a layer blob onto dest with compose semantics. func unpackInto(t *testing.T, blobPath, dest string) *unpackStats { t.Helper() @@ -156,7 +194,7 @@ func TestMaterializeLayerArtifact(t *testing.T) { writeLayerTestLayout(t, p, img) desc := layerDescFromImage(t, img, 0) - m := &manager{paths: p} + m := testLayerArtifactManager(p) record, err := m.materializeLayerArtifact(context.Background(), desc) require.NoError(t, err) @@ -213,7 +251,7 @@ func TestMaterializeLayerArtifactRecoversCorruptRecord(t *testing.T) { writeLayerTestLayout(t, p, img) desc := layerDescFromImage(t, img, 0) - m := &manager{paths: p} + m := testLayerArtifactManager(p) first, err := m.materializeLayerArtifact(context.Background(), desc) require.NoError(t, err) @@ -252,7 +290,7 @@ func TestMaterializeLayerArtifactOutlivesCancelledCaller(t *testing.T) { require.NoError(t, err) writeLayerTestLayout(t, p, img) desc := layerDescFromImage(t, img, 0) - m := &manager{paths: p} + m := testLayerArtifactManager(p) ctx, cancel := context.WithCancel(context.Background()) cancel() @@ -270,9 +308,19 @@ func TestMaterializeLayerArtifactOutlivesCancelledCaller(t *testing.T) { require.FileExists(t, p.ImageLayerArtifactForFormat(layerHex, layerArtifactFormat())) } -func TestMaterializeLayerArtifactMissingBlob(t *testing.T) { +func TestMaterializeLayerArtifactRejectsUnsupportedHost(t *testing.T) { p := paths.New(t.TempDir()) m := &manager{paths: p} + _, err := m.materializeLayerArtifact(context.Background(), layerDescriptor{ + Digest: "sha256:" + strings.Repeat("ab", 32), + MediaType: testTarGzMediaType, + }) + require.ErrorContains(t, err, "unsupported on this host") +} + +func TestMaterializeLayerArtifactMissingBlob(t *testing.T) { + p := paths.New(t.TempDir()) + m := testLayerArtifactManager(p) _, err := m.materializeLayerArtifact(context.Background(), layerDescriptor{ Digest: "sha256:abababababababababababababababababababababababababababababababab", diff --git a/lib/images/manager.go b/lib/images/manager.go index 8f5276496..a53dc2dc2 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -78,7 +78,6 @@ type manager struct { createMu sync.Mutex layerFlights singleflight.Group layerBuilds chan struct{} - layerArtifactSupportKnown bool layerArtifactsSupported bool diskUsageMu sync.RWMutex diskUsageGeneration uint64 @@ -119,7 +118,6 @@ func NewManager(p *paths.Paths, maxConcurrentBuilds int, meter metric.Meter) (Ma tagGenerations: make(map[string]uint64), requestedTags: make(map[string]string), } - m.layerArtifactSupportKnown = true m.layerArtifactsSupported = probeLayerArtifactSupport(p.ImageLayersDir()) // Initialize metrics if meter is provided From 82ab6515ec0029ce6e7d2e00546c682d95913b39 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 8 Sep 2026 20:30:05 +0000 Subject: [PATCH 30/41] Serialize image store ownership --- lib/images/README.md | 2 ++ lib/images/disk_usage.go | 53 +++++++--------------------- lib/images/image_store_lock.go | 14 ++++++++ lib/images/image_store_lock_other.go | 7 ++++ lib/images/image_store_lock_test.go | 25 +++++++++++++ lib/images/image_store_lock_unix.go | 26 ++++++++++++++ lib/images/manager.go | 9 ++++- 7 files changed, 95 insertions(+), 41 deletions(-) create mode 100644 lib/images/image_store_lock.go create mode 100644 lib/images/image_store_lock_other.go create mode 100644 lib/images/image_store_lock_test.go create mode 100644 lib/images/image_store_lock_unix.go diff --git a/lib/images/README.md b/lib/images/README.md index eb2d422be..2d88678c4 100644 --- a/lib/images/README.md +++ b/lib/images/README.md @@ -92,6 +92,8 @@ Content-addressable storage with tag symlinks (similar to Docker/Unikraft): - Shared blob storage enables automatic layer deduplication across all images - Orphaned digests are automatically deleted when the last tag referencing them is removed - Symlinks only created after successful build (status: ready) +- One Hypeman process owns a data directory at a time; the image manager holds `images/.lock` for its lifetime and rejects a second owner. +- Disk accounting uses logical file sizes, matching image metadata and storage admission rather than filesystem block allocation. ## Reference Handling (reference.go) diff --git a/lib/images/disk_usage.go b/lib/images/disk_usage.go index 3e191d0d0..923195fda 100644 --- a/lib/images/disk_usage.go +++ b/lib/images/disk_usage.go @@ -3,7 +3,6 @@ package images import ( "encoding/json" "fmt" - "io/fs" "os" "path/filepath" "syscall" @@ -83,60 +82,34 @@ func totalReadyImageBytesFromMetadata(imagesDir string) (int64, error) { return total, nil } -// totalLayerArtifactBytesFromFilesystem sums the layer store, including -// records and in-progress extraction/install trees. -func totalLayerArtifactBytesFromFilesystem(layersDir string) (int64, error) { +func totalFileBytes(dir, label string) (int64, error) { var total int64 - err := filepath.WalkDir(layersDir, func(path string, d fs.DirEntry, err error) error { + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { if err != nil { if os.IsNotExist(err) { return nil } return err } - if d.IsDir() { - return nil - } - info, err := d.Info() - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err + if !info.IsDir() { + total += info.Size() } - total += info.Size() return nil }) - if err != nil { - return 0, fmt.Errorf("walk layer artifacts: %w", err) + if err != nil && !os.IsNotExist(err) { + return 0, fmt.Errorf("walk %s: %w", label, err) } return total, nil } -// totalOCICacheBlobBytesFromFilesystem sums blob sizes directly from the OCI cache blob store. -// This counts the actual bytes on disk, including any blob files that are currently -// present but no longer referenced by the OCI layout index. -func totalOCICacheBlobBytesFromFilesystem(blobDir string) (int64, error) { - var total int64 - - err := filepath.Walk(blobDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - if info.IsDir() { - return nil - } - total += info.Size() - return nil - }) - if err != nil && !os.IsNotExist(err) { - return 0, fmt.Errorf("walk OCI cache blobs: %w", err) - } +// totalLayerArtifactBytesFromFilesystem includes records and in-progress trees. +func totalLayerArtifactBytesFromFilesystem(layersDir string) (int64, error) { + return totalFileBytes(layersDir, "layer artifacts") +} - return total, nil +// totalOCICacheBlobBytesFromFilesystem includes unreferenced blobs still on disk. +func totalOCICacheBlobBytesFromFilesystem(blobDir string) (int64, error) { + return totalFileBytes(blobDir, "OCI cache blobs") } func (m *manager) getDiskUsageTotals() (int64, int64, error) { diff --git a/lib/images/image_store_lock.go b/lib/images/image_store_lock.go new file mode 100644 index 000000000..35d81eb56 --- /dev/null +++ b/lib/images/image_store_lock.go @@ -0,0 +1,14 @@ +package images + +import "os" + +type imageStoreLock struct { + file *os.File +} + +func (l *imageStoreLock) Close() error { + if l == nil || l.file == nil { + return nil + } + return l.file.Close() +} diff --git a/lib/images/image_store_lock_other.go b/lib/images/image_store_lock_other.go new file mode 100644 index 000000000..84a72a7ed --- /dev/null +++ b/lib/images/image_store_lock_other.go @@ -0,0 +1,7 @@ +//go:build !linux && !darwin + +package images + +func acquireImageStoreLock(imagesDir string) (*imageStoreLock, error) { + return &imageStoreLock{}, nil +} diff --git a/lib/images/image_store_lock_test.go b/lib/images/image_store_lock_test.go new file mode 100644 index 000000000..f1f27ec8e --- /dev/null +++ b/lib/images/image_store_lock_test.go @@ -0,0 +1,25 @@ +//go:build linux || darwin + +package images + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestImageStoreLockIsExclusive(t *testing.T) { + imagesDir := t.TempDir() + first, err := acquireImageStoreLock(imagesDir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, first.Close()) }) + + _, err = acquireImageStoreLock(imagesDir) + require.Error(t, err) + + require.NoError(t, first.Close()) + first = nil + second, err := acquireImageStoreLock(imagesDir) + require.NoError(t, err) + require.NoError(t, second.Close()) +} diff --git a/lib/images/image_store_lock_unix.go b/lib/images/image_store_lock_unix.go new file mode 100644 index 000000000..0c7056697 --- /dev/null +++ b/lib/images/image_store_lock_unix.go @@ -0,0 +1,26 @@ +//go:build linux || darwin + +package images + +import ( + "fmt" + "os" + "path/filepath" + + "golang.org/x/sys/unix" +) + +func acquireImageStoreLock(imagesDir string) (*imageStoreLock, error) { + if err := os.MkdirAll(imagesDir, 0755); err != nil { + return nil, fmt.Errorf("create image store: %w", err) + } + file, err := os.OpenFile(filepath.Join(imagesDir, ".lock"), os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + return nil, fmt.Errorf("open image store lock: %w", err) + } + if err := unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB); err != nil { + _ = file.Close() + return nil, fmt.Errorf("image store is already in use: %w", err) + } + return &imageStoreLock{file: file}, nil +} diff --git a/lib/images/manager.go b/lib/images/manager.go index a53dc2dc2..5b7894460 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -76,6 +76,7 @@ type manager struct { ociClient *ociClient queue *queue.Queue createMu sync.Mutex + storeLock *imageStoreLock layerFlights singleflight.Group layerBuilds chan struct{} layerArtifactsSupported bool @@ -97,10 +98,14 @@ type manager struct { // NewManager creates a new image manager. // If meter is nil, metrics are disabled. func NewManager(p *paths.Paths, maxConcurrentBuilds int, meter metric.Meter) (Manager, error) { - // Create cache directory under dataDir for OCI layouts + storeLock, err := acquireImageStoreLock(p.ImagesDir()) + if err != nil { + return nil, err + } cacheDir := p.SystemOCICache() ociClient, err := newOCIClient(cacheDir) if err != nil { + _ = storeLock.Close() return nil, fmt.Errorf("create oci client: %w", err) } @@ -109,6 +114,7 @@ func NewManager(p *paths.Paths, maxConcurrentBuilds int, meter metric.Meter) (Ma } m := &manager{ paths: p, + storeLock: storeLock, ociClient: ociClient, queue: queue.New(maxConcurrentBuilds), layerBuilds: make(chan struct{}, maxConcurrentBuilds), @@ -124,6 +130,7 @@ func NewManager(p *paths.Paths, maxConcurrentBuilds int, meter metric.Meter) (Ma if meter != nil { metrics, err := newMetrics(meter, m) if err != nil { + _ = storeLock.Close() return nil, fmt.Errorf("create metrics: %w", err) } m.metrics = metrics From 2e8d6c8e0d22c101fd58f809ad982d8462abcbf8 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 8 Sep 2026 20:34:31 +0000 Subject: [PATCH 31/41] Record layer blob encoding --- lib/images/layer_artifact.go | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/lib/images/layer_artifact.go b/lib/images/layer_artifact.go index 033ab3357..22e4f4d9f 100644 --- a/lib/images/layer_artifact.go +++ b/lib/images/layer_artifact.go @@ -23,7 +23,7 @@ import ( ) const ( - layerRecordSchemaVersion = 2 + layerRecordSchemaVersion = 3 maxLayerUnpackedBytes = 100 << 30 // layerBuildTimeout bounds a shared layer build end to end. Builds are @@ -35,11 +35,13 @@ const ( var errCorruptLayerRecord = errors.New("corrupt layer record") // layerArtifact is the persisted record for one materialized layer artifact. -// The key is the compressed layer blob digest plus the artifact format, so the -// same layer can coexist in several materializations. +// The key is the compressed layer blob digest plus the artifact format. The +// encoding records the descriptor's decompression class so conflicting +// descriptors cannot overwrite one artifact path. type layerArtifact struct { SchemaVersion int `json:"schema_version"` Digest string `json:"digest"` // compressed layer blob digest, sha256:... + Encoding string `json:"encoding"` DiffID string `json:"diff_id,omitempty"` Format string `json:"format"` SizeBytes int64 `json:"size_bytes"` // artifact bytes on disk @@ -64,6 +66,9 @@ func (a *layerArtifact) validate() error { if a.Format != string(FormatErofs) && a.Format != string(FormatExt4) { return fmt.Errorf("invalid format: %s", a.Format) } + if a.Encoding != "raw" && a.Encoding != "gzip" && a.Encoding != "zstd" { + return fmt.Errorf("invalid encoding: %s", a.Encoding) + } if a.SizeBytes < 0 || a.UnpackedBytes < 0 { return fmt.Errorf("invalid size") } @@ -75,12 +80,23 @@ func (a *layerArtifact) validate() error { // matches any record: the compressed digest content-addresses the blob, and // unpackCachedLayer re-verifies the diff ID whenever one is supplied. func (a *layerArtifact) matches(desc layerDescriptor) bool { - if a.Digest != desc.Digest || a.Format != layerArtifactFormat() { + if a.Digest != desc.Digest || a.Format != layerArtifactFormat() || a.Encoding != layerEncoding(desc.MediaType) { return false } return desc.DiffID == "" || a.DiffID == desc.DiffID } +func layerEncoding(mediaType string) string { + switch { + case strings.HasSuffix(mediaType, "+zstd"), strings.Contains(mediaType, "tar.zstd"): + return "zstd" + case strings.HasSuffix(mediaType, "+gzip"), strings.Contains(mediaType, "tar.gzip"): + return "gzip" + default: + return "raw" + } +} + func layerArtifactFormat() string { switch DefaultImageFormat { case FormatErofs, FormatExt4: @@ -243,7 +259,11 @@ func (m *manager) materializeLayerArtifact(ctx context.Context, desc layerDescri if shared.Err != nil { return nil, shared.Err } - return shared.Val.(*layerArtifact), nil + artifact := shared.Val.(*layerArtifact) + if !artifact.matches(desc) { + return nil, fmt.Errorf("layer artifact metadata does not match descriptor %s", desc.Digest) + } + return artifact, nil } } @@ -263,6 +283,8 @@ func (m *manager) materializeLayerArtifactOnce(ctx context.Context, desc layerDe if discardErr := discardLayerCache(m.paths, layerHex); discardErr != nil { return nil, fmt.Errorf("discard corrupt layer cache: %w", discardErr) } + } else if record != nil && record.Digest == desc.Digest && record.Format == layerArtifactFormat() && record.Encoding != layerEncoding(desc.MediaType) { + return nil, fmt.Errorf("layer artifact encoding does not match descriptor %s", desc.Digest) } else if record != nil && record.matches(desc) { artifactInfo, statErr := os.Lstat(layerArtifactPath(m.paths, layerHex)) if statErr == nil && artifactInfo.Mode().IsRegular() && record.SizeBytes > 0 && artifactInfo.Size() == record.SizeBytes { @@ -310,6 +332,7 @@ func (m *manager) installLayerArtifact(ctx context.Context, desc layerDescriptor record := &layerArtifact{ SchemaVersion: layerRecordSchemaVersion, Digest: desc.Digest, + Encoding: layerEncoding(desc.MediaType), DiffID: stats.diffID, Format: layerArtifactFormat(), UnpackedBytes: stats.unpackedBytes, From 3f5c8c1eed7f7964b480378bc2f79b07bb24045c Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 8 Sep 2026 20:37:24 +0000 Subject: [PATCH 32/41] Trim image store lock implementation --- lib/images/image_store_lock.go | 14 -------------- lib/images/image_store_lock_other.go | 7 ------- lib/images/image_store_lock_unix.go | 11 +++++++++++ 3 files changed, 11 insertions(+), 21 deletions(-) delete mode 100644 lib/images/image_store_lock.go delete mode 100644 lib/images/image_store_lock_other.go diff --git a/lib/images/image_store_lock.go b/lib/images/image_store_lock.go deleted file mode 100644 index 35d81eb56..000000000 --- a/lib/images/image_store_lock.go +++ /dev/null @@ -1,14 +0,0 @@ -package images - -import "os" - -type imageStoreLock struct { - file *os.File -} - -func (l *imageStoreLock) Close() error { - if l == nil || l.file == nil { - return nil - } - return l.file.Close() -} diff --git a/lib/images/image_store_lock_other.go b/lib/images/image_store_lock_other.go deleted file mode 100644 index 84a72a7ed..000000000 --- a/lib/images/image_store_lock_other.go +++ /dev/null @@ -1,7 +0,0 @@ -//go:build !linux && !darwin - -package images - -func acquireImageStoreLock(imagesDir string) (*imageStoreLock, error) { - return &imageStoreLock{}, nil -} diff --git a/lib/images/image_store_lock_unix.go b/lib/images/image_store_lock_unix.go index 0c7056697..f3fb86702 100644 --- a/lib/images/image_store_lock_unix.go +++ b/lib/images/image_store_lock_unix.go @@ -10,6 +10,17 @@ import ( "golang.org/x/sys/unix" ) +type imageStoreLock struct { + file *os.File +} + +func (l *imageStoreLock) Close() error { + if l == nil || l.file == nil { + return nil + } + return l.file.Close() +} + func acquireImageStoreLock(imagesDir string) (*imageStoreLock, error) { if err := os.MkdirAll(imagesDir, 0755); err != nil { return nil, fmt.Errorf("create image store: %w", err) From 29fc86ce199d9edf7fb9e6c3558ac6289919a37c Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 8 Sep 2026 20:38:57 +0000 Subject: [PATCH 33/41] Reuse normalized layer encoding --- lib/images/layer_artifact.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/images/layer_artifact.go b/lib/images/layer_artifact.go index 22e4f4d9f..df1f25f20 100644 --- a/lib/images/layer_artifact.go +++ b/lib/images/layer_artifact.go @@ -470,15 +470,15 @@ func unpackLayerBlob(ctx context.Context, blobPath, mediaType, dest string, onDi // OCI-style suffixes (+gzip, +zstd) and docker-style media types (tar.gzip, // tar.zstd) are matched so neither encoding falls through to the raw path. func decompressLayer(r io.Reader, mediaType string) (io.Reader, io.Closer, error) { - switch { - case strings.HasSuffix(mediaType, "+zstd"), strings.Contains(mediaType, "tar.zstd"): + switch layerEncoding(mediaType) { + case "zstd": decoder, err := zstd.NewReader(r) if err != nil { return nil, nil, fmt.Errorf("zstd reader: %w", err) } decodeCloser := decoder.IOReadCloser() return decodeCloser, decodeCloser, nil - case strings.HasSuffix(mediaType, "+gzip"), strings.Contains(mediaType, "tar.gzip"): + case "gzip": gz, err := gzip.NewReader(r) if err != nil { return nil, nil, fmt.Errorf("gzip reader: %w", err) From eef8d3cc3310b5e4fb666ad4e9e6388cd3706e88 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 8 Sep 2026 20:41:12 +0000 Subject: [PATCH 34/41] Simplify image store lock ownership --- lib/images/image_store_lock_test.go | 6 +++++- lib/images/image_store_lock_unix.go | 15 ++------------- lib/images/manager.go | 2 +- 3 files changed, 8 insertions(+), 15 deletions(-) diff --git a/lib/images/image_store_lock_test.go b/lib/images/image_store_lock_test.go index f1f27ec8e..76c7919a1 100644 --- a/lib/images/image_store_lock_test.go +++ b/lib/images/image_store_lock_test.go @@ -12,7 +12,11 @@ func TestImageStoreLockIsExclusive(t *testing.T) { imagesDir := t.TempDir() first, err := acquireImageStoreLock(imagesDir) require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, first.Close()) }) + t.Cleanup(func() { + if first != nil { + require.NoError(t, first.Close()) + } + }) _, err = acquireImageStoreLock(imagesDir) require.Error(t, err) diff --git a/lib/images/image_store_lock_unix.go b/lib/images/image_store_lock_unix.go index f3fb86702..389e4144a 100644 --- a/lib/images/image_store_lock_unix.go +++ b/lib/images/image_store_lock_unix.go @@ -10,18 +10,7 @@ import ( "golang.org/x/sys/unix" ) -type imageStoreLock struct { - file *os.File -} - -func (l *imageStoreLock) Close() error { - if l == nil || l.file == nil { - return nil - } - return l.file.Close() -} - -func acquireImageStoreLock(imagesDir string) (*imageStoreLock, error) { +func acquireImageStoreLock(imagesDir string) (*os.File, error) { if err := os.MkdirAll(imagesDir, 0755); err != nil { return nil, fmt.Errorf("create image store: %w", err) } @@ -33,5 +22,5 @@ func acquireImageStoreLock(imagesDir string) (*imageStoreLock, error) { _ = file.Close() return nil, fmt.Errorf("image store is already in use: %w", err) } - return &imageStoreLock{file: file}, nil + return file, nil } diff --git a/lib/images/manager.go b/lib/images/manager.go index 5b7894460..cd2750154 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -76,7 +76,7 @@ type manager struct { ociClient *ociClient queue *queue.Queue createMu sync.Mutex - storeLock *imageStoreLock + storeLock *os.File layerFlights singleflight.Group layerBuilds chan struct{} layerArtifactsSupported bool From bac608dd0dc22b4a22f72178e56db8a590714dbe Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 8 Sep 2026 21:05:26 +0000 Subject: [PATCH 35/41] Reject unstable disk snapshots and bad artifacts --- lib/images/disk.go | 5 ----- lib/images/disk_usage.go | 2 +- lib/images/layer_artifact.go | 20 +++++++++++++++----- lib/images/layer_artifact_smoke_test.go | 2 +- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/lib/images/disk.go b/lib/images/disk.go index f092f0e8d..e1359c151 100644 --- a/lib/images/disk.go +++ b/lib/images/disk.go @@ -232,11 +232,6 @@ func convertToErofs(ctx context.Context, rootfsDir, diskPath string) (int64, err return stat.Size(), nil } -// dirSize calculates the total size of a directory. -func dirSize(path string) (int64, error) { - return dirSizeWithContext(context.Background(), path) -} - func dirSizeWithContext(ctx context.Context, path string) (int64, error) { var size int64 err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error { diff --git a/lib/images/disk_usage.go b/lib/images/disk_usage.go index 923195fda..dac527896 100644 --- a/lib/images/disk_usage.go +++ b/lib/images/disk_usage.go @@ -133,7 +133,7 @@ func (m *manager) getDiskUsageTotals() (int64, int64, error) { if m.diskUsageGeneration != generation { m.diskUsageMu.Unlock() if attempt >= 2 { - return readyImageBytes, ociCacheBytes, nil + return 0, 0, fmt.Errorf("disk usage changed during scan") } continue } diff --git a/lib/images/layer_artifact.go b/lib/images/layer_artifact.go index df1f25f20..4c89da614 100644 --- a/lib/images/layer_artifact.go +++ b/lib/images/layer_artifact.go @@ -165,7 +165,7 @@ func (m *manager) layerArtifactSupport() bool { } // layerArtifactOnDiskFormat is the extraction format for per-layer artifacts. -// Whiteouts become overlayfs whiteout inodes and opaque xattrs, the form an +// Whiteouts become overlayfs whiteout inodes and opaque xattrs, which an // overlayfs mount of stacked layers understands, so the artifact retains the // layer's deletions without a private marker format. Callers must check // supportsLayerArtifacts before materializing this format. @@ -217,6 +217,11 @@ func discardLayerCache(p *paths.Paths, layerHex string) error { return nil } +func (m *manager) clearLayerCache(layerHex string) error { + m.invalidateDiskUsageTotals() + return discardLayerCache(m.paths, layerHex) +} + // materializeLayerArtifact ensures a layer has a materialized artifact keyed // by its blob digest, building it from the shared OCI cache blob when absent. // The layer is unpacked into an isolated temp directory, converted to the @@ -279,8 +284,7 @@ func (m *manager) materializeLayerArtifactOnce(ctx context.Context, desc layerDe if !errors.Is(err, errCorruptLayerRecord) { return nil, err } - m.invalidateDiskUsageTotals() - if discardErr := discardLayerCache(m.paths, layerHex); discardErr != nil { + if discardErr := m.clearLayerCache(layerHex); discardErr != nil { return nil, fmt.Errorf("discard corrupt layer cache: %w", discardErr) } } else if record != nil && record.Digest == desc.Digest && record.Format == layerArtifactFormat() && record.Encoding != layerEncoding(desc.MediaType) { @@ -293,12 +297,18 @@ func (m *manager) materializeLayerArtifactOnce(ctx context.Context, desc layerDe return record, nil } } - m.invalidateDiskUsageTotals() - if discardErr := discardLayerCache(m.paths, layerHex); discardErr != nil { + if discardErr := m.clearLayerCache(layerHex); discardErr != nil { return nil, fmt.Errorf("discard invalid layer cache: %w", discardErr) } } + artifactPath := layerArtifactPath(m.paths, layerHex) + if artifactInfo, statErr := os.Lstat(artifactPath); statErr == nil && !artifactInfo.Mode().IsRegular() { + if discardErr := m.clearLayerCache(layerHex); discardErr != nil { + return nil, fmt.Errorf("discard invalid layer artifact: %w", discardErr) + } + } + layerDir := m.paths.ImageLayerDir(layerHex) if err := os.MkdirAll(layerDir, 0755); err != nil { return nil, fmt.Errorf("create layer directory: %w", err) diff --git a/lib/images/layer_artifact_smoke_test.go b/lib/images/layer_artifact_smoke_test.go index 79e2c6c98..2ab96ddff 100644 --- a/lib/images/layer_artifact_smoke_test.go +++ b/lib/images/layer_artifact_smoke_test.go @@ -17,7 +17,7 @@ import ( "golang.org/x/sys/unix" ) -// Smoke tests for PR #456 edge cases beyond the standard suite. +// Smoke tests for layer-artifact edge cases beyond the standard suite. // Media types, path validation, cache-miss rebuilds, singleflight sharing, // device/fifo entries (root only), and disk-usage accounting. From e952cbc40fba8a806aa368265b57fd98e1e9eba5 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 8 Sep 2026 21:06:13 +0000 Subject: [PATCH 36/41] Keep layer size helper for lifecycle reconciliation --- lib/images/disk.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/images/disk.go b/lib/images/disk.go index e1359c151..7b9b85b19 100644 --- a/lib/images/disk.go +++ b/lib/images/disk.go @@ -232,6 +232,11 @@ func convertToErofs(ctx context.Context, rootfsDir, diskPath string) (int64, err return stat.Size(), nil } +// dirSize is used by layer-store reconciliation. +func dirSize(path string) (int64, error) { + return dirSizeWithContext(context.Background(), path) +} + func dirSizeWithContext(ctx context.Context, path string) (int64, error) { var size int64 err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error { From abce7f31a53cf73b8e6292343c1012cef2ec3a9a Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Wed, 9 Sep 2026 03:16:39 +0000 Subject: [PATCH 37/41] Honor cancellation during disk accounting --- lib/images/disk_usage.go | 71 ++++++++++++++++++++++++++++------- lib/images/disk_usage_test.go | 11 ++++++ lib/images/manager.go | 4 +- 3 files changed, 70 insertions(+), 16 deletions(-) diff --git a/lib/images/disk_usage.go b/lib/images/disk_usage.go index dac527896..5d7a45f48 100644 --- a/lib/images/disk_usage.go +++ b/lib/images/disk_usage.go @@ -1,6 +1,7 @@ package images import ( + "context" "encoding/json" "fmt" "os" @@ -8,16 +9,29 @@ import ( "syscall" ) +func walkWithContext(ctx context.Context, root string, walkFn filepath.WalkFunc) error { + return filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + return walkFn(path, info, err) + }) +} + // totalReadyImageBytesFromMetadata sums ready image sizes directly from metadata.json files. // This is conservative for admission control and disk accounting: if metadata says an // image is ready, we count its recorded size without re-validating the rootfs path. If // the metadata file is unreadable or malformed, we fall back to counting any rootfs disk // files found in the digest directory so we do not undercount host disk usage. func totalReadyImageBytesFromMetadata(imagesDir string) (int64, error) { + return totalReadyImageBytesFromMetadataWithContext(context.Background(), imagesDir) +} + +func totalReadyImageBytesFromMetadataWithContext(ctx context.Context, imagesDir string) (int64, error) { var total int64 seenRootfs := make(map[rootfsIdentity]struct{}) - err := filepath.Walk(imagesDir, func(path string, info os.FileInfo, err error) error { + err := walkWithContext(ctx, imagesDir, func(path string, info os.FileInfo, err error) error { if err != nil { if os.IsNotExist(err) { return nil @@ -30,7 +44,7 @@ func totalReadyImageBytesFromMetadata(imagesDir string) (int64, error) { data, err := os.ReadFile(path) if err != nil { - rootfsBytes, fallbackErr := totalUniqueRootfsBytesInDigestDir(filepath.Dir(path), seenRootfs) + rootfsBytes, fallbackErr := totalUniqueRootfsBytesInDigestDirWithContext(ctx, filepath.Dir(path), seenRootfs) if fallbackErr == nil { total += rootfsBytes return nil @@ -40,7 +54,7 @@ func totalReadyImageBytesFromMetadata(imagesDir string) (int64, error) { var meta imageMetadata if err := json.Unmarshal(data, &meta); err != nil { - rootfsBytes, fallbackErr := totalUniqueRootfsBytesInDigestDir(filepath.Dir(path), seenRootfs) + rootfsBytes, fallbackErr := totalUniqueRootfsBytesInDigestDirWithContext(ctx, filepath.Dir(path), seenRootfs) if fallbackErr == nil { total += rootfsBytes return nil @@ -67,7 +81,7 @@ func totalReadyImageBytesFromMetadata(imagesDir string) (int64, error) { total += meta.SizeBytes return nil } - rootfsBytes, err := totalRootfsBytesInDigestDir(filepath.Dir(path)) + rootfsBytes, err := totalRootfsBytesInDigestDirWithContext(ctx, filepath.Dir(path)) if err != nil { return fmt.Errorf("stat ready image rootfs for %s: %w", path, err) } @@ -83,8 +97,12 @@ func totalReadyImageBytesFromMetadata(imagesDir string) (int64, error) { } func totalFileBytes(dir, label string) (int64, error) { + return totalFileBytesWithContext(context.Background(), dir, label) +} + +func totalFileBytesWithContext(ctx context.Context, dir, label string) (int64, error) { var total int64 - err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + err := walkWithContext(ctx, dir, func(path string, info os.FileInfo, err error) error { if err != nil { if os.IsNotExist(err) { return nil @@ -104,16 +122,27 @@ func totalFileBytes(dir, label string) (int64, error) { // totalLayerArtifactBytesFromFilesystem includes records and in-progress trees. func totalLayerArtifactBytesFromFilesystem(layersDir string) (int64, error) { - return totalFileBytes(layersDir, "layer artifacts") + return totalLayerArtifactBytesFromFilesystemWithContext(context.Background(), layersDir) +} + +func totalLayerArtifactBytesFromFilesystemWithContext(ctx context.Context, layersDir string) (int64, error) { + return totalFileBytesWithContext(ctx, layersDir, "layer artifacts") } // totalOCICacheBlobBytesFromFilesystem includes unreferenced blobs still on disk. func totalOCICacheBlobBytesFromFilesystem(blobDir string) (int64, error) { - return totalFileBytes(blobDir, "OCI cache blobs") + return totalOCICacheBlobBytesFromFilesystemWithContext(context.Background(), blobDir) +} + +func totalOCICacheBlobBytesFromFilesystemWithContext(ctx context.Context, blobDir string) (int64, error) { + return totalFileBytesWithContext(ctx, blobDir, "OCI cache blobs") } -func (m *manager) getDiskUsageTotals() (int64, int64, error) { +func (m *manager) getDiskUsageTotals(ctx context.Context) (int64, int64, error) { for attempt := 0; ; attempt++ { + if err := ctx.Err(); err != nil { + return 0, 0, err + } m.diskUsageMu.RLock() generation := m.diskUsageGeneration if m.diskUsageLoaded { @@ -124,7 +153,7 @@ func (m *manager) getDiskUsageTotals() (int64, int64, error) { } m.diskUsageMu.RUnlock() - readyImageBytes, ociCacheBytes, err := m.computeDiskUsageTotals() + readyImageBytes, ociCacheBytes, err := m.computeDiskUsageTotals(ctx) if err != nil { return 0, 0, err } @@ -180,7 +209,7 @@ func (m *manager) refreshDiskUsageTotals() { generation := m.diskUsageGeneration m.diskUsageMu.RUnlock() - readyImageBytes, ociCacheBytes, err := m.computeDiskUsageTotals() + readyImageBytes, ociCacheBytes, err := m.computeDiskUsageTotals(context.Background()) if err != nil { return } @@ -194,16 +223,16 @@ func (m *manager) refreshDiskUsageTotals() { m.diskUsageMu.Unlock() } -func (m *manager) computeDiskUsageTotals() (int64, int64, error) { - readyImageBytes, err := totalReadyImageBytesFromMetadata(m.paths.ImagesDir()) +func (m *manager) computeDiskUsageTotals(ctx context.Context) (int64, int64, error) { + readyImageBytes, err := totalReadyImageBytesFromMetadataWithContext(ctx, m.paths.ImagesDir()) if err != nil { return 0, 0, err } - ociCacheBytes, err := totalOCICacheBlobBytesFromFilesystem(m.paths.OCICacheBlobDir()) + ociCacheBytes, err := totalOCICacheBlobBytesFromFilesystemWithContext(ctx, m.paths.OCICacheBlobDir()) if err != nil { return 0, 0, err } - layerArtifactBytes, err := totalLayerArtifactBytesFromFilesystem(m.paths.ImageLayersDir()) + layerArtifactBytes, err := totalLayerArtifactBytesFromFilesystemWithContext(ctx, m.paths.ImageLayersDir()) if err != nil { return 0, 0, err } @@ -211,6 +240,10 @@ func (m *manager) computeDiskUsageTotals() (int64, int64, error) { } func totalRootfsBytesInDigestDir(digestDir string) (int64, error) { + return totalRootfsBytesInDigestDirWithContext(context.Background(), digestDir) +} + +func totalRootfsBytesInDigestDirWithContext(ctx context.Context, digestDir string) (int64, error) { rootfsPaths, err := filepath.Glob(filepath.Join(digestDir, "rootfs.*")) if err != nil { return 0, err @@ -221,6 +254,9 @@ func totalRootfsBytesInDigestDir(digestDir string) (int64, error) { var total int64 for _, rootfsPath := range rootfsPaths { + if err := ctx.Err(); err != nil { + return 0, err + } info, err := os.Stat(rootfsPath) if err != nil { if os.IsNotExist(err) { @@ -258,6 +294,10 @@ func markUniqueRootfs(info os.FileInfo, seen map[rootfsIdentity]struct{}) bool { } func totalUniqueRootfsBytesInDigestDir(digestDir string, seen map[rootfsIdentity]struct{}) (int64, error) { + return totalUniqueRootfsBytesInDigestDirWithContext(context.Background(), digestDir, seen) +} + +func totalUniqueRootfsBytesInDigestDirWithContext(ctx context.Context, digestDir string, seen map[rootfsIdentity]struct{}) (int64, error) { rootfsPaths, err := filepath.Glob(filepath.Join(digestDir, "rootfs.*")) if err != nil { return 0, err @@ -269,6 +309,9 @@ func totalUniqueRootfsBytesInDigestDir(digestDir string, seen map[rootfsIdentity var total int64 found := false for _, rootfsPath := range rootfsPaths { + if err := ctx.Err(); err != nil { + return 0, err + } info, err := os.Stat(rootfsPath) if err != nil { if os.IsNotExist(err) { diff --git a/lib/images/disk_usage_test.go b/lib/images/disk_usage_test.go index c5d0871cd..7ecd4e6a6 100644 --- a/lib/images/disk_usage_test.go +++ b/lib/images/disk_usage_test.go @@ -1,6 +1,7 @@ package images import ( + "context" "os" "path/filepath" "testing" @@ -64,6 +65,16 @@ func TestTotalReadyImageBytesFromMetadata_DeduplicatesMalformedAliases(t *testin require.Equal(t, int64(len("shared-rootfs")), total) } +func TestTotalFileBytesWithContextStopsWhenCanceled(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, err := totalFileBytesWithContext(ctx, t.TempDir(), "test files") + require.ErrorIs(t, err, context.Canceled) +} + func TestTotalLayerArtifactBytesFromFilesystem(t *testing.T) { t.Parallel() diff --git a/lib/images/manager.go b/lib/images/manager.go index cd2750154..2c2b8a5ff 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -865,7 +865,7 @@ func (m *manager) deleteTaggedImage(repository, tag string) error { // TotalImageBytes returns the total size of all ready images on disk. func (m *manager) TotalImageBytes(ctx context.Context) (int64, error) { - readyImageBytes, _, err := m.getDiskUsageTotals() + readyImageBytes, _, err := m.getDiskUsageTotals(ctx) if err != nil { return 0, err } @@ -874,7 +874,7 @@ func (m *manager) TotalImageBytes(ctx context.Context) (int64, error) { // TotalOCICacheBytes returns the total size of the OCI and materialized layer caches. func (m *manager) TotalOCICacheBytes(ctx context.Context) (int64, error) { - _, ociCacheBytes, err := m.getDiskUsageTotals() + _, ociCacheBytes, err := m.getDiskUsageTotals(ctx) if err != nil { return 0, err } From 1d761f76db1d1d44a1c43d962ebf48dabcbb42d8 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Wed, 9 Sep 2026 03:33:20 +0000 Subject: [PATCH 38/41] Propagate disk accounting cancellation --- lib/images/README.md | 5 + lib/images/disk.go | 7 +- lib/images/disk_usage.go | 30 +- lib/images/disk_usage_test.go | 20 +- lib/images/layer_artifact.go | 3 +- lib/oapi/oapi.go | 528 +++++++++++++++++----------------- lib/resources/disk.go | 11 +- openapi.yaml | 2 +- 8 files changed, 305 insertions(+), 301 deletions(-) diff --git a/lib/images/README.md b/lib/images/README.md index 2d88678c4..e52868f97 100644 --- a/lib/images/README.md +++ b/lib/images/README.md @@ -65,6 +65,10 @@ Content-addressable storage with tag symlinks (similar to Docker/Unikraft): rootfs.erofs latest -> abc123def456... # Tag symlink to digest 3.18 -> def456abc123... # Another tag + layers/ # Shared materialized layer artifacts + abc123def456.../ + layer.erofs + layer.erofs.json system/ oci-cache/ # Shared OCI layout for all images index.json # Manifest index with digest-based tags @@ -82,6 +86,7 @@ Content-addressable storage with tag symlinks (similar to Docker/Unikraft): - Natural hierarchy: All versions of an image grouped under repository - Easy inspection: Clear which digest belongs to which image - Layer caching: All images share the same blob storage, layers deduplicated automatically +- Materialized layer artifacts are reference-protected and reconciled by the layer lifecycle manager; stale temporary trees are age-gated before removal. **Design:** - Images stored by manifest digest (content hash) diff --git a/lib/images/disk.go b/lib/images/disk.go index 7b9b85b19..3dc2bd107 100644 --- a/lib/images/disk.go +++ b/lib/images/disk.go @@ -43,7 +43,7 @@ func ExportRootfsWithContext(ctx context.Context, rootfsDir, outputPath string, case FormatErofs: return convertToErofs(ctx, rootfsDir, outputPath) case FormatCpio: - return convertToCpio(rootfsDir, outputPath) + return convertToCpio(ctx, rootfsDir, outputPath) default: return 0, fmt.Errorf("unsupported export format: %s", format) } @@ -51,7 +51,7 @@ func ExportRootfsWithContext(ctx context.Context, rootfsDir, outputPath string, // convertToCpio packages directory as uncompressed cpio archive (initramfs format) // Uses uncompressed format for faster boot (kernel loads directly without decompression) -func convertToCpio(rootfsDir, outputPath string) (int64, error) { +func convertToCpio(ctx context.Context, rootfsDir, outputPath string) (int64, error) { // Ensure parent directory exists if err := os.MkdirAll(filepath.Dir(outputPath), 0755); err != nil { return 0, fmt.Errorf("create output dir: %w", err) @@ -72,6 +72,9 @@ func convertToCpio(rootfsDir, outputPath string) (int64, error) { // Walk the rootfs directory and add all files err = filepath.Walk(rootfsDir, func(path string, info os.FileInfo, err error) error { + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } if err != nil { return err } diff --git a/lib/images/disk_usage.go b/lib/images/disk_usage.go index 5d7a45f48..ce9abff66 100644 --- a/lib/images/disk_usage.go +++ b/lib/images/disk_usage.go @@ -23,10 +23,6 @@ func walkWithContext(ctx context.Context, root string, walkFn filepath.WalkFunc) // image is ready, we count its recorded size without re-validating the rootfs path. If // the metadata file is unreadable or malformed, we fall back to counting any rootfs disk // files found in the digest directory so we do not undercount host disk usage. -func totalReadyImageBytesFromMetadata(imagesDir string) (int64, error) { - return totalReadyImageBytesFromMetadataWithContext(context.Background(), imagesDir) -} - func totalReadyImageBytesFromMetadataWithContext(ctx context.Context, imagesDir string) (int64, error) { var total int64 seenRootfs := make(map[rootfsIdentity]struct{}) @@ -49,6 +45,9 @@ func totalReadyImageBytesFromMetadataWithContext(ctx context.Context, imagesDir total += rootfsBytes return nil } + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } return fmt.Errorf("read image metadata %s: %w", path, err) } @@ -59,6 +58,9 @@ func totalReadyImageBytesFromMetadataWithContext(ctx context.Context, imagesDir total += rootfsBytes return nil } + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } return fmt.Errorf("unmarshal image metadata %s: %w", path, err) } if meta.Status == StatusReady { @@ -96,10 +98,6 @@ func totalReadyImageBytesFromMetadataWithContext(ctx context.Context, imagesDir return total, nil } -func totalFileBytes(dir, label string) (int64, error) { - return totalFileBytesWithContext(context.Background(), dir, label) -} - func totalFileBytesWithContext(ctx context.Context, dir, label string) (int64, error) { var total int64 err := walkWithContext(ctx, dir, func(path string, info os.FileInfo, err error) error { @@ -121,19 +119,11 @@ func totalFileBytesWithContext(ctx context.Context, dir, label string) (int64, e } // totalLayerArtifactBytesFromFilesystem includes records and in-progress trees. -func totalLayerArtifactBytesFromFilesystem(layersDir string) (int64, error) { - return totalLayerArtifactBytesFromFilesystemWithContext(context.Background(), layersDir) -} - func totalLayerArtifactBytesFromFilesystemWithContext(ctx context.Context, layersDir string) (int64, error) { return totalFileBytesWithContext(ctx, layersDir, "layer artifacts") } // totalOCICacheBlobBytesFromFilesystem includes unreferenced blobs still on disk. -func totalOCICacheBlobBytesFromFilesystem(blobDir string) (int64, error) { - return totalOCICacheBlobBytesFromFilesystemWithContext(context.Background(), blobDir) -} - func totalOCICacheBlobBytesFromFilesystemWithContext(ctx context.Context, blobDir string) (int64, error) { return totalFileBytesWithContext(ctx, blobDir, "OCI cache blobs") } @@ -239,10 +229,6 @@ func (m *manager) computeDiskUsageTotals(ctx context.Context) (int64, int64, err return readyImageBytes, ociCacheBytes + layerArtifactBytes, nil } -func totalRootfsBytesInDigestDir(digestDir string) (int64, error) { - return totalRootfsBytesInDigestDirWithContext(context.Background(), digestDir) -} - func totalRootfsBytesInDigestDirWithContext(ctx context.Context, digestDir string) (int64, error) { rootfsPaths, err := filepath.Glob(filepath.Join(digestDir, "rootfs.*")) if err != nil { @@ -293,10 +279,6 @@ func markUniqueRootfs(info os.FileInfo, seen map[rootfsIdentity]struct{}) bool { return true } -func totalUniqueRootfsBytesInDigestDir(digestDir string, seen map[rootfsIdentity]struct{}) (int64, error) { - return totalUniqueRootfsBytesInDigestDirWithContext(context.Background(), digestDir, seen) -} - func totalUniqueRootfsBytesInDigestDirWithContext(ctx context.Context, digestDir string, seen map[rootfsIdentity]struct{}) (int64, error) { rootfsPaths, err := filepath.Glob(filepath.Join(digestDir, "rootfs.*")) if err != nil { diff --git a/lib/images/disk_usage_test.go b/lib/images/disk_usage_test.go index 7ecd4e6a6..2b5404d74 100644 --- a/lib/images/disk_usage_test.go +++ b/lib/images/disk_usage_test.go @@ -18,7 +18,7 @@ func TestTotalReadyImageBytesFromMetadata_UsesRootfsFallbackForMalformedMetadata require.NoError(t, os.WriteFile(filepath.Join(digestDir, "metadata.json"), []byte("{not-json"), 0o644)) require.NoError(t, os.WriteFile(filepath.Join(digestDir, "rootfs.erofs"), []byte("rootfs-data"), 0o644)) - total, err := totalReadyImageBytesFromMetadata(imagesDir) + total, err := totalReadyImageBytesFromMetadataWithContext(context.Background(), imagesDir) require.NoError(t, err) require.Equal(t, int64(len("rootfs-data")), total) } @@ -40,7 +40,7 @@ func TestTotalReadyImageBytesFromMetadata_DeduplicatesHardLinkedAliases(t *testi require.NoError(t, os.WriteFile(filepath.Join(sourceDir, "metadata.json"), metadata, 0o644)) require.NoError(t, os.WriteFile(filepath.Join(targetDir, "metadata.json"), metadata, 0o644)) - total, err := totalReadyImageBytesFromMetadata(imagesDir) + total, err := totalReadyImageBytesFromMetadataWithContext(context.Background(), imagesDir) require.NoError(t, err) require.Equal(t, int64(len("shared-rootfs")), total) } @@ -60,7 +60,7 @@ func TestTotalReadyImageBytesFromMetadata_DeduplicatesMalformedAliases(t *testin require.NoError(t, os.WriteFile(filepath.Join(malformedDir, "metadata.json"), []byte("{not-json"), 0o644)) require.NoError(t, os.WriteFile(filepath.Join(validDir, "metadata.json"), []byte(`{"status":"ready","size_bytes":13}`), 0o644)) - total, err := totalReadyImageBytesFromMetadata(imagesDir) + total, err := totalReadyImageBytesFromMetadataWithContext(context.Background(), imagesDir) require.NoError(t, err) require.Equal(t, int64(len("shared-rootfs")), total) } @@ -75,6 +75,16 @@ func TestTotalFileBytesWithContextStopsWhenCanceled(t *testing.T) { require.ErrorIs(t, err, context.Canceled) } +func TestExportRootfsWithContextStopsCpioWhenCanceled(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, err := ExportRootfsWithContext(ctx, t.TempDir(), filepath.Join(t.TempDir(), "rootfs.cpio"), FormatCpio) + require.ErrorIs(t, err, context.Canceled) +} + func TestTotalLayerArtifactBytesFromFilesystem(t *testing.T) { t.Parallel() @@ -90,7 +100,7 @@ func TestTotalLayerArtifactBytesFromFilesystem(t *testing.T) { require.NoError(t, os.MkdirAll(unpackDir, 0o755)) require.NoError(t, os.WriteFile(filepath.Join(unpackDir, "layer.bin"), []byte("unpacked-content"), 0o644)) - total, err := totalLayerArtifactBytesFromFilesystem(layersDir) + total, err := totalLayerArtifactBytesFromFilesystemWithContext(context.Background(), layersDir) require.NoError(t, err) require.Equal(t, int64(len("erofs")+len("ext4")+len("record")+len("unpacked-content")), total) } @@ -104,7 +114,7 @@ func TestTotalReadyImageBytesFromMetadata_UsesRootfsFallbackForReadyImageWithout require.NoError(t, os.WriteFile(filepath.Join(digestDir, "metadata.json"), []byte(`{"status":"ready","size_bytes":0}`), 0o644)) require.NoError(t, os.WriteFile(filepath.Join(digestDir, "rootfs.erofs"), []byte("another-rootfs"), 0o644)) - total, err := totalReadyImageBytesFromMetadata(imagesDir) + total, err := totalReadyImageBytesFromMetadataWithContext(context.Background(), imagesDir) require.NoError(t, err) require.Equal(t, int64(len("another-rootfs")), total) } diff --git a/lib/images/layer_artifact.go b/lib/images/layer_artifact.go index 4c89da614..abe4ed58a 100644 --- a/lib/images/layer_artifact.go +++ b/lib/images/layer_artifact.go @@ -218,8 +218,9 @@ func discardLayerCache(p *paths.Paths, layerHex string) error { } func (m *manager) clearLayerCache(layerHex string) error { + err := discardLayerCache(m.paths, layerHex) m.invalidateDiskUsageTotals() - return discardLayerCache(m.paths, layerHex) + return err } // materializeLayerArtifact ensures a layer has a materialized artifact keyed diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index b1d2067b8..ad6135826 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -934,7 +934,7 @@ type DiskBreakdown struct { // ImagesBytes Disk used by exported rootfs images ImagesBytes *int64 `json:"images_bytes,omitempty"` - // OciCacheBytes Disk used by OCI layer cache (shared blobs) + // OciCacheBytes Disk used by OCI layer cache (shared blobs and materialized layers) OciCacheBytes *int64 `json:"oci_cache_bytes,omitempty"` // OverlaysBytes Disk used by instance overlays (rootfs + volume overlays) @@ -19367,269 +19367,269 @@ var swaggerSpec = []string{ "sUHvPvc5fHd8tGNNkZu3RuP64mjMYVZ2VEbro41cEi8TlYtgjL4XCt8Qg3/r0Po7ipQvcSdXvWtW4ly/", "eRcw0yEgLItUeXMg6DrDXIs26k1u2aCzyMD8UJ4ST5C3WRURDaYcH1F5+aMg+BLg85fvbQBgGJn7LBxo", "BygZ4wWgQoGRS3CuJhIV4Cml8Xt779ne892ne88HgwCQ8TLB84iOIG+m1QDeHB6jBC8KTL8N8N/HaJzw", - "cZXQn+w+ff5s8GJ7p+04jMe63ToU+rf7Cm3YFfmLK3HmnlQGtbPz7Onu7u7g6dOdvVajsm6DVoNyLoaK", - "SPJs99ne9vOdvVarENLrXjpg6boAH6pycGAKA0Hep8xIRCc0QgDIhvQHkDI/o4wUztnqmRzj2NWCCN8d", - "CtNErgxrM53ZN429Nc0TRbOEmGewIa1cEjDzI2gpWFyLsaL2xs1asnDcawOe3FyKV0Ccick4n04t1l2x", - "dCc2jb0UnihJ4n1UqGwr+RzsZjmwD010YOfQkhpea9XJos55RGCuLgPRIAgq6MRsWmVWLtaBsiwPkkTj", - "Uv6UC4M+AY0iPOa5Ml51Wyyl7MQAuTDwMucsbmfu+ImLy7WJDfomLmrCtDQYuAx3kwFfy233hb7C+21i", - "BOxzid6aL4yhsPw5y6sF8brQkzUoMpeWHxd2Y9tMW+kyLLeAzdyFy5j+St55T6F8vYmJfvqyGraYEqiF", - "pNZKLJpSTDzSGbzeOk9Kf7jWkNJi3Rm5uo9Fh0y/nibbnmQ4u5sVXxUxXNgavDx9buPs+whOFwT5uQT9", - "2kk7c1ATdsn7Q2YTjTy8AnCkAZaACROfESoQF3RKqx1X7az3GLN7E8p0xHVr6vQ/bIrBk4UbMMQDDNLy", - "xITAmgIgPtCr3ZNOt3NWFI2yjKm6NG+LwltLK1JG1i9j7py+u2nkbSb4hIaAcCASyD61ipqLSX29Nzjr", - "bf/VRM1r8gOJjTITPZTyuFbjyb7f7iJ6dfrutGlMRdUl5I9uaU4r4H2Wo10mOTN1nzybCrb1GC3imsHj", - "MZ310Y9Eqh6ZTLhQ+zaQBRxSXs1XvECCxLmmBx+ajDMk6TiBM1r0qgV+/YueEACBjPPJhIgq6sCLkCzt", - "vT1KA8a9n/RzZF4oYA9OfqzK01pub6u1n1aoAdT2CY4om2623u6AQbA2jXW4gq9O3721ZYOaEJf1Uhal", - "hQzYch/9WhTW0kstS1SlfsBSWKMnE0CtXxjJhIeiUX4ShCB4Vhb01GcPPNgGo7eIGsAxYDRx5soKog2I", - "8oqJRL/nWGCmKCMxev+T/B4NrMv8/U/IZNFYBkSlH9hY1RSfhba0MRvudLaQNMKJWRZT2oMy30oJR7q1", - "mnFafmjtuQFlI1xUzrEPtDGfZjks4Nnb3vGb91tpTObdypggGnHGE6LHvenx2LmDZChT9yqsdd5kLnJb", - "VdQ4HJV1FesHAdCgrR2zTh16bwZoTCLsAEXqmxcVZYjHEHsBVq8rQZXSkiu2RmeInikbMaB5Ek3yJOlb", - "BuiGbMzME+DDHoKpuao0+cDzDIMmVYhR1wo6JUjmUURIXAtfqY/ao7d9lLMUCznDSX1y+0WodzTDAkf6", - "Pvzu+rtOuI6QHpxsy+g96ixumtZk6d0rAXr0zl3TEX//U/V4mqveJAdbAzllVAHYGSBcbxSDNJjcfXSC", - "F9aOksGdaeml3DE8mRjIz+IuESQhWEL9b4ne/9RfG5ipuMJJ0xzO9UPLpzb0hPQp0sPsoqzCBoB3eVhb", - "frdPg1dGOZ+Amwk6rPss+uiYhdmeif4BtDwDfCUlnTKt20kbABRhVqzk0t5V786gWaiOLtipLlxlOt3A", - "BRCimNBtZbIZl9Wc5Xqk/LLKpvjl+hqYppHmfg9dwmXNr2V1SXduIS/TRSIh86nlGMY6KkmGBVYkWdRC", - "oqs1DchyUAm5JtENMj5f6tc/mTo2uSAjNRNEznhSDUXa7S7XQpYQ9j8ntvybmZPne1McpVhcwilzujTK", - "mVmBatbI7jqIp5lS2Q0m9fP5+akxsCki5jip5x3JpSCbI5LgBRoTdUUIc1PBEmE/7Lyety0bSnAJNcqI", - "oLy6hp3dQL9nJhUBTQWOCDJfuUrqBVvTR6/tUtpeAqChUUSkbNjf7VX7az+d5Em7PQ4Na3vdDqvoJht8", - "fnjqSkkVpbrdMu8sr/IpET1z5FzN7tVbuyNXFzVzXTGDBb0ks48JVDmzyV5+LrYLMYQqbvrzSv6zxxyk", - "n7Fr+4FTYJaqa875h1YKXv24h2JpUsziUMlzk+Nj0DGmAJkHiQYiB+2LxkZSMneydz/bVCUtaVFGpKxB", - "C0S5SDrdTm9iZ7W/taX5fQKooXu728+3VkdyrwzhtxGLo5iuMvG4uEYT+eYS2Q3kIky6ShJbOMtaGMHN", - "Oq65H4A9LYcMQ5lzfbd5SpbLIRkMllASrnGkXGFGsIpXoh6wf2wB27sqyOgGU1NrSWu7L/zzOQiGoWRY", - "zarkv7VE+xCaBmHFmkaMxbG2jobIPwalUy5C1fe4UDbpdkxc3HFxH7qoXodiXfGPD577s3z65Mnuk3V8", - "CJhN7ZjbcxeYqnm7CitAvOW259c2ABGSVZnDHenVBYT1uqyhKc0Rl0hq9YLyjLAbreeTvd2dm61n24kc", - "u8jMGl8KoTIdnhwZmUjr8pgyIlBKFI6xwlUmA+ZkzWWgvBMmKWTmTb5fzVoaQph8mKXb1qb7UgEwDWUq", - "3zpU9hQzOgElybzp9yxneOfJ031TTDcmk70nT/v9/k3BZ16WaDOttmLLxMl6ODR9Ofu8fbgDjJk2c/mj", - "c3pw/rNmZLkU5tLakmPK9r1/F/8sH8Af5p9jysLYNG3qL9PJUt3lakhobsG/SbyPyhL7Tu5pE6LX4A+C", - "BAEAxAoiPVYCpe8O0rGgcVqt3HGD9PwV6epaXHnDkkXj2ty6lDIrbKHKK6Hs2xValFOmH1eHuDiLM7xj", - "+zSWjqLS9HJwy61qhcuV5VSXqu1lhBUFVJPE/BVxBtjaoWqqlSvSPWtRjA+uEVt1r+jS/7Ho3fvx0B+I", - "97sr5uf9ZMuqfrhhVNpKgfRvy3Loei7kxNE1hzls/i9uhbYlrC0UZDAd44HvwttEblZ7fzP9r9//jzx9", - "9vft31+/f//f81f/dfQr/e/3yembz4IZWo0C+qBQnl8MvRPCFSsQnm1J6QSrKGCj0+pfwwrbJ8bioKIZ", - "FN1FY7I/ZD30mioiTAnHWv7xsIM2CGhK8JUWd6E+lUn93NQfn5qgAv3xH04M/lRvI7a4EMJuSAH3I/Nx", - "zFNM2eaQDZltC7mJSNAL9F8xinBm6jBShrT+u0BjAUUzrZe37LyL/sBZ9mlzyMAqS64NoH2Goe7gpPBN", - "MBejYUdlItHt66RAfjFJwUNW3NYFDKZx9ffLmhmUJPW0vYZFWa2/Wc3p+SAEGAopZXojoS4UqCAFZWsy", - "KnLd0PPB5rI+t0bHKGhoBfnZWBiTc3yQh8zFTXnKRySmEfAVl6o7s8ncRZa0oTRrxMsEv17A3rw1+aMx", - "wrmaaV4UWWyLiPNLSrqwpV3wSEPwFXxpQmpmPOuNF70ZzwqcEyxMwBk2QSlVJfv/9OxEe++JoBPbUxCu", - "QpNIQOiEI2NnZrJ+C+vC0sTOTUkdpkWfObGvm9pM0pTjMdkUKhfMVYwhUDcWAHUK6iMhmfx7FCUUrE5y", - "Bj68GWBeKt1MCLayMyiy1vA4ismk/u9qVNHOk6egwbp/7+60Tho3S7eKyvIkoNOmjvW14NiGTcIAjHgw", - "cobwNXGA+ga0oRRgp1Ac/nuGXEPliSsYifH0mTRWaQuQJNJLYN0MZhraY2BRe0bYnqY299HSKawkLrZo", - "wQQIwWdJC7yglyZh+vz1GVJEpA7CYiPSuwOnxIDD9KiUua2Fd3B48nKz3wlinVVcWrBVKxMbq4MOwJ3Y", - "gKGmOKjSRoNT0kXHR5Cwbq+VUheDDKOfuECJuRXLy2gf8HKq5h5sqmgeH1kBNFmUUUdGbBl2Nl2LWf16", - "20dvCxUQF0MpUo9L2nJNlpcJNGtjUE3601LrtUx18I9Z9c/ex5DsBCVGDS8GvNbG+6u9zdGhv+mLqmYh", - "u/GF5AeCNdq/vL3/0sjmX15G372ZjG690KNshmWIume+VxNeWtp335FdZfeiOWyv0u9I0uDZ+psrtORd", - "Q4roe67yeQjK90lve/t8e+/m5rubglJX0eg8pMoCl7o9oPRdADMHYJapGjXmdyD92GZzOLvI+xM0w5J9", - "p+BhzTqyvfusjVECem2bGeHnRPCJGVLBpRy0XRHRb0D+LmmSGAFG0inDCXqBNs6OX/1y/Pr1JuqhN29O", - "6lux6ovg/twCnxpuAVhHExwVQDeroHKgIn33/Pw1HK6EQAaUkcMvb49avda02ALF2g3u1ek7cPxjOXKx", - "083pwrhMuSfXVCq5DGzYKgXhc1CzzaelYazNJE0bNqhyLfT2zxVs6CBS5eYdYGa7/JGl5XwAOOmHzNP9", - "+qCsV4JPfy6CtLUz3BGAdOOVFgJfroGSPGm63W4PBX0nw6lgOoXYli/hOBCFW2Mvdzs0kEB+YOP40PFp", - "WWytdEa45mtzerHT3376HOoFbw/aMPYURyv6Pjk4bN/5YMfcMvt4vB/F+6Cw39ZnZQnbqCA4ucILqLdp", - "lnbYMRemp916x9Yqkq3ia5Yhrm+HaF0X4xowq0GcdYFLcpSuLPfTIkO4jluY5haVNaVJQiWJOItlVUae", - "YYlkZsCITdmbQoIfMhhgFxXVx0FKQTiKRF6aHq10beX9PLN0D6V3M860DgC1N34hC4lSCk7QonsIfZSo", - "SESLh2xDuKTFIjsRqu7G+gdIAera5JK4C1HDUN5HfzBkcpYrzcQ2++iQM5mnRFirLBpT8BhtIpkblRbG", - "C6ux0AxT0piIIdOvBeCO/yjUk/2ng8Fg0O0Umtyu/vcgRE136vzsWzhvk3YPgJvMAntDtLvIGcpZTERR", - "gp8YcqiHyN3QcfqZON7u83bilf28lKvCB3Md7Hc7PO/PBVGGoTbo5xAdegvl/MntRfRWaX9OfrUJf/ar", - "0U0iGGxKhtb4xvq2MwY5ElszpCQ216PIvXlnyt9Wp25DjxVHv+dELND7k5NK2IPN0Gg3ceASDfvAsxtt", - "w84aG8na0dzEvexBTt8HzHRdUvEkxC8OKu17GB0OgaHQimGrojivsq9prTKYA0WZ2SdNNCsmWKuAbFBb", - "ysBIv1W5kBMLsWpFaZdTYSHfCwSW0qkvF3JrnMutLKJbNntsC+BxngM8zl4QvyAm81Geh1Qj/cjhIb17", - "d3yENuAXgHc2CTKV7jF+uv188PxF7/l4+2lvLx5s9/D27tPezhM8mOxGz3a3d3ZXJBW1SGi9fY5qUGMO", - "xDEXUesjFz0fCmpuyl2oySY2HvuKsphfVa6/YICs37sNvl3X/XJofeshBFOCEiyVMV80cLITuORJpNs2", - "Aek2abqodhY2dD49H2x/rvUHBtdwR5yLnBm3qoH5KFwIqTdgf7Oq47wdy4cBucSXdavld95+0Qb7T17s", - "P/ncRXPJG+vGWCene9zcpogwB4Neyw5xScKeHckZKDtWJjJWfZtM0ul2inwX+BuEgVosdfG4VRJX04Ht", - "htnIqmulAb/guKKvQKSKgcGM97Wk4vQRKKpSoGRoEegw4XmMPFucQQUEP9yxp7voZsAtZk10JnHZJGNA", - "ZiSVNp+PMs2Iwf+oG7FgB/voFbwLj3Bq1Do7CFNCyHe94Xhh4mX0+XJdGyVr9ZDPrH4F32hlC+l/wbT1", - "MliT7eomjHS2j37l8E2h7TFet/2a10HNWn69bifesIj5DrwGOrOi5j76qRAvCwHVCqQbktg/R5ZhlZhR", - "mxXkDrvjHU0t5c55KBTdjlnRTrfjFgrQKpZxK96VVL90/nxSDAWSEZzAWS7T9HNFE4uUDzOhUtFI2uQR", - "vblNYo/NzSTxyChPTTGpJvXVKljFR06qen+CNgAM9S/IGrb1vzaL+NXKXbfzYu/F02c7L562gjwrB7he", - "ND4EZILlwa2Vk6MsH1nbSNPUD0/fGdtHZKwKRezL+xMfYSYTXLMePXPXoN/5i/4LH+kt5vk48RyLFhbS", - "AEvDhgXBDAte1BAH+TtN5nQyYb9/jC53/i5oun39VO6MtxsQrE1HYbPbsR9csGSjJuOeqVQWBuMCghKy", - "Ea/uLZEwA3RGFAL66SEcgXpT5FNbknOodnbFg4S1t7u7+/zZk51WdGVH5x2cERjhApeyHYF3xOBNtPH2", - "7AxteQRn2nSwLlkA5qB2zpAtMz6oCqT97cFuiEoaLu6Samzb87Rxyd9b9dFOyi46JG8XquXSKQ+u9u7u", - "4Nnek+dP2h1jax4eievVHMalLJnlsbUw/J3fAGny/OAUQULwBEdV246LELvRqNSNRgV1XEz9hRsM7Pmz", - "p0/2dne22wEvhoJOLKRo5cBWeVfg0AWIIrAbgaVYZr3dptsiJE4ZAntLogTT9CByKRa128fUWRgJ81q5", - "CW0uBquBL11cLb5tZdwqTFYmQceIBlygnBXVffrrXbJfxLPazLXN9bCeq4fScphePYsQZqoY3mIpM0Hm", - "lOfyCzTElcmZnSScixt926SwvCUyT5Sx2VCJ3p98BzxF0xqSimRVHcpS4woctVtO7kbnuUIiYSJvWqxW", - "u9Fm61dNuNtwarurADUq3KARvTDWnCtn64M/D3ES5VDPChf7qWcFMHwACZBlycLE9icJ5wxFM8zASSI8", - "0DE040ncD0bC6iejSTCqgl+hhBvc9UtCMlvqyQxCf6ZFGDonaMMvcmhIqVZ6+ElqmIwt5lOlxidpuIZq", - "GFOpSIXX64kV9yDBzScVS2jCpxKUQgVZC/16JYoMC5OMgJkpXTZPjS4ZCLgODLHGzEM3qrlJ+cQquFbk", - "gERzs5I4Elx6cGDvT2r5yyty3oos5vUBndXBtiBd49AMXGUGkK51hcPQ/RjI5/mcGxJoGHIGV4RKOuNk", - "ilkOxZ88QraG+H7rcMgZl2pUwFzdcLBSjaCiSy5ICVhZZN0X9iD3TvBedKztNstl445v9fUSVYWbahpg", - "M08Nrmh4tboFDYbIeBlXbiWUXQnwVwezugleZFkChEpolXrIgWgDcl48tuShQG62CZIJq6y6nyVt1Rbo", - "fb03OGuLrLgaSPEUq9kxm/AA9scNPKfOEm2jVTMiHLhhTBglsdMlCxeqNXVBwngiCYpzYlfOyKc+/B2E", - "OGA1A3skfEjZtMbr6x22MQ+bMawu+AL92hfbhDvJcELtuchhrUy8okS4TK1tFQRK5SjszlpuWJBpnmCB", - "LCZqmyHLRZpQdtmmdblIxzyhEdIf1P3iE54k/GqkH8kfYC6brWanPxg1VQc7M4OzeYFmQ2r9llP4Qc9y", - "s5aVDJaYLfP9FjhG20SPBSPFf6IJsdiU7xi99gi9WhFhb2fQlC3f0GglT34ZnPWmnNuSbPDE5zKQW7hS", - "ynF1zUhsy1QYsSfLpSmx1OJWcjjIzgV4O49ONXHk86BJDg2/rgGToDGBvB83tWWu0YIttplKsLpLLmfo", - "73xcNYi2DfsN1AzcYCVEhiCTYHw/7OhKg7R5Y2lNvN29CQYFsFU9UfjohtAO66orlvFVTfzk7VKhwRmx", - "S0bdHE3RwRZFdFz8RwFfYHttj2NQLwkZiFcGlBqpFlBcGSpaLbw6pxKNuRAAAq8lHM7cbAB2Rcs8eq0d", - "7hU6n5EFEiTFlA0ZZYWRFMDUCGJkToSXJcuFVrKmJO6jv3kqHsDmp5la2HoMYDz/TiJ+xYoxDpk/SN14", - "LnU7B8xYFkWeqUrFVt0saH2aUCBrGZxgSkAJU6pmaCKInPlzD5Wt1TLeFRdxYz2wBXKvQJkp8LEixS8J", - "81lZ0UxQNTQNjcxXy1F8puY0PLX6J6qUgUb1Ms+r+8slEWEhsZhS8Uqr0BXvqHjKiQGBAUQUKO9p/zIs", - "vkBBaYF5Ujb/V9dk+dNp0Xj1t9prHq6JQ/o+MGbboAk2Mmk8tWCfqidtbagKpMGtQrNZ9iWgDRdC7Wok", - "VSUBr1ZRq3uyXSZePVnAjWZLkqja+97zJ8+etiwW9VnOOoPe9aVdc/N0hUuuYadO2vh9nj95/uLF7t6T", - "Fzs38rC4vJKG/WnKLfH3B22Qa6UPa/Kvf/zz/UnN6/MEYrAHNxqUySwJD6khu6Q6oPcn//rHP92obj2g", - "EKNZBulv8Ns3Rukk/k66QIGqC6+dk2yFfn9QMRLggs2gDQJQ3HRORmbdeuVgajAg7aRgnOGIqkWAkeMr", - "E+1evFLD2m7jDqoONiTymrYtKqrmXDIfl0mnG65z9J/GN1yjheeta87JfNzkh35T79V4oUuvhR/j0CLE", - "wFBE2MBdzOcKy0pAt/47grwLl2G2nG1j3liNultPhYAoFlta0QsFDNUaqMmT9iN/+2vb6fktK2ad+op/", - "WHEOm4/gjay+gRs5YPSN1qfW1viDvQBv99Vo7FeDXFlus1I6srx1b95vi+zh5VIlxQ128/68hMmbfFjH", - "BAZ6tGOwS1623a2QRAM1ebkwAQMaT0jPK15gSm3J3HgE9Zm3MPOBDM7okk8mVazbJ83Y6AD7A8lerhes", - "lNZMuohcO5tFHVjbYPwMO0/ksKNVgGFnOx12am6rYPpkiq9HtoMqtstgFVh5mf5eG6R0MxgnPLo0hQ6h", - "fn4fDVBKMJMoZ3D4a1617cFq71C3k3l7U0CDExPitMS2YExjMsNzCvVUrE9lWgnEJNdUSQgYhXb2UcwN", - "2lOlyrOdoX7NJDful5OGSwezhW1YN6jf48xFtJbvgoFvArWl2UcieNeCFWiO/ebNSdcEMEDooRlYJb7R", - "TdSMQDPIootaeYXy93D88DghIxh3Ha4/XV5HPycdPKuCSKKkxe8uyaFGBCjiOVN1HP+0nSJXTStbvpJy", - "BsF+NvwDcNls74ZAUEwiOJFy+SxWCf0WxF3LG7ArHUoc2A2RMByKFfV33lqHcH0AxtjgFWg37fhx3cZL", - "OJKK24p+xakekeuIkLgO+Bl+pW2svP0yGCv/GluMoKJ2un0b4p2XZ9e/uwQvGGvTavsx/YyzHqCTuC21", - "SCIGGtBi1VQJrQI97kFajELwqqEX2mRck+vVa/0ruVaAjx7niQG9C5OuZVX2Mlq34rfObGw60FyQtRUy", - "76BypIk3v1XtSBuq/hDlI+1bd1Iycml3zohy755ZMmrcoWqhl4pLywX8u1eqMTaGlLrIXvBoO92skeDe", - "LGwVsaC8LXM0GU7JKBNkQq9XEI95wSjGVViT8iAVGQwGX3Qjxddo7xmU/pK1sTM6nalkUQ3A2QtgKX1W", - "XVVBFGHOUNhm58vddB8uR7vZ7fRbDwnHZx400FJJEyuSjlbhZh+W3jZrnc/wAqw4jU7CZ7t7g8HuzuBW", - "wNluWDdYrsPyE1uFtNpOU0qd95119FeiVP0WiiTr5dLWplyd55SUShCc7kPiTYYjghIyAZC8IqF1vWex", - "3vXqwVuBymbRFvTvNsrum/PBV0vmFF1ZzHE3jY5zLlYxiPznaxyiDWwmWoLUC+Tc7fYGT8+3d/efPN3f", - "3r4LsOtikZqyPZ593L56luzgyV7yfPHs9+3Zs+lOuhvUwy6pqQzUhlZ/0e82RtmUl2QVy6jC0tCGnUNG", - "RL1meb3WvyQJZaQniwyp9WmKK3iB8b+vPf83s/ObGayUHc6qk/RFCKzKxalQ1sPgb9nJrPRd1GdzfLR6", - "FrfKQKoPJExv9aEAebUbDFSo2O58JjJDzlpeQ++8F1tfRCuz4tZdRSEPO5z04C43rHiIvGvADN6sV13g", - "y5dcwHY65YKqWbr6tiheK2DEIW76o1RxFe+pj46njGvtyf+5CJPzlSj9cafbST7uVc+M/b098pdFIC4I", - "0G61LxW0CCNLyJwkq1cBXikVD2Ei2bWursf8w3Zv+wXEISQf934Y9F5UIw66ZrX85dt2b1d+HbRZQ78E", - "oCsdtf3iRhHXbj1XUdAvNFTArryXLTaxpfGyPLy7OlzCbWWDy8dLe1xD8mkUQD9X0rOX28gXmmKS4EUI", - "m94z1Mqa9ugTGRqTKWWyjd12d1AYbp+kw04fHViAcNBlFS/68ZvXtOLTCU1TElMtYxrVvzmDYaelLa6u", - "S9ysNon7KiCt9cPi2ov1EAnrEq7WXZP9z8jH/Sztt53Guwq9A+xqTkUFDDF4sYvoBGFWK1DqilUbpQMS", - "IyFebd8BtZUka3mALOVAZyfpoilXqEyhb2lvy1mzXbAYP7kGe+sKzAxDEDtfBBClABCjq9jX8RHKBI/z", - "qMwfTWDQJeKHyGsQbSuE/PUhuXdp34DE7AkXaL19o8mg0c4+2bTfNdukJtjmrd4erN/qOzGKdDt5Fq/n", - "YealdhzsRsjta1IQAyaa6rLXJEFvMh9acPS3/gou67zGlhxpkSjPnINF09QyJQXcLeBiCMX1HpGE6Gtq", - "uRHEk7jMkqCy5KLrWer20+ezJhcneKSWB/ILIZnWVQD/CPpLMVsEB+bKjhZ3ycbAoX1L4/DqmXJFdrWq", - "g3u2VhJr3CrfhNtUQsFw+ZrN2+ClXHrm7wLj2xfNlhFQHMOvCGlvm0sA2C9d2Fuj/fguzHIPKaS9sa6H", - "GmyrAxUu0NFd/2UssBbrqsS7F3LPh8jiHE+hVt4aP0ojqYTqSoKTAyOFbXErKLCp/4mgiqh0gC1OQIDP", - "XUavLTaIqEJ4iuuVI5rKUhpP9PoYouZCL+fWRNCEyVtPh/XN7we9/zHmdjTq72/98Jf/u/fhP4Nm95oB", - "QRLRi8kEIq4uyaJnqjApPK1eoX+YEhBaq5jaM0NwCsY0QHu3XMkf75NBwT0Xv+J0aQoQquaVUNpeO6G/", - "/EdzoJe3jO/gwlh7dj+7QspdVJJV3N3LGykRUxdU7zLqNvtDBkl6l2QhkVeYzcp27sR+J4tPvFB8dGHI", - "vU/Y/AKNKVS6lEOm1XscRSTTapWt9UNNuXYObFgQnPjt2AJx7rxYz6wJrCDo/ckSnPGbd+c/vnn369Ho", - "zenLXw+OR7+8/G+IdbnqmR7inqa9vSdPbZF2fyW3g4VCbl7voo9ObL6CjXmY5KDZA2CZRGmucoiOIddR", - "kks6d55Sldy+ssVy1vLtK0V8JhSyUkkoPMNCdid0QiDAAe5VG11EpSNGKqG6vbXyUIaWRRdDOMMOXCmK", - "O0kiWFdEb0V4tcuNrS7601m7u8SAxAYOO6T+6sMWsKK+phKAO1wQjPcy2oAUGleC12UQb94MtPagaDAY", - "gvmFKy0NXnyJaqjvVpY/nfOkp/W8hpIRQbO6WYtgCgE0ZVIzOk3el+k4oMxYG/eUTnHA4RJyrHyRqqVu", - "QGtTx5b2v7F8Wzih46heT8McS7NUtfoPNWuJVL3mfI9Ui/cNCMQAsWySeKkXZFjN2E2Z2rLVhUPAITEH", - "VPdVadvlKXMwkT34aH028koF05uZN5LmvTlxelRN01uxQKd6aa5mRBBvI+CDsk7BDZfMJii1gKMx1Rkz", - "IsrgXZfdpMVz8LtLtFGYwNwSFGnXy36B1XUoTvB10QP4lLBccsTCPMo6WNuvfoSaAW9d7U86cU3AMGpa", - "bhghv0pFq9bEUdXyZvhUtTxv837w4FletYL7NZ2tGnGWfVRIM0SPf8NU/cQF6MXN4C93DrQPl39MBIDh", - "1WH0W2HQ05TEI56r1edfv6alR3PlF/Vhy/rCzgaAgYijSl5zEy9w8CTlGJZXWi8HiXJB1eJMr5eNaod8", - "UFfUFxYSOoKfy46hkOqnT2A9nwQyZ14RRgSNoEytPo8pZqAxofcnXrVCU7hyCbgWRKA3h8fW7uKwj0GP", - "pgpIzwWgHpwed7qdORHG9tAZ9Hf7AzjMGWE4o539zm5/uz/ogFY1gylujXOaxDaR3GrUhQZ/HFtJ6Ef3", - "kv5S4JQo+OK3ACQCBGDa10EFwVNPicwwFVaLzBKAajAEQ/XXUHfBXaj75lbummVvbTyGfGtIAyLZG7u5", - "H0BQhrMD09wZDCzCu7LXLyQxmcyJrb/bMNqy31ZSnV2iQBmCJTXPyZbF0n/qdvYG2zca06qhwNkNdfyO", - "YZvNTEA7f3LDhbhVp8fM5CfabHMbF+afOCAk/6z99kHvmczTFIuFWzB/tTIumwRjIhF27xo9TkkUaVYB", - "xZL66A0j5jnCCmETwi1yBjWm3YeaQqunwLTtNrlAa/qRx4svtoSVPpyN4lOVnenj8mmJnr8c7RRkvLyR", - "9pGDGjdUew8E9CMuCqQ/2EnZG7y4+04POZskNFKoVxCwDcymEmKfEgBOdyBMXKDfc64wKvIaHtGRtjLr", - "uCC3bnkVbf1B40/meCck5A84JSLFzGSJmHfWHPql42x8M+VxXnmrOcI/PurYm8qhEZmLCgS56hH1r626", - "MLh8He0FoChsn2Z68QMS/t49nHA72aJG7kMeOahMinJJHtNxsr7GcSmEBGW5V0R9LTQ/uM8ry1ZT+BOe", - "osdCwK9IIeGVu7V0KWxlImdGAQ5KgG/LzE373XdV4e+8fOKFC4FfQzcNdT2UcTDjeNFHbk2N0q8WgDUl", - "CMwzXr5WTvXwvpYTtnMfJwxmXHiKvl1T366pVafcUIubAhxM75S3sEHcyALx57M/3Nj68M320N720Mry", - "wMiVtS78nY/7yIbmRjwmSM54nsRoTJABfnJBOAqL/vQjwiKa0TkBdD+oVpcnimZYQIhNimKssPGhNxom", - "Vpoliua2dHM9F5BZLnAd0EOSEYTAjJqAOMtQTMoYiZH+xEbNlLiKS3XVzdkPGtiLBsurEV3NuCQFsCFT", - "3m0Oed7SaMfQbH/Izi3irV5AiCp3vEaSBHB7V9h/OEN4yOwH3zsW4iLiJE5LzoUFgCdSA9FptmU5x0+P", - "dCQjHgIdOicMM9WTGYnohEZ2WpdkYQNbgw22KkClB+zG+f6kyFxBO5th4DoIXQqjFB8Vz5ClpKr/hkE0", - "eJTkcenkclhKWIxxkgQrlEwTPsbJyKzPJQn4BF/BG3ZRSodL6U1iPCamln62UDPOzN/5OGcqN3+PBb+S", - "RAw7m/0hg4wUu9Yk7pYCIrqCinZpxvU5Ezw1fW6ZIW79cUkWn/pDdhCnlDmKgE9wIjki1/AdxI0BeIjh", - "Xg30YE5T2A9+mEvFUx8C1tGdGSbPVZYrm1ojieqG4E+HTHH0hwO5/LT1R9njJ3AWExxrOvFeMVMC2bpp", - "1HKE9exH8GrA3U5gAYYdfZGaMI+pwEwZ/NICpRNN/S3dKMpEQOnY+gpHmKGMZ6bEBhDVDGuSq7QBoBU4", - "SZCCo+S+1YI77GTDfCwGYTpuBCA0iHG1Y0QZOvnRO0yDvefh8yRJJEgoouS/zt78iuBW1ntgXivDtUxu", - "C9MCA4pzcJ06nvYSRzNkHFVQVXHYofGwU7hz400Yay5tuEyvBz7FH/TQfjDddGn8Q7+vmzLuyn302x+m", - "lX19lrLUAKIOO5+6yHswpWqWj4tnH8IL2oTjdlZhBGjDXHObwEkwBcgd78Y3VyRmMeL2FkgWCKOSA/mB", - "K2PKsFisyqgMLL1dQT4xkYzeYvwxhMjFYWd/6GIXh53usEPYHH6zAY7DzqfwClivZXMJP7jPCudmQURP", - "B4PN9ZDgdn0DPssWjoEvrAM2akVF/VG9gxaP9s/lH/i31j8L1w9muvMSo8ko/s74/ggdEJ7E7muiARdE", - "TezGLCKJE7vXG3ru33mgNysiSXLfBPpQ5Fm4x4qSBY+KHGGzymO00nz/wBQ3uK9LpWK2fxj6fXT284D1", - "3NrOydyFOocLtgAYj1WlkXkZYYnOYEy9M618v4Rf+/a/TvcDcMmLhE8v9o3qjhI+RQllNh/AC1TW4oFd", - "S/jI4PEU31l4Hlctb8NIEv/6xz9hUJRN//WPf1qQ+3/9459w3LcMzhwU276YESzUmGB1sY9+ISTr4YTO", - "iZsMlMMlcyIWaHdgbf7wCHk1/62UJodsyN4SlQvm5U2YwnXSNmhdBXo+lOVEWjwjSBOa2Ko6JrYxYLdx", - "Z9ks5b2e6G4AFxJm4E1A34qOBgBUj5qK41YT7YRNpmbOFaNpPUxzKVhvPX9R5FoZ6u2ZAd6QwcASh84d", - "PLCTRhtnZy83+wi0LUMVUDkJdIeyGatG9L/xpPU8yXCUKkOBVTa8KcIZHtOEOpNjQ9kXcwRTHM0oI2V8", - "cQG67prYdyPVPObg9BjZQMguvDpkb862wMSqSKRyQbqWEwgLtVrWheM2zwV6AP5FFUSH9ey7QzYhGPKE", - "jo8ME/DQyIvEyKJhBogmEONKVaUEXXfIDKSuhXDWBy/lMUngI+h/ihW5wosuKor+ujIxCVZaIZZd/fKQ", - "mVRDuwY9wGxB3jD7wM/MkHouktfmbAkySbRqDBH4pv459L0x4QLZCOdumVfqujPZpmZYetFSHL050/Ob", - "gibIjT0QWnpz5nZjs4skR1FCgRoizIZsCoFADsWYs8quFgllMyziXsT1JeCjWl0yfpWQeNrEYw99IrtD", - "SabST+A4/Vwn18cmXMyWJ6APsUHqW+25O7LvtHPd2Rb/TL47WxHzBs47Y8Elht+Y1f3myGvhyAuvm3Pq", - "hTxrRw6K8u4ifk0XDxTw62hvec3NE2/JHsKihzYcxg94RbhAp4fHCMexIFJu/nvb+/RMDZWW8p++HzUr", - "fojQEzsWLiz6obW3VAnksbCDt3bUCLt51QsN+/fbVqUKUeNNVxQkKq+8u789ap3e5Bophd6S1r7dJGuD", - "bamMONRbLKmlB6JRQgrxpTinPhWtsyqbMN7iylkpLln2fHzkDuT92Zdt1zmr3w33wBSPagzxARlhNdXa", - "Lx/+mKj5XbGLDnZ7hfn56yLNwf1JQfdtig6R+WNSF+PasmkuaIBOGi/QV0QZeJO71NNtD4GJnxHhTrUZ", - "6MLMupiW+RQZnBaYEFhiVuu+x+aVdqqvae/PpPnC8txEYrFL/k1EaaHslmu1SsE9trWw706/rSC53XPY", - "iiWwwCKDFXXs3E5gWd3AcsGizW+RK1+cok1cY6nECjdvEheWbIOmVOhZ9yXXHTC/8LqW6axeSxmaJHQ6", - "s06AmE4gVk/5hcxhlDv3MMqiYLjAitgQxceY93uqF9l6gedEeEiN/pW69QcEra5XlRzzWnm7vnv7ukdY", - "xOPCedIsk9onX1hhMvRfyeW9/1P3CPNZqRMPmgTGz9h/E0yOCujO/7XzkwXv/F87P+Eko4z8r92DBCsi", - "1eadEcvgvm66+1ZgHjHxaf2FVhdtiTVtKTz181CrlOnga29CmRXYWWfsLhJpvhj1fXkhr47V20rCuze6", - "15rUfSOWHLuyBGZLuXDoVTVI4n9voe+sQtD3bq+rdE8ljAAEu0cYpcyFQ13FFraaOuK2vIlNAdd7jTGi", - "eKulPcK9/6cySZhJ38goUazrN7tEG7uEv1wrTRN2K+7UOGH6eCDve0FsodWGR9/gtu7BoWMp0oPbqni4", - "S8CtGZcKHj0+7AV7ldCC4vxro6VnsjyQK68PR7rHR11YSCj7D2WHbGrzPfkp3TjuXfG2/d6/0HOQjuk0", - "57n0s6ZTrKIZkRZRICFVBvzYTALl9dxoFPiKqXRwn1fHvev83+j+jqwR9Q01zNsEG6yT+d1bbWV++76W", - "+Q3askVdsLWRuq5u3mZDEojDW25LxhVY6uXklNC4QroIeqcVlVJdQKBB7A/Z/9b6x2+K4PTDDy69Ox8M", - "dp7C74TNP/zgMrzZiSMVwpSgtszpwa9HEOExhSB+qIRagknUx4HSXCpDeq7kyb+dglQGubTXkBwVftOQ", - "WmlI3nKt1pDsXtytilQtm3TvOpKjt9CC23oHf04t6U/uuq1ocDKfTGhECYPiU2D8k0uxykaT++a1vSVY", - "ArOxEl6gY0USaa1GFlxrjYReFv7/kpGE3cYaFBxhpUiaKTQVOCKTPDFVW5Cc5SrmV8yVpIAJuupmtJxP", - "6Hp3TY1cI+EE2XCJ9raablGW8b5VXdvxI81Q5ZmtMG6Vy1K0adYuH5Z471anbHHV3r9W+ZhJzKhvy0uX", - "aQ0hUGLNuHnS3KTzFl+W6Ix9dH7+2qXuavVEuIJ9irsqfa6S85D5Vfr66GVZ/tC84FrQ6gOJbao/JDTb", - "uncxwXFCGYFcByJDWbbV2poPeiy+vAQcLhx63y7uNsfS1sR+OAn4wVjBvciahU9fWYdscTS9mqLFaXHy", - "JpyaR8WvLAMKMJ6QrLeFc8V7Fgxga8YNQmQYJPc0wRFg5OrXDHyjxV8xeK1+UwCqIniSEGFgObNcOXFr", - "yIrBUaZ4UXjaSmYXuvlRzhRNLrom1BCwlSTCbGGx6Yas0pmV+QAjAfA/YISCZGbEtSq6etCU5xLeAjgD", - "v0uEkyu8kENmURXM51B6XZDIINgmSR/9zAHQxlST9hivKeX6nRyyCxonZGTxaC4QlUjOuFCEkRilfE5k", - "tV+CRUKJgEkcYr1yEqV4AcCQBiPXrA/PiAFfrKDecP1vzGIKRUF1z8WU94cMo53BAKUEM2kxLCSewIVj", - "20AwiMqAvkcY7Q1e2K9q+wbg5W75N/RpEoLMeYTHyQIRTcWm2PYmbGBqi/Saqu96+yZUSLNfhX3TVl+s", - "bCyVruZs3EU5K1E6wNafswJUQ2+XygWDeVovIKGiuAYtMNGYRFivJ+PVfgASlkdRLkIXpN5qr1r0v6Pg", - "6E3vDJYqjIGRgMkgIjHsOeNqBmeaw1Ha/L6Bqkqi+nNcNMFDwgXCyKPr0qJBohxY4wZAqF6UpU+ZK2V+", - "sfm9Ozv6+FpG4I6/ATF9LPcTEBGfTCoHcP3VZA7wqtyzZRL+s57TQ1fz2mdxMcVTxqWikWOGDqHaV5q/", - "KYStFMLVKxuk5gkXl80Bxz9xcdlWA3OBkY9LEfNn+BU6IvTwAAT/4f0RYA03yoommntX0ur0VZxSELqo", - "kkVIMEo4m+pTVFrl791t4Gt1GwbQUl+mwji7C/gxrYSM7I+mbLYX1g0uhsi2+tC8SPd+D86oX7lCNM0S", - "khIoq90zxKY3u4SqGy8slF4B2HYzXqlPlY+rYHRBaeIPuk4cArpyG7YB0vvydgWZasKn6wFRi84d+mcA", - "EXXI3klTquDCuJ4uUMGDtUBryo+gqxmNZoCOCnqrbt+Ap+IsuyiA4Tf30Ss4yD4+PnS+YYqOaFqTPCEG", - "9HSephf7y4Wj35+cwEcGGNWUiL7YR65YdHF/SP2Wj3aqZ5FgqdCvFsN1o1DGYUcvFNb6ZjG/TYuDWgL3", - "D1kIE5WRK9sgnaALDx71ogG7z/Hb13wqvxpXUVluxcxFcWRVR6BNwuJOU5AHTcKOn+3BIFQFoCVKqxnG", - "HYO0Lg3mNZ8WpV4qpIyzrC352mECFc/TdAUNow0P8lGqmOfqL1LFRAj42FJ3E3GjDRzZMn/4UhOqBfh0", - "B3sTyC8YymRqLwSXSjPVTrdDWJ529n+z/5qnaafbsePR312pLPVqN9xAyF+DeltvcDn0Ru+QB237TTy/", - "CWhtlel7qLW1G8Sq1c2S+Vvzwp/ea+hsdw9IhiAn1Iy5X5Mo6o23avhhvEDfhZG9uI+RAUQvihIuScXR", - "83gA/qzBqyY7NhuM3Br39PDi3FVEaxPJcmY/PXNffgU6+LqYETdm5KZ778EjyyN4zGAFcmk2Ey7qqHDr", - "okq+ekL6cluyNNU2FPKNNm9ubWxFmFpfWGYR9oPYVMjEueIpVjSC6mzRjHPpkX0B4W7qKFojckGZYGIx", - "2q7NJLjQpHphzdEXVp3Yt6YzhP1Hto8+fG7zD8JfuEflFz951oGC43edCgAVTCTCaCwomaAM55JoqS5P", - "CYoWkeaKphwfwdEMRThTuSBQaZSglDKa5qmPza93bI4BR+hiO73oonGuUILFFLQz89AF3UQ8TQmLCdjp", - "hmxG8Jxq1VKgBCvCokVPEqhQPifoiovLhOMYTA1ZjMHjAxVOBdEUCIUOUqJwjBUGQedCn/iRSWa6KIqW", - "G/WekeuSGuIhEzn73lRd0c1euIFeIAJlBaicFcVtIxwTFgXh9s++bjb25W3SZ0TVJ/pAEUK34qUPGTLk", - "217dcL6OaKJHCw3Rgs2vEHplswpbzQJxZPTveaTNXN0cH8jRVCzxqlP8dXiYCqL7arxMD+9G4gLFuenO", - "O5VA5n9W31DBUPygK8gwNdt4WwdRUcWzWOYb8bytP9yfx7ew5X0lnLDbqNg31YsrJ/01sFy7qrfiuQ9k", - "xLS2JN8m93As2EV2PZj4xIXH5R6LsbWC0FbwbZ87KYFB++LsG9uus20b+HBbtu1ss0uufY+RU9aDWNEw", - "B7dm3EZWbU0H/6ZZKbXZeSzzwVlk6bm4d7hFxxozvEg4jv8MwcIr/EcRF8LAYACwxmOCiPashn6aANjm", - "ykKUXZe1+f7kZLOJSwi1kkcI9Yg5hJeaoz9L42UD7ps5EYLGDgbz8OTIhu1SiUTO+uhNShVSHF0SkpWZ", - "LZBd2Nfzc4AgtWHXkT+6HcKUWGScMrV2FOWrdzOY8gducVG+QlHS1hz45g5v7Q4Hy/7jY2fAZSB3w0xg", - "tWaqsFpbC5myCRepkcvwmOe6dc2D9DLp/TSIBROaELmQiqQmOnGSJ3DcoH6NrVFuvzO73IXYXH1yTNpc", - "RkRKpaScySGzOSMZEbpv/blu3wu0CjoEFC7466lhkl9HEJ8ejIlbw6pp1QC6CWofd/Y7WzjLtmKscEOg", - "mB3eZwzpJ4jKQ3KRjnlCI5RQdinRRkIvjXqC5hIl+o/NlWF9I/juS1dgv/3J0it9zCY8WN/S0GxBzH+q", - "7C7L1pxj8tGxtVfEPyyO/8BGh9na+hrvguCkBzXTHYAPyhVN6EfD6nQjVCoamdQjXKzd+5OCqfaH7IQo", - "od/BkOKWJAbZALTLrUzwaGuYDwa7UUYBBW6XwOCA4TU/TqHHw9N3Jh2VpFwsukOm/wENnx+cGu/uBFtr", - "gjdQW9wdHW+9WRPofAbL9G8cIWgmuBLFILjh31yCN8caaTxDsuGI8myVqsSzP30Iq5XgvtkVHqddAcCe", - "itlsFABfDpUrbEOY8yRP9T/MH8fr8M0Ujmbv4dWvRto1w1nbjZvgoziUdk4xMfV3H8TpYRbsscas6oVz", - "UwAhphINGLwFDtSfkbq/vPneX8ev0N1pV9TVtv5qztZ933x2DA5pw1+Px3LMDaW5mSi+2vp0hWmz9enH", - "hEeX0kKy+GZDrbcBzrr+scTFti5CEBMgQxRZKCMDmEVkd8hqBkiD/CMRRoqIlDKcbMGcTSOA8O2sWHjO", - "KSRqR5Cn0pM0BuykBGC8AQZPzwYMVa4Bz6MrbfU//x3fGak4GpOIp8Shnm+GVLe/Yap+4qIKYf618MVz", - "b/0BGhBTsLevQW1v7vGzUNxP8DWESse5dSi7EW284uWPxhTURbA3w87uQA47XTTs7KTDjt6BQwwmVKzQ", - "E5RSlisi++jI2LcgFffpAEkScRZLB77uLHi7A9mUmGvIsiHL8yl8d59ij6UqWMq3tpMQe9DvIf09JO2g", - "Df/A2TMZd+HQxYjnypj77bmyb8VEgXlk8959td4Z+abbt+Hkf7PHt8KjYJc1u/S23nD2LJcz0mxye20K", - "GuVqDKDergCynKG/87HsIkaujDVcSNVf4nv661PTwX0UHNBd3aTYgJ37t0oDLSoNlGsVBm00AZb6SnbU", - "YZAbyXXGhQI0R5tzb2gINAlAkIAyhW8Oj4cs0qzIQAwKknLgThYX3dzCB387Qy8P33bRERTjRT/n480+", - "esOSha1hb300Q2YkMcO8IszQ2FAtiUPXsxk7UM9dBovrDh6our05GQHPitsrFyTe7cwIjkEi+aPzmpvO", - "AujDb1/rAwQAwObLYts7K4WPzluixKJ3MFFELDd7YvOkWIGdYS9pB0VnBTcDgKk7lA6BrezTyAYGImN3", - "pxNAzPj0rfjD3Rdxvh8vmYkTMWX3xrl6tPVb4SAWzDHEAv3ruiif0JQlbHnZSgUDumyK/P6KTO4reVcF", - "Y/7f9XTBTB+toymr7JMm4qLsylpPr0sOnhlYZOuoinCGI6oWXYSTxN5R9iYoIlJ6hfg7FgRfxvyK9Yfs", - "bVHwxSb0osPTd13nqEUxlZemBeuL7aM3cyJkPi4Gh+CgGa8xrDmJh0xxFOEkyhMtbpDJhESQiwt1XGSD", - "L7cYSucOz07ZSbDojBfVnj+6WndhmoDdK8miTnFbZqu3BIkSTNNmEHIrqEHAIYQajHWjnCHKJokNqYoE", - "lxLZpnokoVM6TmyAkOyj8xlBEqdkyLIEM0YEyqWJitdD72WCSJmbBG/dAID1GorqohJgMBNc2dCEhHMh", - "TTSBpvD3J0gqkq0gs7em5ROY8x3JtqZx29MDGalrY2g2hdhXkN4QQylmwTUd5YkLYLzXUHQzoIeWEh/L", - "wT8XdDolQp8KbJisCcczx9otpzn0lYzlxrqXZ8Vb7epeFq16WYlext5KgLhRibkdd24W9Rfo/JI2Ygja", - "RzfLIv5Ff9Sy72q2angQ9tFnzjJUwvPfsVrmmZck2NaAVVL4YzMneSOvHNVKou16WK3WmbV3menaGj/r", - "wWCzHjNaFq6kzzYpvF8fIQzuF+XhvoutPW7aqqBdVXTThpT/9aj6XwUF3g2c/gOjnNwCTv+ryrsHvPOH", - "wz8JHtSHyqOv+J5d0d0/PSL+XaXPG1h8gGNrSp83XM8Gr65UlN7bd9qpSbbFP5MEb+MdbyC/u2X/pvW3", - "UBm8xVrngtYET9JMLVxAm/VVlkFnkn4k/QZHcBG3eneu4FuEdH458nB02hjQ+eeskf8gMaO2hCCV6Pgo", - "UHz+kWEM+meucrFs6Vunh0U0o3PSbHSvnmC7RJkgvYxn4FyJzYLZ9XB3mcKiP/2IbPMWc9X+C2pQAlQ/", - "iVFMBYlUsjD1QDVHMH18J5HgWhOA51wsmqNEzBH5SfD0wM5mzX1oz5Q1hpVxhumiF2OFe3PHbVaY0D4j", - "utPFU2qGhyhDr35EG+RaCVPpAk205oPopFhSch0REkugyU1/wNuDBssm/UhG03GbUa6oWfLG1oRBUS4V", - "T93eHx+hDaiBNiVM74UW9ScgyWaCz2lM4soYO3OemFXdbljQm9pdtVBRFLBzyoUZ3IPIMG0upOlHmlXZ", - "QhESM6YMw+DWVgWpnimTxK/7w5S5ABy7R24U364wq/ltOGVHUyLU47SLqDg3EM+b3665x3zN+clQ7k6r", - "3HYuPGe18bpdflTLtKW7KPxQ5M7dr9n6/deT0kPlo8zmsabzeaGQNpnNvy4SHNzf/XDf5vL3jzgF9BVx", - "yrdnKocGdIshgnkNMd0xmZOEZynURYd3O91OLpLOfmemVLa/tQWx3zMu1f7ei2e7nU8fPv3/AQAA//8A", - "qZIc0fkBAA==", + "sckmrER/wqvVA/Bk9+nzZ4MX2zttx2c82e3Wp9DL3Vdow67UX1zpM/ekMqidnWdPd3d3B0+f7uy1GpV1", + "J7QalHM9VESVZ7vP9raf7+y1WoWQvvfSAU7XBftQ9YMDUzAI8kFlRiI6oRECoDakP4BU+hllpHDaVs/q", + "GMeuRkT4TlGYJnJluJvpzL5p7LBpniiaJcQ8gw1p5aqAmR9BS8GiW4wVNTlu1pKF6V4bCOXmUrwCYk5M", + "xvl0ajHwiqU7sentpVBFSRLvo0KVW8n/YDfLgX1oogM7h5bU8FqrVBaNziMCc6UZ6AZBUEEnZtMqs3Ix", + "EJRleZAkGpfyp1wYVApoFOExz5XxttsiKmUnBuCFgfc5Z3E7M8hPXFyuTXjQN3RRK6alIcFlvpvM+FrO", + "uy8MFl5xEztgn0v01nxhDIjlz1leLZTXhZ6soZG5dP24sCfbZtpKnWF5BmzpLozG9FfyznsK8etNTFTU", + "l9W8xZRAjSS1VpLRlGLilM7g9db5U/rDtQaWFuvOyNV9LDpkAPY02fYkw9ndrPiqSOLCBuHl73Mbf99H", + "cLog+M8l7tdO2pmDoLBL3h8ym4Dk4RiAgw0wBkz4+IxQgbigU1rtuGp/vcdY3ptQpiOuW1On/2FTbJ4s", + "3IMhHmAQmCcmNNYUBvEBYO2edLqds6KYlGVM1aV5WxTkWlqRMuJ+GYvn9N1NI3IzwSc0BJADEUL2qVXg", + "XKzq673BWW/7ryaaXpMfSGyUmaiilMe12k/2/XYX0avTd6dNYyqqMSF/dEtzWgH7sxwFM8mZqQfl2Vqw", + "rdNokdgMTo/prI9+JFL1yGTChdq3AS7gqPJqweIFEiTONT34kGWcIUnHCZzRoletCOhf9IQAIGScTyZE", + "VNEIXoRkae/tURow+v2knyPzQgGHcPJjVZ7Wcntbbf60Qg2gzk9wRNl0s/V2BwyFtWmswxt8dfrurS0n", + "1ITErJeyKDlkQJj76Nei4JZealmiLfUDFsQaPZnAav3CSCY8FKXykyAEwbOy0Kc+e+DZNti9RTQBjgG7", + "iTNXbhBtQPRXTCT6PccCM0UZidH7n+T3aGBd6e9/Qia7xjIgKv2Ax6qm+Cy0pY1ZcqezhaQRTsyymJIf", + "lPnWSzjSrdWM0/JDa+cNKBvhYnOOfaCN+TTLYQHP3vaO37zfSmMy71bGBFGKM54QPe5Nj8fOHVRDmdJX", + "Ya3zJjOS26qi9uGorLdYPwiAEm3tm3Xq0HszQGMSYQc0Ut+8qChPPIaYDLCGXQmqlJZcsTVGQ1RN2YgB", + "05NokidJ3zJAN2Rjfp4AH/aQTc1VpckHnmcYNKlCjLpW0ClBMo8iQuJaWEt91B697aOcpVjIGU7qk9sv", + "QsCjGRY40vfhd9ffdcL1hfTgZFtG71FncdO0JkvvXgnQo3fumo74+5+qx9Nc9SZp2BrOKaMKQNAA+Xqj", + "GKTB6u6jE7ywdpQM7kxLL+WO4cnEQIEWd4kgCcES6oJL9P6n/tqATcUVTprmcK4fWj61oSekT5EeZhdl", + "FTYAvMvD4PK7fRq8Msr5BNxP0GHdl9FHxyzM9kxUEKDoGUAsKemUad1O2sCgCLNiJZf2rnp3Bs1CddTB", + "TnXhKtPpBi6AEMWEbiuT5bis5izXKeWXVTbFL9fXxjSNNPd76BIxa/4uq0u6cwv5mi5CCZlPLccwVlNJ", + "MiywIsmiFipdrXVAloNNyDWJbpAJ+lK//snUt8kFGamZIHLGk2qI0m53uUayhHSAObFl4cycPJ+c4ijF", + "4hJOmdOlUc7MClSzSXbXQT/NlMpuMKmfz89PjYFNETHHST0fSS4F3xyRBC/QmKgrQpibCpYI++Ho9Xxu", + "2VCaS6hRRgTl1TXs7Ab6PTMpCmgqcESQ+cpVWC/Ymj56bZfS9hIAE40iImXD/m6v2l/76SRP2u1xaFjb", + "63ZYRTfZ4PPDU1diqijh7ZZ5Z3mVT4nomSPnanmv3todubrYmeuKGYzoJZl9TKD6mU0C83O0XeghVHfT", + "n1fyoj3mIP1MXtsPnAKzVF1zzj+0UvDqxz0UY5NiFodKoZvcH4OaMQUoPUhAEDloXzQ2kpK5k7372aYw", + "aUmLMiJlDXIgykXS6XZ6Ezur/a0tze8TQBPd291+vrU6wntlaL+NZBzFdJWJx8U7mog4l+BuoBhh0lWS", + "2MJZ1sIIbtZxzf0A7Gk5lBjKn+u7zVOyXG7JYLCEnnCNI+UKNoJVvBINgf1jC5jfVUFGN5iaGkxa233h", + "n89BMDwlw2pWJf+tJdqHkDUIN9Y0YiyOtXU0RP4xKJ1yEarKx4Wyybhj4uKRi/vQRfs6dOuK33zw3J/l", + "0ydPdp+s40PAbGrH3J67wFTN21W4AeIttz2/tgGInKzKHO5Iry4srNdlDU1pjrhEUqsXlGeE3Wg9n+zt", + "7txsPdtO5NhFbNb4Ugit6fDkyMhEWpfHlBGBUqJwjBWuMhkwJ2suA2WfMEkhY2/y/WrW0hDa5MMv3bZm", + "3ZcKjGkoX/nWobWnmNEJKEnmTb9nOcM7T57umyK7MZnsPXna7/dvCkrzskShabUVWyZ+1sOn6cvZ5+3D", + "HWDPtJnLH53Tg/OfNSPLpTCX1pYcU7bv/bv4Z/kA/jD/HFMWxqxpU5eZTpbqMVdDRXMLCk7ifVSW3ndy", + "T5vQvQZ/ECQOQKhEEAGyEkB9d1CPBY3TakWPG6Ttr0hj1+LKG5YsGtfm1iWWWWELVV5pZd+u0KLMMv24", + "OvTFWZzhHdunsXQUFaiXg1tuVUNcriyzulSFLyOsKKyaJOaviDPA3A5VWa1cke5ZiyJ9cI3YanxFl/6P", + "Re/ej4f+QLzfXZE/7ydbbvXDDaPVVgqkf1uWQ9dzISeOrjnMYfN/cSu0LW1tISKDaRoPfBfeJqKz2vub", + "6X/9/n/k6bO/b//++v37/56/+q+jX+l/v09O33wW/NBqdNAHhfj8YqieEMZYgfZsS0onWEUBG51W/xpW", + "2D4xFgcVzaAYLxqT/SHroddUEWFKO9bykocdtEFAU4KvtLgLdatMSuim/vjUBBXoj/9wYvCnehuxxYsQ", + "dkMKGCCZj2OeYso2h2zIbFvITUSCXqD/ilGEM1OfkTKk9d8FGgsopmm9vGXnXfQHzrJPm0MGVllybYDu", + "Mwz1CCeFb4K5GA07KhOhbl8nBSKMSRYesuK2LuAxjau/X9bSoCSpp/M1LMpq/c1qTs8HISBRSDXTGwn1", + "okAFKShbk1GRA4eeDzaX9bk1OkZBQyvIz8bCmFzkgzxkLm7KXz4iMY2Ar7gU3plN8i6ypw2lWSNeJvj1", + "AvbmrckrjRHO1UzzoshiXkScX1LShS3tgkcagq/gSxNSM+NZb7zozXhW4J9gYQLOsAlKqSrZ/6dnJ9p7", + "TwSd2J6CMBaaRAJCJxwZOzOTDVxYF5Ymdm5K7TAt+syJfd3UbJKmTI/JslC5YK6SDIF6sgC0U1AfCcnk", + "36MooWB1kjPw4c0AC1PpZkJwlp1Bkc2Gx1FMJvV/V6OKdp48BQ3W/Xt3p3UyuVm6VVSWJwGdNnWsrwXH", + "NmwSBmDEg5EzhK+JA9Q3oA2lADuF4vDfM+QaKk9cwUiMp8+kt0pbmCSRXmLrZjAD0R4Di+YzwvY0tbmP", + "lk5hJaGxRQsmQAg+S1rgCL00idTnr8+QIiJ10BYbkd4dOCUGNKZHpcxtjbyDw5OXm/1OEAOt4tKCrVqZ", + "8FgddAAGxQYMNcVBlTYanJIuOj6CRHZ7rZS6GGQe/cQFSsytWF5G+4CjUzX3YFNd8/jICqDJoow6MmLL", + "sLPpWszq19s+eluogLgYSpGSXNKWa7K8TKBZG4Nq0qKWWq9lsIN/zKp/9j6GJCgoPWp4MeC4Nt5f7W2O", + "DhVOX1Q1C9mNLyQ/EKzR/uXt/ZdGPP/yMvruzWR064UeZTMsQ9Q9872a8NLSvvuO7Cq7F81he5V+R5IG", + "z9bfXAEm7xpSRN9zlc9DEL9Petvb59t7Nzff3RSsuopS5yFYFnjV7YGm7wKwOQC/TNWoMb8D6cc2m8PZ", + "Rd6foBmW7DsFD2vWke3dZ22MEtBr28wIPyeCT8yQCi7lIO+KiH4D/ndJk8QIMJJOGU7QC7Rxdvzql+PX", + "rzdRD715c1LfilVfBPfnFrjVcAvAOprgqADqWQWtAxVpvefnr+FwJQQyo4wcfnl7NOu1psUW6NZucK9O", + "34HjH8uRi51uTiPGZSo+uaZSyWXAw1YpCJ+Dpm0+LQ1jbSZp2rBBlWshuX+uYEYHESw37wBL2+WPLC3n", + "A8BMP2T+7tcHcb0SlPpzkaWtneGOgKUbr7QQKHMNrORJ0+12e4joOxlOBespxLZ8CceBK9wak7nboYHE", + "8gMbx4eOT8sibKUzwjVfm9OLnf720+dQR3h70Iaxpzha0ffJwWH7zgc75pbZx+P9KN4Hhf22PitL2EYF", + "wckVXkAdTrO0w465MD3t1ju2VpFsFV+zDH19O6TruhjXgGUN4qwLXJKjdGUZoBYZwnU8wzS3aK0pTRIq", + "ScRZLKsy8gxLJDMDUmzK4RQS/JDBALuoqEoOUgrCUSTy0vRopWsr7+eZpXsoyZtxpnUAqMnxC1lIlFJw", + "ghbdQ+ijREUiWjxkG8IlLRbZiVCNN9Y/QApQ1yaXxF2IGobEb/3BkMlZrjQT2+yjQ85knhJhrbJoTMFj", + "tIlkblRaGC+sxkIzTEljIoZMvxaAQf6jUE/2nw4Gg0G3U2hyu/rfgxA13anzs29hvk06PgBxMgv4DdHu", + "ImcoZzERRWl+YsihHiJ3Q8fpZ+J7u8/biVf281KuCh/MdXDg7XC+PxdcGYbaoJ9DdOgtlPMntxfRW6X9", + "OfnVJvzZr0Y3iWCwKRla4xvr284Y5EhszZCS2FyPIvfmnSmLW526DT1WHP2eE7FA709OKmEPNkOj3cSB", + "SzTsA89utA07a2wka0dzE/eyB0V9H/DTdUnFkxC/ONi072F0OASGQiuGrYrivMq+prXKYA4UZWafNNGs", + "mGCtMrJBcykDI/1W5UJOLPSqFaVdToWFgi+QWUqnvlzIrXEut7KIbtnssS2AzXkOsDl7QfyCmMxHeR5S", + "jfQjh5P07t3xEdqAXwD22STIVLrH+On288HzF73n4+2nvb14sN3D27tPeztP8GCyGz3b3d7ZXZFU1CKh", + "9fY5qkGNORDHXEStj1z0fCiouSl3oSab2HjsK8piflW5/oIBsn7vNvh2XffLofWthxBMCUqwVMZ80cDJ", + "TuCSJ5Fu2wSk26Tpogpa2ND59Hyw/bnWHxhcwx1xLnJm3KoG5qNwIaTegP3Nqo7zdiwfBuQSX9atlt95", + "+0Ub7D95sf/kcxfNJW+sG2OdnO5xc5siwhw8ei07xCUJe3YkZ6DsWJnIWPVtMkmn2ynyXeBvEAZqsdTF", + "41ZJXE0HthtmI6uulQb8guOKvgKRKgYeM97XkorTR6DYSoGSoUWgw4TnMfJscQYtEPxwx57uopsBt5g1", + "0ZnEZZOMAZmRVNp8Pso0Iwb/o27Egh3so1fwLjzCqVHr7CBMaSHf9YbjhYmX0efLdW2UrNVDPrP6FXyj", + "lS2k/wXT1stgTbarmzDS2T76lcM3hbbHeN32a14HNWv59bqdeMMi6TvwGujMipr76KdCvCwEVCuQbkhi", + "/xxZhlViRm1WkDvsjnc0tZQ756FQdDtmRTvdjlsoQKtYxq14V1L90vnzSTEUSEZwAme5TNPPFU0sgj7M", + "hEpFI2mTR/TmNok9NjeTxCOjPDXFpJrUV6tgFR85qer9CdoAkNS/IGvY1v/aLOJXK3fdzou9F0+f7bx4", + "2gryrBzgetH4EJAJlge3Vk6OsnxkbSNNUz88fWdsH5GxKhSxL+9PfISZTHDNevTMXYN+5y/6L3ykt5jn", + "48RzLFq4SAM4DRsWBDkseFFDHOTvNJnTyYT9/jG63Pm7oOn29VO5M95uQLY2HYXNbsd+cMGSjZqMe6aC", + "WRiMCwhKyEa8urdEwgzQGVEI6KeHcATqTZFPbUnOodrZFQ8S1t7u7u7zZ092WtGVHZ13cEZghAtcynYE", + "3hGDN9HG27MztOURnGnTwbpkAZiD2jlDtvz4oCqQ9rcHuyEqabi4S6qxbc/TxiV/b9VHOym76JC8XaiW", + "S6c8uNq7u4Nne0+eP2l3jK15eCSuV3MYl7JklsfWyPB3fgOkyfODUwQJwRMcVW07LkLsRqNSNxoV1Hcx", + "dRluMLDnz54+2dvd2W4HvBgKOrFQo5UDW+VdgUMXIIrAbgSWYpn1dptui5A4ZQjsLYkSTNODyKVY1G4f", + "U39hJMxr5Sa0uRisBr50cbX4tpVxqzBZmQQdIxpwgXJWVP3pr3fJfhHPajPXNtfDeq4eSsthevUsQpip", + "bniLpcwEmVOeyy/QEFcmZ3aScC5u9G2TwvKWyDxRxmZDJXp/8h3wFE1rSCqSVXUoS40rcNRuObkbnecK", + "iYSJvGmxWu1Gm61fNeFuw6ntrgLUqHCDRvTCWHOunK0P/jzESZRDnStc7KeeFcDwASRAliULE9ufJJwz", + "FM0wAyeJ8EDH0IwncT8YCaufjCbBqAp+hRJu8NgvCclsCSgzCP2ZFmHonKANv/ihIaVaSeInqWEytshP", + "lRqfpOHaqmFMpSIVXq8nVtyDCjefVCyhCZ8a2GQFWQv9eoWKDAuTjICZKWk2T40uGQi4DgyxxsxDN6q5", + "SfnEKrhW5IBEc7OSOBJcenBg709q+csrct6KLOb1AZ3VwbYgXePQDFxlBpCudeXD0P0YyOf5nBsSaBhy", + "BleESjrjZIpZDkWhPEK2hvh+63DIGZdqVMBc3XCwUo2g0ksuSAlYWWTdF/Yg907wXnSs7TbLZeOOb/X1", + "ElWFm2oaYDNPDa5oeLW6BQ2GyHgZV24llF0J8FcHs7oJXmRZGoRKaJV6yIFoA3JePLbkoUButgmSCaus", + "up8lbdUW7n29Nzhri6y4GkjxFKvZMZvwAPbHDTynzhJto1UzIhy4YUwYJbHTJQsXqjV1QcJ4IgmKc2JX", + "zsinPvwdhDhgNQN7JHxI2bTG6+sdtjEPmzGsLgQD/doX24Q7yXBC7bnIYa1MvKJEuEytbRUESuUo7M5a", + "bliQaZ5ggSwmapshy0WaUHbZpnW5SMc8oRHSH9T94hOeJPxqpB/JH2Aum61mpz8YNVUNOzODs3mBZkNq", + "/ZZT+EHPcrOWlQyWmC3z/RY4RttEjwUjxX+iCbHYlO8YvfYIvVoRYW9n0JQt39BoJU9+GZz1ppzbkmzw", + "xOcykFu4Uspx9c5cTQor9mS5NKWXWtxKDgfZuQBv59GpJo58HjTJoeHXNWASNCaQ9+Omtsw1WrDFNlMJ", + "Vn3J5Qz9nY+rBtG2Yb+BWoIbrITIEGQSjO83VUZWGaTNG0tr4u3uTTAogK3qicJHN4R2WFd1sYyvauIn", + "b5cKEM6IXTLq5miKEbYoruPiPwr4AttrexyDeqnIQLwyoNRItYCiy1DpauHVP5VozIUAEHgt4XDmZgOw", + "K1rm0WvtcK/Q+YwskCAppmzIKCuMpACmRhAjcyK8LFkutJI1JXEf/c1T8QA2P83UwtZjAOP5dxLxK1aM", + "ccj8QerGc6nbOWDGsijyTFUquepmQevThAJZy+AEUwJKm1I1QxNB5Myfe6icrZbxrriIG+uELZB7BcpP", + "gY8VKX5JmM/KimaCqqFpaGS+Wo7iM7Wo4anVP1GlPDSql39e3V8uiQgLicWUildaha54R8VTTgwIDCCi", + "QNlP+5dh8QUKSgvMk7L5v7omy59Oi8arv9Ve83BNHNL3gTHbBk2wkUnjqQX7VD1pa0NVIA1uFZrNsi8B", + "bbgQalcjqSoJeLWKWt2T7TLx6skCbjRbkkTV3veeP3n2tGWxqM9y1hn0ri/tmpunK1xyDTt10sbv8/zJ", + "8xcvdveevNi5kYfF5ZU07E9Tbom/P2iDXCt9WJN//eOf709qXp8nEIM9uNGgTGZJeEgN2SXVAb0/+dc/", + "/ulGdesBhRjNMkh/g9++MUon8XfSBQpUXXjtnGQr9PuDipEAF2wGbRCA4qZzMjLr1isHU4MBaScF4wxH", + "VC0CjBxfmWj34pUa1nYbd1B1sCGR17RtUVE155L5uEw63XCdo/80vuEaLTxvXXNO5uMmP/Sbeq/GC116", + "LfwYhxYhBoYiwgbuYj5XWFYCuvXfEeRduAyz5Wwb88Zq1N16KgREsdiSi14oYKjWQE2etB/521/bTs9v", + "WTHr1Ff8w4pz2HwEb2T1DdzIAaNvtD61tsYf7AV4u69GY79K5MoynJWSkuWte/N+W2QPL5cqKW6wm/fn", + "JUze5MM6JjDQox2DXfKy7W6FJBqoycuFCRjQeEJ6XvECU2pL5sYjqM+8hZkPZHBGl3wyqWLdPmnGRgfY", + "H0j2cr1gpbRm0kXk2tks6sDaBuNn2Hkihx2tAgw72+mwU3NbBdMnU3w9sh1UsV0Gq8DKy/T32iClm8E4", + "4dGlKXQIdfX7aIBSgplEOYPDX/OqbQ9We4e6nczbmwIanJgQpyW2BWMakxmeU6inYn0q00ogJrmmSkLA", + "KLSzj2Ju0J4q1Z/tDPVrJrlxv5w0XDqYLWzDukH9HmcuorV8Fwx8E6g5zT4SwbsWrEBz7DdvTromgAFC", + "D83AKvGNbqJmBJpBFl3UyiuUv4fjh8cJGcG463D96fI6+jnp4FkVRBIlLX53SQ41IkARz5mq4/in7RS5", + "alrZ8pWUMwj2s+EfgMtmezcEgmISwYmUy2exSui3IO5a3oBd6VDiwG6IhOFQrKi/89Y6hOsDMMYGr3C7", + "aceP6zZewpFU3Fb0K071iFxHhMR1wM/wK21j5e2XwVj519hiBBU11e3bEO+8PLv+3SV4wVibVtuP6Wec", + "9QCdxG2pRRIx0IAWq6ZKaBXocQ/SYhSCVw290CbjmlyvXutfybUCfPQ4TwzoXZh0Lauyl9G6Fb91ZmPT", + "geaCrK2QeQeVI028+a1qR9pQ9YcoH2nfupOSkUu7c0aUe/fMklHjDlULvVRcWi7g371SjbExpNRF9oJH", + "2+lmjQT3ZmGriAXlbZmjyXBKRpkgE3q9gnjMC0YxrsKalAepyGAw+KIbKb5Ge8+g9JesjZ3R6Uwli2oA", + "zl4AS+mz6qoKoghzhsI2O1/upvtwOdrNbqffekg4PvOggZZKmliRdLQKN/uw9LZZ63yGF2DFaXQSPtvd", + "Gwx2dwa3As52w7rBch2Wn9gqpNV2mlLqvO+so78Speq3UCRZL5e2NuXqPKekVILgdB8SbzIcEZSQCYDk", + "FQmt6z2L9a5XD94KVDaLtqB/t1F235wPvloyp+jKYo67aXScc7GKQeQ/X+MQbWAz0RKkXiDnbrc3eHq+", + "vbv/5On+9vZdgF0Xi9SU7fHs4/bVs2QHT/aS54tnv2/Pnk130t2gHnZJTWWgNrT6i363McqmvCSrWEYV", + "loY27BwyIuo1y+u1/iVJKCM9WWRIrU9TXMELjP997fm/mZ3fzGCl7HBWnaQvQmBVLk6Fsh4Gf8tOZqXv", + "oj6b46PVs7hVBlJ9IGF6qw8FyKvdYKBCxXbnM5EZctbyGnrnvdj6IlqZFbfuKgp52OGkB3e5YcVD5F0D", + "ZvBmveoCX77kArbTKRdUzdLVt0XxWgEjDnHTH6WKq3hPfXQ8ZVxrT/7PRZicr0TpjzvdTvJxr3pm7O/t", + "kb8sAnFBgHarfamgRRhZQuYkWb0K8EqpeAgTya51dT3mH7Z72y8gDiH5uPfDoPeiGnHQNavlL9+2e7vy", + "66DNGvolAF3pqO0XN4q4duu5ioJ+oaECduW9bLGJLY2X5eHd1eESbisbXD5e2uMakk+jAPq5kp693Ea+", + "0BSTBC9C2PSeoVbWtEefyNCYTCmTbey2u4PCcPskHXb66MAChIMuq3jRj9+8phWfTmiakphqGdOo/s0Z", + "DDstbXF1XeJmtUncVwFprR8W116sh0hYl3C17prsf0Y+7mdpv+003lXoHWBXcyoqYIjBi11EJwizWoFS", + "V6zaKB2QGAnxavsOqK0kWcsDZCkHOjtJF025QmUKfUt7W86a7YLF+Mk12FtXYGYYgtj5IoAoBYAYXcW+", + "jo9QJnicR2X+aAKDLhE/RF6DaFsh5K8Pyb1L+wYkZk+4QOvtG00GjXb2yab9rtkmNcE2b/X2YP1W34lR", + "pNvJs3g9DzMvteNgN0JuX5OCGDDRVJe9Jgl6k/nQgqO/9VdwWec1tuRIi0R55hwsmqaWKSngbgEXQyiu", + "94gkRF9Ty40gnsRllgSVJRddz1K3nz6fNbk4wSO1PJBfCMm0rgL4R9BfitkiODBXdrS4SzYGDu1bGodX", + "z5QrsqtVHdyztZJY41b5JtymEgqGy9ds3gYv5dIzfxcY375otoyA4hh+RUh721wCwH7pwt4a7cd3YZZ7", + "SCHtjXU91GBbHahwgY7u+i9jgbVYVyXevZB7PkQW53gKtfLW+FEaSSVUVxKcHBgpbItbQYFN/U8EVUSl", + "A2xxAgJ87jJ6bbFBRBXCU1yvHNFUltJ4otfHEDUXejm3JoImTN56Oqxvfj/o/Y8xt6NRf3/rh7/8370P", + "/xk0u9cMCJKIXkwmEHF1SRY9U4VJ4Wn1Cv3DlIDQWsXUnhmCUzCmAdq75Ur+eJ8MCu65+BWnS1OAUDWv", + "hNL22gn95T+aA728ZXwHF8bas/vZFVLuopKs4u5e3kiJmLqgepdRt9kfMkjSuyQLibzCbFa2cyf2O1l8", + "4oXiowtD7n3C5hdoTKHSpRwyrd7jKCKZVqtsrR9qyrVzYMOC4MRvxxaIc+fFemZNYAVB70+W4IzfvDv/", + "8c27X49Gb05f/npwPPrl5X9DrMtVz/QQ9zTt7T15aou0+yu5HSwUcvN6F310YvMVbMzDJAfNHgDLJEpz", + "lUN0DLmOklzSufOUquT2lS2Ws5ZvXyniM6GQlUpC4RkWsjuhEwIBDnCv2ugiKh0xUgnV7a2VhzK0LLoY", + "whl24EpR3EkSwboieivCq11ubHXRn87a3SUGJDZw2CH1Vx+2gBX1NZUA3OGCYLyX0Qak0LgSvC6DePNm", + "oLUHRYPBEMwvXGlp8OJLVEN9t7L86ZwnPa3nNZSMCJrVzVoEUwigKZOa0WnyvkzHAWXG2rindIoDDpeQ", + "Y+WLVC11A1qbOra0/43l28IJHUf1ehrmWJqlqtV/qFlLpOo153ukWrxvQCAGiGWTxEu9IMNqxm7K1Jat", + "LhwCDok5oLqvStsuT5mDiezBR+uzkVcqmN7MvJE0782J06Nqmt6KBTrVS3M1I4J4GwEflHUKbrhkNkGp", + "BRyNqc6YEVEG77rsJi2eg99doo3CBOaWoEi7XvYLrK5DcYKvix7Ap4TlkiMW5lHWwdp+9SPUDHjran/S", + "iWsChlHTcsMI+VUqWrUmjqqWN8OnquV5m/eDB8/yqhXcr+ls1Yiz7KNCmiF6/Bum6icuQC9uBn+5c6B9", + "uPxjIgAMrw6j3wqDnqYkHvFcrT7/+jUtPZorv6gPW9YXdjYADEQcVfKam3iBgycpx7C80no5SJQLqhZn", + "er1sVDvkg7qivrCQ0BH8XHYMhVQ/fQLr+SSQOfOKMCJoBGVq9XlMMQONCb0/8aoVmsKVS8C1IAK9OTy2", + "dheHfQx6NFVAei4A9eD0uNPtzIkwtofOoL/bH8BhzgjDGe3sd3b72/1BB7SqGUxxa5zTJLaJ5FajLjT4", + "49hKQj+6l/SXAqdEwRe/BSARIADTvg4qCJ56SmSGqbBaZJYAVIMhGKq/hroL7kLdN7dy1yx7a+Mx5FtD", + "GhDJ3tjN/QCCMpwdmObOYGAR3pW9fiGJyWRObP3dhtGW/baS6uwSBcoQLKl5TrYslv5Tt7M32L7RmFYN", + "Bc5uqON3DNtsZgLa+ZMbLsStOj1mJj/RZpvbuDD/xAEh+Wfttw96z2Seplgs3IL5q5Vx2SQYE4mwe9fo", + "cUqiSLMKKJbUR28YMc8RVgibEG6RM6gx7T7UFFo9BaZtt8kFWtOPPF58sSWs9OFsFJ+q7Ewfl09L9Pzl", + "aKcg4+WNtI8c1Lih2nsgoB9xUSD9wU7K3uDF3Xd6yNkkoZFCvYKAbWA2lRD7lABwugNh4gL9nnOFUZHX", + "8IiOtJVZxwW5dcuraOsPGn8yxzshIX/AKREpZiZLxLyz5tAvHWfjmymP88pbzRH+8VHH3lQOjchcVCDI", + "VY+of23VhcHl62gvAEVh+zTTix+Q8Pfu4YTbyRY1ch/yyEFlUpRL8piOk/U1jkshJCjLvSLqa6H5wX1e", + "Wbaawp/wFD0WAn5FCgmv3K2lS2ErEzkzCnBQAnxbZm7a776rCn/n5RMvXAj8GrppqOuhjIMZx4s+cmtq", + "lH61AKwpQWCe8fK1cqqH97WcsJ37OGEw48JT9O2a+nZNrTrlhlrcFOBgeqe8hQ3iRhaIP5/94cbWh2+2", + "h/a2h1aWB0aurHXh73zcRzY0N+IxQXLG8yRGY4IM8JMLwlFY9KcfERbRjM4JoPtBtbo8UTTDAkJsUhRj", + "hY0PvdEwsdIsUTS3pZvruYDMcoHrgB6SjCAEZtQExFmGYlLGSIz0JzZqpsRVXKqrbs5+0MBeNFhejehq", + "xiUpgA2Z8m5zyPOWRjuGZvtDdm4Rb/UCQlS54zWSJIDbu8L+wxnCQ2Y/+N6xEBcRJ3Faci4sADyRGohO", + "sy3LOX56pCMZ8RDo0DlhmKmezEhEJzSy07okCxvYGmywVQEqPWA3zvcnReYK2tkMA9dB6FIYpfioeIYs", + "JVX9NwyiwaMkj0snl8NSwmKMkyRYoWSa8DFORmZ9LknAJ/gK3rCLUjpcSm8S4zExtfSzhZpxZv7OxzlT", + "ufl7LPiVJGLY2ewPGWSk2LUmcbcUENEVVLRLM67PmeCp6XPLDHHrj0uy+NQfsoM4pcxRBHyCE8kRuYbv", + "IG4MwEMM92qgB3Oawn7ww1wqnvoQsI7uzDB5rrJc2dQaSVQ3BH86ZIqjPxzI5aetP8oeP4GzmOBY04n3", + "ipkSyNZNo5YjrGc/glcD7nYCCzDs6IvUhHlMBWbK4JcWKJ1o6m/pRlEmAkrH1lc4wgxlPDMlNoCoZliT", + "XKUNAK3ASYIUHCX3rRbcYScb5mMxCNNxIwChQYyrHSPK0MmP3mEa7D0PnydJIkFCESX/dfbmVwS3st4D", + "81oZrmVyW5gWGFCcg+vU8bSXOJoh46iCqorDDo2HncKdG2/CWHNpw2V6PfAp/qCH9oPppkvjH/p93ZRx", + "V+6j3/4wrezrs5SlBhB12PnURd6DKVWzfFw8+xBe0CYct7MKI0Ab5prbBE6CKUDueDe+uSIxixG3t0Cy", + "QBiVHMgPXBlThsViVUZlYOntCvKJiWT0FuOPIUQuDjv7Qxe7OOx0hx3C5vCbDXAcdj6FV8B6LZtL+MF9", + "Vjg3CyJ6OhhsrocEt+sb8Fm2cAx8YR2wUSsq6o/qHbR4tH8u/8C/tf5ZuH4w052XGE1G8XfG90fogPAk", + "dl8TDbggamI3ZhFJnNi93tBz/84DvVkRSZL7JtCHIs/CPVaULHhU5AibVR6jleb7B6a4wX1dKhWz/cPQ", + "76Oznwes59Z2TuYu1DlcsAXAeKwqjczLCEt0BmPqnWnl+yX82rf/dbofgEteJHx6sW9Ud5TwKUoos/kA", + "XqCyFg/sWsJHBo+n+M7C87hqeRtGkvjXP/4Jg6Js+q9//NOC3P/rH/+E475lcOag2PbFjGChxgSri330", + "CyFZDyd0TtxkoBwumROxQLsDa/OHR8ir+W+lNDlkQ/aWqFwwL2/CFK6TtkHrKtDzoSwn0uIZQZrQxFbV", + "MbGNAbuNO8tmKe/1RHcDuJAwA28C+lZ0NACgetRUHLeaaCdsMjVzrhhN62GaS8F66/mLItfKUG/PDPCG", + "DAaWOHTu4IGdNNo4O3u52UegbRmqgMpJoDuUzVg1ov+NJ63nSYajVBkKrLLhTRHO8Jgm1JkcG8q+mCOY", + "4mhGGSnjiwvQddfEvhup5jEHp8fIBkJ24dUhe3O2BSZWRSKVC9K1nEBYqNWyLhy3eS7QA/AvqiA6rGff", + "HbIJwZAndHxkmICHRl4kRhYNM0A0gRhXqiol6LpDZiB1LYSzPngpj0kCH0H/U6zIFV50UVH015WJSbDS", + "CrHs6peHzKQa2jXoAWYL8obZB35mhtRzkbw2Z0uQSaJVY4jAN/XPoe+NCRfIRjh3y7xS153JNjXD0ouW", + "4ujNmZ7fFDRBbuyB0NKbM7cbm10kOYoSCtQQYTZkUwgEcijGnFV2tUgom2ER9yKuLwEf1eqS8auExNMm", + "HnvoE9kdSjKVfgLH6ec6uT424WK2PAF9iA1S32rP3ZF9p53rzrb4Z/Ld2YqYN3DeGQsuMfzGrO43R14L", + "R1543ZxTL+RZO3JQlHcX8Wu6eKCAX0d7y2tunnhL9hAWPbThMH7AK8IFOj08RjiOBZFy89/b3qdnaqi0", + "lP/0/ahZ8UOEntixcGHRD629pUogj4UdvLWjRtjNq15o2L/ftipViBpvuqIgUXnl3f3tUev0JtdIKfSW", + "tPbtJlkbbEtlxKHeYkktPRCNElKIL8U59alonVXZhPEWV85Kccmy5+MjdyDvz75su85Z/W64B6Z4VGOI", + "D8gIq6nWfvnwx0TN74pddLDbK8zPXxdpDu5PCrpvU3SIzB+TuhjXlk1zQQN00niBviLKwJvcpZ5uewhM", + "/IwId6rNQBdm1sW0zKfI4LTAhMASs1r3PTavtFN9TXt/Js0XlucmEotd8m8iSgtlt1yrVQrusa2FfXf6", + "bQXJ7Z7DViyBBRYZrKhj53YCy+oGlgsWbX6LXPniFG3iGkslVrh5k7iwZBs0pULPui+57oD5hde1TGf1", + "WsrQJKHTmXUCxHQCsXrKL2QOo9y5h1EWBcMFVsSGKD7GvN9TvcjWCzwnwkNq9K/UrT8gaHW9quSY18rb", + "9d3b1z3CIh4XzpNmmdQ++cIKk6H/Si7v/Z+6R5jPSp140CQwfsb+m2ByVEB3/q+dnyx45//a+QknGWXk", + "f+0eJFgRqTbvjFgG93XT3bcC84iJT+svtLpoS6xpS+Gpn4dapUwHX3sTyqzAzjpjd5FI88Wo78sLeXWs", + "3lYS3r3Rvdak7hux5NiVJTBbyoVDr6pBEv97C31nFYK+d3tdpXsqYQQg2D3CKGUuHOoqtrDV1BG35U1s", + "Crjea4wRxVst7RHu/T+VScJM+kZGiWJdv9kl2tgl/OVaaZqwW3GnxgnTxwN53wtiC602PPoGt3UPDh1L", + "kR7cVsXDXQJuzbhU8OjxYS/Yq4QWFOdfGy09k+WBXHl9ONI9PurCQkLZfyg7ZFOb78lP6cZx74q37ff+", + "hZ6DdEynOc+lnzWdYhXNiLSIAgmpMuDHZhIor+dGo8BXTKWD+7w67l3n/0b3d2SNqG+oYd4m2GCdzO/e", + "aivz2/e1zG/Qli3qgq2N1HV18zYbkkAc3nJbMq7AUi8np4TGFdJF0DutqJTqAgINYn/I/rfWP35TBKcf", + "fnDp3flgsPMUfids/uEHl+HNThypEKYEtWVOD349ggiPKQTxQyXUEkyiPg6U5lIZ0nMlT/7tFKQyyKW9", + "huSo8JuG1EpD8pZrtYZk9+JuVaRq2aR715EcvYUW3NY7+HNqSX9y121Fg5P5ZEIjShgUnwLjn1yKVTaa", + "3Dev7S3BEpiNlfACHSuSSGs1suBaayT0svD/l4wk7DbWoOAIK0XSTKGpwBGZ5Imp2oLkLFcxv2KuJAVM", + "0FU3o+V8Qte7a2rkGgknyIZLtLfVdIuyjPet6tqOH2mGKs9shXGrXJaiTbN2+bDEe7c6ZYur9v61ysdM", + "YkZ9W166TGsIgRJrxs2T5iadt/iyRGfso/Pz1y51V6snwhXsU9xV6XOVnIfMr9LXRy/L8ofmBdeCVh9I", + "bFP9IaHZ1r2LCY4TygjkOhAZyrKt1tZ80GPx5SXgcOHQ+3ZxtzmWtib2w0nAD8YK7kXWLHz6yjpki6Pp", + "1RQtTouTN+HUPCp+ZRlQgPGEZL0tnCves2AAWzNuECLDILmnCY4AI1e/ZuAbLf6KwWv1mwJQFcGThAgD", + "y5nlyolbQ1YMjjLFi8LTVjK70M2PcqZoctE1oYaArSQRZguLTTdklc6szAcYCYD/ASMUJDMjrlXR1YOm", + "PJfwFsAZ+F0inFzhhRwyi6pgPofS64JEBsE2SfroZw6ANqaatMd4TSnX7+SQXdA4ISOLR3OBqERyxoUi", + "jMQo5XMiq/0SLBJKBEziEOuVkyjFCwCGNBi5Zn14Rgz4YgX1hut/YxZTKAqqey6mvD9kGO0MBiglmEmL", + "YSHxBC4c2waCQVQG9D3CaG/wwn5V2zcAL3fLv6FPkxBkziM8ThaIaCo2xbY3YQNTW6TXVH3X2zehQpr9", + "KuybtvpiZWOpdDVn4y7KWYnSAbb+nBWgGnq7VC4YzNN6AQkVxTVogYnGJMJ6PRmv9gOQsDyKchG6IPVW", + "e9Wi/x0FR296Z7BUYQyMBEwGEYlhzxlXMzjTHI7S5vcNVFUS1Z/jogkeEi4QRh5dlxYNEuXAGjcAQvWi", + "LH3KXCnzi83v3dnRx9cyAnf8DYjpY7mfgIj4ZFI5gOuvJnOAV+WeLZPwn/WcHrqa1z6LiymeMi4VjRwz", + "dAjVvtL8TSFspRCuXtkgNU+4uGwOOP6Ji8u2GpgLjHxcipg/w6/QEaGHByD4D++PAGu4UVY00dy7klan", + "r+KUgtBFlSxCglHC2VSfotIqf+9uA1+r2zCAlvoyFcbZXcCPaSVkZH80ZbO9sG5wMUS21YfmRbr3e3BG", + "/coVommWkJRAWe2eITa92SVU3XhhofQKwLab8Up9qnxcBaMLShN/0HXiENCV27ANkN6XtyvIVBM+XQ+I", + "WnTu0D8DiKhD9k6aUgUXxvV0gQoerAVaU34EXc1oNAN0VNBbdfsGPBVn2UUBDL+5j17BQfbx8aHzDVN0", + "RNOa5AkxoKfzNL3YXy4c/f7kBD4ywKimRPTFPnLFoov7Q+q3fLRTPYsES4V+tRiuG4UyDjt6obDWN4v5", + "bVoc1BK4f8hCmKiMXNkG6QRdePCoFw3YfY7fvuZT+dW4ispyK2YuiiOrOgJtEhZ3moI8aBJ2/GwPBqEq", + "AC1RWs0w7hikdWkwr/m0KPVSIWWcZW3J1w4TqHiepitoGG14kI9SxTxXf5EqJkLAx5a6m4gbbeDIlvnD", + "l5pQLcCnO9ibQH7BUCZTeyG4VJqpdrodwvK0s/+b/dc8TTvdjh2P/u5KZalXu+EGQv4a1Nt6g8uhN3qH", + "PGjbb+L5TUBrq0zfQ62t3SBWrW6WzN+aF/70XkNnu3tAMgQ5oWbM/ZpEUW+8VcMP4wX6LozsxX2MDCB6", + "UZRwSSqOnscD8GcNXjXZsdlg5Na4p4cX564iWptIljP76Zn78ivQwdfFjLgxIzfdew8eWR7BYwYrkEuz", + "mXBRR4VbF1Xy1RPSl9uSpam2oZBvtHlza2MrwtT6wjKLsB/EpkImzhVPsaIRVGeLZpxLj+wLCHdTR9Ea", + "kQvKBBOL0XZtJsGFJtULa46+sOrEvjWdIew/sn304XObfxD+wj0qv/jJsw4UHL/rVACoYCIRRmNByQRl", + "OJdES3V5SlC0iDRXNOX4CI5mKMKZygWBSqMEpZTRNE99bH69Y3MMOEIX2+lFF41zhRIspqCdmYcu6Cbi", + "aUpYTMBON2QzgudUq5YCJVgRFi16kkCF8jlBV1xcJhzHYGrIYgweH6hwKoimQCh0kBKFY6wwCDoX+sSP", + "TDLTRVG03Kj3jFyX1BAPmcjZ96bqim72wg30AhEoK0DlrChuG+GYsCgIt3/2dbOxL2+TPiOqPtEHihC6", + "FS99yJAh3/bqhvN1RBM9WmiIFmx+hdArm1XYahaII6N/zyNt5urm+ECOpmKJV53ir8PDVBDdV+Nleng3", + "Ehcozk133qkEMv+z+oYKhuIHXUGGqdnG2zqIiiqexTLfiOdt/eH+PL6FLe8r4YTdRsW+qV5cOemvgeXa", + "Vb0Vz30gI6a1Jfk2uYdjwS6y68HEJy48LvdYjK0VhLaCb/vcSQkM2hdn39h2nW3bwIfbsm1nm11y7XuM", + "nLIexIqGObg14zayams6+DfNSqnNzmOZD84iS8/FvcMtOtaY4UXCcfxnCBZe4T+KuBAGBgOANR4TRLRn", + "NfTTBMA2Vxai7LqszfcnJ5tNXEKolTxCqEfMIbzUHP1ZGi8bcN/MiRA0djCYhydHNmyXSiRy1kdvUqqQ", + "4uiSkKzMbIHswr6enwMEqQ27jvzR7RCmxCLjlKm1oyhfvZvBlD9wi4vyFYqStubAN3d4a3c4WPYfHzsD", + "LgO5G2YCqzVThdXaWsiUTbhIjVyGxzzXrWsepJdJ76dBLJjQhMiFVCQ10YmTPIHjBvVrbI1y+53Z5S7E", + "5uqTY9LmMiJSKiXlTA6ZzRnJiNB96891+16gVdAhoHDBX08Nk/w6gvj0YEzcGlZNqwbQTVD7uLPf2cJZ", + "thVjhRsCxezwPmNIP0FUHpKLdMwTGqGEskuJNhJ6adQTNJco0X9srgzrG8F3X7oC++1Pll7pYzbhwfqW", + "hmYLYv5TZXdZtuYck4+Orb0i/mFx/Ac2OszW1td4FwQnPaiZ7gB8UK5oQj8aVqcboVLRyKQe4WLt3p8U", + "TLU/ZCdECf0OhhS3JDHIBqBdbmWCR1vDfDDYjTIKKHC7BAYHDK/5cQo9Hp6+M+moJOVi0R0y/Q9o+Pzg", + "1Hh3J9haE7yB2uLu6HjrzZpA5zNYpn/jCEEzwZUoBsEN/+YSvDnWSOMZkg1HlGerVCWe/elDWK0E982u", + "8DjtCgD2VMxmowD4cqhcYRvCnCd5qv9h/jheh2+mcDR7D69+NdKuGc7abtwEH8WhtHOKiam/+yBOD7Ng", + "jzVmVS+cmwIIMZVowOAtcKD+jNT95c33/jp+he5Ou6KutvVXc7bu++azY3BIG/56PJZjbijNzUTx1dan", + "K0ybrU8/Jjy6lBaSxTcbar0NcNb1jyUutnURgpgAGaLIQhkZwCwiu0NWM0Aa5B+JMFJEpJThZAvmbBoB", + "hG9nxcJzTiFRO4I8lZ6kMWAnJQDjDTB4ejZgqHINeB5daav/+e/4zkjF0ZhEPCUO9XwzpLr9DVP1ExdV", + "CPOvhS+ee+sP0ICYgr19DWp7c4+fheJ+gq8hVDrOrUPZjWjjFS9/NKagLoK9GXZ2B3LY6aJhZycddvQO", + "HGIwoWKFnqCUslwR2UdHxr4FqbhPB0iSiLNYOvB1Z8HbHcimxFxDlg1Znk/hu/sUeyxVwVK+tZ2E2IN+", + "D+nvIWkHbfgHzp7JuAuHLkY8V8bcb8+VfSsmCswjm/fuq/XOyDfdvg0n/5s9vhUeBbus2aW39YazZ7mc", + "kWaT22tT0ChXYwD1dgWQ5Qz9nY9lFzFyZazhQqr+Et/TX5+aDu6j4IDu6ibFBuzcv1UaaFFpoFyrMGij", + "CbDUV7KjDoPcSK4zLhSgOdqce0NDoEkAggSUKXxzeDxkkWZFBmJQkJQDd7K46OYWPvjbGXp5+LaLjqAY", + "L/o5H2/20RuWLGwNe+ujGTIjiRnmFWGGxoZqSRy6ns3YgXruMlhcd/BA1e3NyQh4VtxeuSDxbmdGcAwS", + "yR+d19x0FkAffvtaHyAAADZfFtveWSl8dN4SJRa9g4kiYrnZE5snxQrsDHtJOyg6K7gZAEzdoXQIbGWf", + "RjYwEBm7O50AYsanb8Uf7r6I8/14yUyciCm7N87Vo63fCgexYI4hFuhf10X5hKYsYcvLVioY0GVT5PdX", + "ZHJfybsqGPP/rqcLZvpoHU1ZZZ80ERdlV9Z6el1y8MzAIltHVYQzHFG16CKcJPaOsjdBEZHSK8TfsSD4", + "MuZXrD9kb4uCLzahFx2evus6Ry2Kqbw0LVhfbB+9mRMh83ExOAQHzXiNYc1JPGSKowgnUZ5ocYNMJiSC", + "XFyo4yIbfLnFUDp3eHbKToJFZ7yo9vzR1boL0wTsXkkWdYrbMlu9JUiUYJo2g5BbQQ0CDiHUYKwb5QxR", + "NklsSFUkuJTINtUjCZ3ScWIDhGQfnc8IkjglQ5YlmDEiUC5NVLweei8TRMrcJHjrBgCs11BUF5UAg5ng", + "yoYmJJwLaaIJNIW/P0FSkWwFmb01LZ/AnO9ItjWN254eyEhdG0OzKcS+gvSGGEoxC67pKE9cAOO9hqKb", + "AT20lPhYDv65oNMpEfpUYMNkTTieOdZuOc2hr2QsN9a9PCvealf3smjVy0r0MvZWAsSNSsztuHOzqL9A", + "55e0EUPQPrpZFvEv+qOWfVezVcODsI8+c5ahEp7/jtUyz7wkwbYGrJLCH5s5yRt55ahWEm3Xw2q1zqy9", + "y0zX1vhZDwab9ZjRsnAlfbZJ4f36CGFwvygP911s7XHTVgXtqqKbNqT8r0fV/yoo8G7g9B8Y5eQWcPpf", + "Vd494J0/HP5J8KA+VB59xffsiu7+6RHx7yp93sDiAxxbU/q84Xo2eHWlovTevtNOTbIt/pkkeBvveAP5", + "3S37N62/hcrgLdY6F7QmeJJmauEC2qyvsgw6k/Qj6Tc4gou41btzBd8ipPPLkYej08aAzj9njfwHiRm1", + "JQSpRMdHgeLzjwxj0D9zlYtlS986PSyiGZ2TZqN79QTbJcoE6WU8A+dKbBbMroe7yxQW/elHZJu3mKv2", + "X1CDEqD6SYxiKkikkoWpB6o5gunjO4kE15oAPOdi0RwlYo7IT4KnB3Y2a+5De6asMayMM0wXvRgr3Js7", + "brPChPYZ0Z0unlIzPEQZevUj2iDXSphKF2iiNR9EJ8WSkuuIkFgCTW76A94eNFg26Ucymo7bjHJFzZI3", + "tiYMinKpeOr2/vgIbUANtClhei+0qD8BSTYTfE5jElfG2JnzxKzqdsOC3tTuqoWKooCdUy7M4B5Ehmlz", + "IU0/0qzKFoqQmDFlGAa3tipI9UyZJH7dH6bMBeDYPXKj+HaFWc1vwyk7mhKhHqddRMW5gXje/HbNPeZr", + "zk+Gcnda5bZz4Tmrjdft8qNapi3dReGHInfufs3W77+elB4qH2U2jzWdzwuFtMls/nWR4OD+7of7Npe/", + "f8QpoK+IU749Uzk0oFsMEcxriOmOyZwkPEuhLjq82+l2cpF09jszpbL9rS2I/Z5xqfb3Xjzb7Xz68On/", + "DwAA//+uIZKx6fkBAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/resources/disk.go b/lib/resources/disk.go index c8847ce7b..50e106828 100644 --- a/lib/resources/disk.go +++ b/lib/resources/disk.go @@ -2,6 +2,7 @@ package resources import ( "context" + "fmt" "strings" "github.com/c2h5oh/datasize" @@ -75,13 +76,15 @@ func (d *DiskResource) GetBreakdown(ctx context.Context) (*DiskBreakdown, error) // Get image sizes if d.imageLister != nil { imageBytes, err := d.imageLister.TotalImageBytes(ctx) - if err == nil { - breakdown.Images = imageBytes + if err != nil { + return nil, fmt.Errorf("get image disk usage: %w", err) } + breakdown.Images = imageBytes ociCacheBytes, err := d.imageLister.TotalOCICacheBytes(ctx) - if err == nil { - breakdown.OCICache = ociCacheBytes + if err != nil { + return nil, fmt.Errorf("get OCI cache disk usage: %w", err) } + breakdown.OCICache = ociCacheBytes } // Get volume sizes diff --git a/openapi.yaml b/openapi.yaml index 3621c3e8c..d221016c9 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -2283,7 +2283,7 @@ components: oci_cache_bytes: type: integer format: int64 - description: Disk used by OCI layer cache (shared blobs) + description: Disk used by OCI layer cache (shared blobs and materialized layers) example: 53687091200 volumes_bytes: type: integer From c93732068aed1774068007ad0df2426e25d15980 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Wed, 9 Sep 2026 03:34:18 +0000 Subject: [PATCH 39/41] Harden disk usage error handling --- lib/images/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/images/README.md b/lib/images/README.md index e52868f97..e1d346a7a 100644 --- a/lib/images/README.md +++ b/lib/images/README.md @@ -68,7 +68,7 @@ Content-addressable storage with tag symlinks (similar to Docker/Unikraft): layers/ # Shared materialized layer artifacts abc123def456.../ layer.erofs - layer.erofs.json + artifact.erofs.json system/ oci-cache/ # Shared OCI layout for all images index.json # Manifest index with digest-based tags From 861e3fc70c6fe6a1f9b2d02200b9db7ba6b8cde9 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Wed, 9 Sep 2026 12:34:04 +0000 Subject: [PATCH 40/41] Allow same-process image manager reuse --- lib/images/image_store_lock_test.go | 32 ++++++++---- lib/images/image_store_lock_unix.go | 77 ++++++++++++++++++++++++++--- lib/images/manager.go | 2 +- 3 files changed, 94 insertions(+), 17 deletions(-) diff --git a/lib/images/image_store_lock_test.go b/lib/images/image_store_lock_test.go index 76c7919a1..a2c130edc 100644 --- a/lib/images/image_store_lock_test.go +++ b/lib/images/image_store_lock_test.go @@ -3,6 +3,8 @@ package images import ( + "os" + "os/exec" "testing" "github.com/stretchr/testify/require" @@ -12,18 +14,30 @@ func TestImageStoreLockIsExclusive(t *testing.T) { imagesDir := t.TempDir() first, err := acquireImageStoreLock(imagesDir) require.NoError(t, err) - t.Cleanup(func() { - if first != nil { - require.NoError(t, first.Close()) - } - }) - _, err = acquireImageStoreLock(imagesDir) - require.Error(t, err) + cmd := exec.Command(os.Args[0], "-test.run=^TestImageStoreLockChild$") + cmd.Env = append(os.Environ(), "HYPEMAN_TEST_LOCK_DIR="+imagesDir) + require.NoError(t, cmd.Run()) - require.NoError(t, first.Close()) - first = nil second, err := acquireImageStoreLock(imagesDir) require.NoError(t, err) require.NoError(t, second.Close()) + require.NoError(t, first.Close()) + + third, err := acquireImageStoreLock(imagesDir) + require.NoError(t, err) + require.NoError(t, third.Close()) +} + +func TestImageStoreLockChild(t *testing.T) { + imagesDir := os.Getenv("HYPEMAN_TEST_LOCK_DIR") + if imagesDir == "" { + return + } + + lock, err := acquireImageStoreLock(imagesDir) + if err == nil { + _ = lock.Close() + t.Fatal("acquired image store lock held by another process") + } } diff --git a/lib/images/image_store_lock_unix.go b/lib/images/image_store_lock_unix.go index 389e4144a..171996377 100644 --- a/lib/images/image_store_lock_unix.go +++ b/lib/images/image_store_lock_unix.go @@ -6,21 +6,84 @@ import ( "fmt" "os" "path/filepath" + "sync" "golang.org/x/sys/unix" ) -func acquireImageStoreLock(imagesDir string) (*os.File, error) { +type imageStoreLock struct { + file *os.File + path string + once sync.Once + err error +} + +type imageStoreLockState struct { + file *os.File + refs int +} + +var imageStoreLocks = struct { + sync.Mutex + byPath map[string]*imageStoreLockState +}{byPath: make(map[string]*imageStoreLockState)} + +func acquireImageStoreLock(imagesDir string) (*imageStoreLock, error) { if err := os.MkdirAll(imagesDir, 0755); err != nil { return nil, fmt.Errorf("create image store: %w", err) } - file, err := os.OpenFile(filepath.Join(imagesDir, ".lock"), os.O_CREATE|os.O_RDWR, 0600) + lockPath, err := filepath.Abs(filepath.Join(imagesDir, ".lock")) if err != nil { - return nil, fmt.Errorf("open image store lock: %w", err) + return nil, fmt.Errorf("resolve image store lock: %w", err) } - if err := unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB); err != nil { - _ = file.Close() - return nil, fmt.Errorf("image store is already in use: %w", err) + + imageStoreLocks.Lock() + defer imageStoreLocks.Unlock() + + state := imageStoreLocks.byPath[lockPath] + if state == nil { + file, err := os.OpenFile(lockPath, os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + return nil, fmt.Errorf("open image store lock: %w", err) + } + if err := unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB); err != nil { + _ = file.Close() + return nil, fmt.Errorf("image store is already in use: %w", err) + } + state = &imageStoreLockState{file: file} + imageStoreLocks.byPath[lockPath] = state } - return file, nil + + fd, err := unix.FcntlInt(state.file.Fd(), unix.F_DUPFD_CLOEXEC, 0) + if err != nil { + if state.refs == 0 { + delete(imageStoreLocks.byPath, lockPath) + _ = state.file.Close() + } + return nil, fmt.Errorf("duplicate image store lock: %w", err) + } + state.refs++ + return &imageStoreLock{file: os.NewFile(uintptr(fd), lockPath), path: lockPath}, nil +} + +func (l *imageStoreLock) Close() error { + l.once.Do(func() { + if err := l.file.Close(); err != nil { + l.err = err + } + + imageStoreLocks.Lock() + state := imageStoreLocks.byPath[l.path] + if state != nil { + state.refs-- + if state.refs == 0 { + delete(imageStoreLocks.byPath, l.path) + if err := state.file.Close(); l.err == nil { + l.err = err + } + } + } + imageStoreLocks.Unlock() + }) + return l.err } diff --git a/lib/images/manager.go b/lib/images/manager.go index 2c2b8a5ff..f7b53ba62 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -76,7 +76,7 @@ type manager struct { ociClient *ociClient queue *queue.Queue createMu sync.Mutex - storeLock *os.File + storeLock *imageStoreLock layerFlights singleflight.Group layerBuilds chan struct{} layerArtifactsSupported bool From 49efdfd3139825b4f1f2b0e99dc7a79dcd33add7 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Wed, 9 Sep 2026 20:36:23 +0000 Subject: [PATCH 41/41] Harden shared layer storage --- lib/images/disk.go | 42 +++++++- lib/images/disk_usage.go | 165 +++++++++++++----------------- lib/images/layer_artifact.go | 85 ++++++++------- lib/images/layer_artifact_test.go | 8 +- lib/images/layer_store.go | 35 +++++++ lib/images/manager.go | 25 ++--- lib/images/oci.go | 2 + lib/images/tag.go | 2 +- lib/images/tag_test.go | 7 +- lib/resources/disk.go | 16 +-- lib/resources/resource_test.go | 24 ++++- 11 files changed, 246 insertions(+), 165 deletions(-) create mode 100644 lib/images/layer_store.go diff --git a/lib/images/disk.go b/lib/images/disk.go index 3dc2bd107..68184b785 100644 --- a/lib/images/disk.go +++ b/lib/images/disk.go @@ -3,6 +3,7 @@ package images import ( "context" "fmt" + "io" "os" "os/exec" "path/filepath" @@ -62,7 +63,13 @@ func convertToCpio(ctx context.Context, rootfsDir, outputPath string) (int64, er if err != nil { return 0, fmt.Errorf("create output file: %w", err) } - defer outFile.Close() + keepOutput := false + defer func() { + _ = outFile.Close() + if !keepOutput { + _ = os.Remove(outputPath) + } + }() // Create newc format cpio writer (kernel-compatible format) cpioWriter := cpio.Newc.Writer(outFile) @@ -98,6 +105,13 @@ func convertToCpio(ctx context.Context, rootfsDir, outputPath string) (int64, er // Set the name to be relative to root rec.Name = relPath + if rec.ReaderAt != nil { + rec.ReaderAt = &contextReaderAt{ + ctx: ctx, + reader: rec.ReaderAt, + closer: readerCloser(rec.ReaderAt), + } + } // Write the record to the archive if err := cpioWriter.WriteRecord(rec); err != nil { @@ -122,9 +136,35 @@ func convertToCpio(ctx context.Context, rootfsDir, outputPath string) (int64, er return 0, fmt.Errorf("stat output: %w", err) } + keepOutput = true return stat.Size(), nil } +type contextReaderAt struct { + ctx context.Context + reader io.ReaderAt + closer io.Closer +} + +func (r *contextReaderAt) ReadAt(p []byte, off int64) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.reader.ReadAt(p, off) +} + +func (r *contextReaderAt) Close() error { + if r.closer == nil { + return nil + } + return r.closer.Close() +} + +func readerCloser(reader io.ReaderAt) io.Closer { + closer, _ := reader.(io.Closer) + return closer +} + // sectorSize is the block size for disk images (required by Virtualization.framework) const sectorSize = 4096 diff --git a/lib/images/disk_usage.go b/lib/images/disk_usage.go index ce9abff66..ca72c1994 100644 --- a/lib/images/disk_usage.go +++ b/lib/images/disk_usage.go @@ -26,6 +26,7 @@ func walkWithContext(ctx context.Context, root string, walkFn filepath.WalkFunc) func totalReadyImageBytesFromMetadataWithContext(ctx context.Context, imagesDir string) (int64, error) { var total int64 seenRootfs := make(map[rootfsIdentity]struct{}) + layersDir := filepath.Join(imagesDir, "layers") err := walkWithContext(ctx, imagesDir, func(path string, info os.FileInfo, err error) error { if err != nil { @@ -34,13 +35,19 @@ func totalReadyImageBytesFromMetadataWithContext(ctx context.Context, imagesDir } return err } - if info.IsDir() || info.Name() != "metadata.json" { + if info.IsDir() { + if filepath.Clean(path) == filepath.Clean(layersDir) { + return filepath.SkipDir + } + return nil + } + if info.Name() != "metadata.json" { return nil } data, err := os.ReadFile(path) if err != nil { - rootfsBytes, fallbackErr := totalUniqueRootfsBytesInDigestDirWithContext(ctx, filepath.Dir(path), seenRootfs) + rootfsBytes, fallbackErr := totalRootfsBytesInDigestDirWithContext(ctx, filepath.Dir(path), seenRootfs) if fallbackErr == nil { total += rootfsBytes return nil @@ -53,7 +60,7 @@ func totalReadyImageBytesFromMetadataWithContext(ctx context.Context, imagesDir var meta imageMetadata if err := json.Unmarshal(data, &meta); err != nil { - rootfsBytes, fallbackErr := totalUniqueRootfsBytesInDigestDirWithContext(ctx, filepath.Dir(path), seenRootfs) + rootfsBytes, fallbackErr := totalRootfsBytesInDigestDirWithContext(ctx, filepath.Dir(path), seenRootfs) if fallbackErr == nil { total += rootfsBytes return nil @@ -83,7 +90,7 @@ func totalReadyImageBytesFromMetadataWithContext(ctx context.Context, imagesDir total += meta.SizeBytes return nil } - rootfsBytes, err := totalRootfsBytesInDigestDirWithContext(ctx, filepath.Dir(path)) + rootfsBytes, err := totalRootfsBytesInDigestDirWithContext(ctx, filepath.Dir(path), nil) if err != nil { return fmt.Errorf("stat ready image rootfs for %s: %w", path, err) } @@ -128,108 +135,108 @@ func totalOCICacheBlobBytesFromFilesystemWithContext(ctx context.Context, blobDi return totalFileBytesWithContext(ctx, blobDir, "OCI cache blobs") } -func (m *manager) getDiskUsageTotals(ctx context.Context) (int64, int64, error) { +func (s *layerStore) getDiskUsageTotals(ctx context.Context) (int64, int64, error) { for attempt := 0; ; attempt++ { if err := ctx.Err(); err != nil { return 0, 0, err } - m.diskUsageMu.RLock() - generation := m.diskUsageGeneration - if m.diskUsageLoaded { - readyImageBytes := m.readyImageBytes - ociCacheBytes := m.ociCacheBytes - m.diskUsageMu.RUnlock() + s.diskUsageMu.RLock() + generation := s.diskUsageGeneration + if s.diskUsageLoaded { + readyImageBytes := s.readyImageBytes + ociCacheBytes := s.ociCacheBytes + s.diskUsageMu.RUnlock() return readyImageBytes, ociCacheBytes, nil } - m.diskUsageMu.RUnlock() + s.diskUsageMu.RUnlock() - readyImageBytes, ociCacheBytes, err := m.computeDiskUsageTotals(ctx) + readyImageBytes, ociCacheBytes, err := s.computeDiskUsageTotals(ctx) if err != nil { return 0, 0, err } - m.diskUsageMu.Lock() - if m.diskUsageGeneration != generation { - m.diskUsageMu.Unlock() + s.diskUsageMu.Lock() + if s.diskUsageGeneration != generation { + s.diskUsageMu.Unlock() if attempt >= 2 { return 0, 0, fmt.Errorf("disk usage changed during scan") } continue } - if m.activeLayerBuilds == 0 { - if !m.diskUsageLoaded { - m.readyImageBytes = readyImageBytes - m.ociCacheBytes = ociCacheBytes - m.diskUsageLoaded = true + if s.activeBuilds == 0 { + if !s.diskUsageLoaded { + s.readyImageBytes = readyImageBytes + s.ociCacheBytes = ociCacheBytes + s.diskUsageLoaded = true } - readyImageBytes = m.readyImageBytes - ociCacheBytes = m.ociCacheBytes + readyImageBytes = s.readyImageBytes + ociCacheBytes = s.ociCacheBytes } - m.diskUsageMu.Unlock() + s.diskUsageMu.Unlock() return readyImageBytes, ociCacheBytes, nil } } -func (m *manager) invalidateDiskUsageTotals() { - m.diskUsageMu.Lock() - m.diskUsageGeneration++ - m.diskUsageLoaded = false - m.diskUsageMu.Unlock() +func (s *layerStore) invalidateDiskUsageTotals() { + s.diskUsageMu.Lock() + s.diskUsageGeneration++ + s.diskUsageLoaded = false + s.diskUsageMu.Unlock() } -func (m *manager) beginLayerBuild() { - m.diskUsageMu.Lock() - m.activeLayerBuilds++ - m.diskUsageGeneration++ - m.diskUsageLoaded = false - m.diskUsageMu.Unlock() +func (s *layerStore) beginLayerBuild() { + s.diskUsageMu.Lock() + s.activeBuilds++ + s.diskUsageGeneration++ + s.diskUsageLoaded = false + s.diskUsageMu.Unlock() } -func (m *manager) endLayerBuild() { - m.diskUsageMu.Lock() - m.activeLayerBuilds-- - m.diskUsageGeneration++ - m.diskUsageLoaded = false - m.diskUsageMu.Unlock() +func (s *layerStore) endLayerBuild() { + s.diskUsageMu.Lock() + s.activeBuilds-- + s.diskUsageGeneration++ + s.diskUsageLoaded = false + s.diskUsageMu.Unlock() } -func (m *manager) refreshDiskUsageTotals() { - m.diskUsageMu.RLock() - generation := m.diskUsageGeneration - m.diskUsageMu.RUnlock() +func (s *layerStore) refreshDiskUsageTotals() { + s.diskUsageMu.RLock() + generation := s.diskUsageGeneration + s.diskUsageMu.RUnlock() - readyImageBytes, ociCacheBytes, err := m.computeDiskUsageTotals(context.Background()) + readyImageBytes, ociCacheBytes, err := s.computeDiskUsageTotals(context.Background()) if err != nil { return } - m.diskUsageMu.Lock() - if m.diskUsageGeneration == generation && m.activeLayerBuilds == 0 { - m.readyImageBytes = readyImageBytes - m.ociCacheBytes = ociCacheBytes - m.diskUsageLoaded = true + s.diskUsageMu.Lock() + if s.diskUsageGeneration == generation && s.activeBuilds == 0 { + s.readyImageBytes = readyImageBytes + s.ociCacheBytes = ociCacheBytes + s.diskUsageLoaded = true } - m.diskUsageMu.Unlock() + s.diskUsageMu.Unlock() } -func (m *manager) computeDiskUsageTotals(ctx context.Context) (int64, int64, error) { - readyImageBytes, err := totalReadyImageBytesFromMetadataWithContext(ctx, m.paths.ImagesDir()) +func (s *layerStore) computeDiskUsageTotals(ctx context.Context) (int64, int64, error) { + readyImageBytes, err := totalReadyImageBytesFromMetadataWithContext(ctx, s.paths.ImagesDir()) if err != nil { return 0, 0, err } - ociCacheBytes, err := totalOCICacheBlobBytesFromFilesystemWithContext(ctx, m.paths.OCICacheBlobDir()) + ociCacheBytes, err := totalOCICacheBlobBytesFromFilesystemWithContext(ctx, s.paths.OCICacheBlobDir()) if err != nil { return 0, 0, err } - layerArtifactBytes, err := totalLayerArtifactBytesFromFilesystemWithContext(ctx, m.paths.ImageLayersDir()) + layerArtifactBytes, err := totalLayerArtifactBytesFromFilesystemWithContext(ctx, s.paths.ImageLayersDir()) if err != nil { return 0, 0, err } return readyImageBytes, ociCacheBytes + layerArtifactBytes, nil } -func totalRootfsBytesInDigestDirWithContext(ctx context.Context, digestDir string) (int64, error) { +func totalRootfsBytesInDigestDirWithContext(ctx context.Context, digestDir string, seen map[rootfsIdentity]struct{}) (int64, error) { rootfsPaths, err := filepath.Glob(filepath.Join(digestDir, "rootfs.*")) if err != nil { return 0, err @@ -239,6 +246,7 @@ func totalRootfsBytesInDigestDirWithContext(ctx context.Context, digestDir strin } var total int64 + found := false for _, rootfsPath := range rootfsPaths { if err := ctx.Err(); err != nil { return 0, err @@ -253,9 +261,13 @@ func totalRootfsBytesInDigestDirWithContext(ctx context.Context, digestDir strin if info.IsDir() { continue } + found = true + if seen != nil && !markUniqueRootfs(info, seen) { + continue + } total += info.Size() } - if total == 0 { + if !found || seen == nil && total == 0 { return 0, os.ErrNotExist } return total, nil @@ -278,40 +290,3 @@ func markUniqueRootfs(info os.FileInfo, seen map[rootfsIdentity]struct{}) bool { seen[identity] = struct{}{} return true } - -func totalUniqueRootfsBytesInDigestDirWithContext(ctx context.Context, digestDir string, seen map[rootfsIdentity]struct{}) (int64, error) { - rootfsPaths, err := filepath.Glob(filepath.Join(digestDir, "rootfs.*")) - if err != nil { - return 0, err - } - if len(rootfsPaths) == 0 { - return 0, os.ErrNotExist - } - - var total int64 - found := false - for _, rootfsPath := range rootfsPaths { - if err := ctx.Err(); err != nil { - return 0, err - } - info, err := os.Stat(rootfsPath) - if err != nil { - if os.IsNotExist(err) { - continue - } - return 0, err - } - if info.IsDir() { - continue - } - found = true - if !markUniqueRootfs(info, seen) { - continue - } - total += info.Size() - } - if !found { - return 0, os.ErrNotExist - } - return total, nil -} diff --git a/lib/images/layer_artifact.go b/lib/images/layer_artifact.go index abe4ed58a..fb13f1b48 100644 --- a/lib/images/layer_artifact.go +++ b/lib/images/layer_artifact.go @@ -18,6 +18,7 @@ import ( "github.com/kernel/hypeman/lib/paths" "github.com/klauspost/compress/zstd" + v1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/opencontainers/umoci/oci/layer" "golang.org/x/sys/unix" ) @@ -87,10 +88,11 @@ func (a *layerArtifact) matches(desc layerDescriptor) bool { } func layerEncoding(mediaType string) string { + normalized := convertToOCIMediaType(mediaType) switch { - case strings.HasSuffix(mediaType, "+zstd"), strings.Contains(mediaType, "tar.zstd"): + case normalized == v1.MediaTypeImageLayerZstd, strings.HasSuffix(normalized, ".tar.zstd"): return "zstd" - case strings.HasSuffix(mediaType, "+gzip"), strings.Contains(mediaType, "tar.gzip"): + case normalized == v1.MediaTypeImageLayerGzip, strings.HasSuffix(normalized, ".tar.gzip"): return "gzip" default: return "raw" @@ -161,7 +163,11 @@ func probeLayerArtifactSupport(layersDir string) bool { } func (m *manager) layerArtifactSupport() bool { - return m.layerArtifactsSupported + return m.layers.artifactsSupported +} + +func (m *manager) materializeLayerArtifact(ctx context.Context, desc layerDescriptor) (*layerArtifact, error) { + return m.layers.materializeLayerArtifact(ctx, desc) } // layerArtifactOnDiskFormat is the extraction format for per-layer artifacts. @@ -217,9 +223,9 @@ func discardLayerCache(p *paths.Paths, layerHex string) error { return nil } -func (m *manager) clearLayerCache(layerHex string) error { - err := discardLayerCache(m.paths, layerHex) - m.invalidateDiskUsageTotals() +func (s *layerStore) clearLayerCache(layerHex string) error { + err := discardLayerCache(s.paths, layerHex) + s.invalidateDiskUsageTotals() return err } @@ -236,27 +242,27 @@ func (m *manager) clearLayerCache(layerHex string) error { // initiating caller's cancellation so one cancelled pull cannot fail every // other pull waiting on the same layer; each caller still returns as soon as // its own context is done. -func (m *manager) materializeLayerArtifact(ctx context.Context, desc layerDescriptor) (*layerArtifact, error) { +func (s *layerStore) materializeLayerArtifact(ctx context.Context, desc layerDescriptor) (*layerArtifact, error) { layerHex, err := layerDigestHex(desc.Digest) if err != nil { return nil, err } key := desc.Digest + "\x00" + layerArtifactFormat() - result := m.layerFlights.DoChan(key, func() (any, error) { + result := s.flights.DoChan(key, func() (any, error) { // The build outlives the initiating request, so this deadline is its // only bound: without it a hung cache-blob read would wedge the // singleflight key, and every future caller for the layer, forever. buildCtx, cancel := context.WithTimeout(context.Background(), layerBuildTimeout) defer cancel() - if m.layerBuilds != nil { + if s.builds != nil { select { - case m.layerBuilds <- struct{}{}: - defer func() { <-m.layerBuilds }() + case s.builds <- struct{}{}: + defer func() { <-s.builds }() case <-buildCtx.Done(): return nil, buildCtx.Err() } } - return m.materializeLayerArtifactOnce(buildCtx, desc, layerHex) + return s.materializeLayerArtifactOnce(buildCtx, desc, layerHex) }) select { case <-ctx.Done(): @@ -273,44 +279,44 @@ func (m *manager) materializeLayerArtifact(ctx context.Context, desc layerDescri } } -func (m *manager) materializeLayerArtifactOnce(ctx context.Context, desc layerDescriptor, layerHex string) (*layerArtifact, error) { +func (s *layerStore) materializeLayerArtifactOnce(ctx context.Context, desc layerDescriptor, layerHex string) (*layerArtifact, error) { if layerArtifactFormat() == "" { return nil, fmt.Errorf("unsupported layer artifact format: %s", DefaultImageFormat) } - if !m.layerArtifactSupport() { + if !s.artifactsSupported { return nil, fmt.Errorf("layer artifacts are unsupported on this host") } - if record, err := readLayerRecord(m.paths, layerHex); err != nil { + if record, err := readLayerRecord(s.paths, layerHex); err != nil { if !errors.Is(err, errCorruptLayerRecord) { return nil, err } - if discardErr := m.clearLayerCache(layerHex); discardErr != nil { + if discardErr := s.clearLayerCache(layerHex); discardErr != nil { return nil, fmt.Errorf("discard corrupt layer cache: %w", discardErr) } } else if record != nil && record.Digest == desc.Digest && record.Format == layerArtifactFormat() && record.Encoding != layerEncoding(desc.MediaType) { return nil, fmt.Errorf("layer artifact encoding does not match descriptor %s", desc.Digest) } else if record != nil && record.matches(desc) { - artifactInfo, statErr := os.Lstat(layerArtifactPath(m.paths, layerHex)) + artifactInfo, statErr := os.Lstat(layerArtifactPath(s.paths, layerHex)) if statErr == nil && artifactInfo.Mode().IsRegular() && record.SizeBytes > 0 && artifactInfo.Size() == record.SizeBytes { - artifactDigest, digestErr := fileSHA256(ctx, layerArtifactPath(m.paths, layerHex)) + artifactDigest, digestErr := fileSHA256(ctx, layerArtifactPath(s.paths, layerHex)) if digestErr == nil && artifactDigest == record.ArtifactDigest { return record, nil } } - if discardErr := m.clearLayerCache(layerHex); discardErr != nil { + if discardErr := s.clearLayerCache(layerHex); discardErr != nil { return nil, fmt.Errorf("discard invalid layer cache: %w", discardErr) } } - artifactPath := layerArtifactPath(m.paths, layerHex) + artifactPath := layerArtifactPath(s.paths, layerHex) if artifactInfo, statErr := os.Lstat(artifactPath); statErr == nil && !artifactInfo.Mode().IsRegular() { - if discardErr := m.clearLayerCache(layerHex); discardErr != nil { + if discardErr := s.clearLayerCache(layerHex); discardErr != nil { return nil, fmt.Errorf("discard invalid layer artifact: %w", discardErr) } } - layerDir := m.paths.ImageLayerDir(layerHex) + layerDir := s.paths.ImageLayerDir(layerHex) if err := os.MkdirAll(layerDir, 0755); err != nil { return nil, fmt.Errorf("create layer directory: %w", err) } @@ -318,28 +324,28 @@ func (m *manager) materializeLayerArtifactOnce(ctx context.Context, desc layerDe if err != nil { return nil, fmt.Errorf("create unpack directory: %w", err) } - m.beginLayerBuild() + s.beginLayerBuild() defer func() { if err := removePath(unpackDir); err != nil { slog.Warn("failed to remove layer unpack directory", "dir", unpackDir, "error", err) } - m.endLayerBuild() + s.endLayerBuild() }() - stats, err := unpackCachedLayer(ctx, m.paths, desc, unpackDir, layerArtifactOnDiskFormat()) + stats, err := unpackCachedLayer(ctx, s.paths, desc, unpackDir, layerArtifactOnDiskFormat()) if err != nil { return nil, err } - record, err := m.installLayerArtifact(ctx, desc, layerHex, unpackDir, stats) + record, err := s.installLayerArtifact(ctx, desc, layerHex, unpackDir, stats) if err != nil { - m.invalidateDiskUsageTotals() + s.invalidateDiskUsageTotals() return nil, err } - m.invalidateDiskUsageTotals() + s.invalidateDiskUsageTotals() return record, nil } -func (m *manager) installLayerArtifact(ctx context.Context, desc layerDescriptor, layerHex, unpackDir string, stats *unpackStats) (*layerArtifact, error) { +func (s *layerStore) installLayerArtifact(ctx context.Context, desc layerDescriptor, layerHex, unpackDir string, stats *unpackStats) (*layerArtifact, error) { record := &layerArtifact{ SchemaVersion: layerRecordSchemaVersion, Digest: desc.Digest, @@ -350,7 +356,7 @@ func (m *manager) installLayerArtifact(ctx context.Context, desc layerDescriptor CreatedAt: time.Now(), } - if err := installAtomically(layerArtifactPath(m.paths, layerHex), func(path string) error { + if err := installAtomically(layerArtifactPath(s.paths, layerHex), func(path string) error { size, err := ExportRootfsWithContext(ctx, unpackDir, path, DefaultImageFormat) if err != nil { return err @@ -366,8 +372,8 @@ func (m *manager) installLayerArtifact(ctx context.Context, desc layerDescriptor if err != nil { return nil, fmt.Errorf("marshal layer record: %w", err) } - if err := writeJSONAtomic(layerArtifactRecordPath(m.paths, layerHex), data); err != nil { - _ = os.Remove(layerArtifactPath(m.paths, layerHex)) + if err := writeJSONAtomic(layerArtifactRecordPath(s.paths, layerHex), data); err != nil { + _ = os.Remove(layerArtifactPath(s.paths, layerHex)) return nil, fmt.Errorf("write layer record: %w", err) } return record, nil @@ -446,13 +452,8 @@ func unpackLayerBlob(ctx context.Context, blobPath, mediaType, dest string, onDi defer blob.Close() blobHash := sha256.New() - if _, err := io.Copy(blobHash, contextReader{ctx: ctx, reader: blob}); err != nil { - return nil, fmt.Errorf("hash blob: %w", err) - } - if _, err := blob.Seek(0, io.SeekStart); err != nil { - return nil, fmt.Errorf("rewind blob: %w", err) - } - reader, closer, err := decompressLayer(blob, mediaType) + compressed := io.TeeReader(contextReader{ctx: ctx, reader: blob}, blobHash) + reader, closer, err := decompressLayer(compressed, mediaType) if err != nil { return nil, err } @@ -467,6 +468,9 @@ func unpackLayerBlob(ctx context.Context, blobPath, mediaType, dest string, onDi if _, err := io.Copy(io.Discard, hashed); err != nil { return nil, fmt.Errorf("drain layer: %w", err) } + if _, err := io.Copy(io.Discard, compressed); err != nil { + return nil, fmt.Errorf("drain blob: %w", err) + } if limited.N == 0 { return nil, fmt.Errorf("layer exceeds maximum unpacked size of %d bytes", maxLayerUnpackedBytes) } @@ -503,6 +507,9 @@ func decompressLayer(r io.Reader, mediaType string) (io.Reader, io.Closer, error // removePath removes a tree that may contain read-only directories restored // from layer metadata. func removePath(path string) error { + if err := os.RemoveAll(path); err == nil || os.IsNotExist(err) { + return nil + } if err := makeTreeWritable(path); err != nil && !os.IsNotExist(err) { return err } diff --git a/lib/images/layer_artifact_test.go b/lib/images/layer_artifact_test.go index 9711a682a..596c44196 100644 --- a/lib/images/layer_artifact_test.go +++ b/lib/images/layer_artifact_test.go @@ -44,7 +44,9 @@ const testTarGzMediaType = "application/vnd.oci.image.layer.v1.tar+gzip" // writeLayerTestLayout writes img into the shared OCI cache of p tagged with // the image's digest, mirroring pullToOCILayout. func testLayerArtifactManager(p *paths.Paths) *manager { - return &manager{paths: p, layerArtifactsSupported: true, layerBuilds: make(chan struct{}, 1)} + store := newLayerStore(p, 1) + store.artifactsSupported = true + return &manager{paths: p, layers: store} } func writeLayerTestLayout(t *testing.T, p *paths.Paths, img gcr.Image) { @@ -310,7 +312,9 @@ func TestMaterializeLayerArtifactOutlivesCancelledCaller(t *testing.T) { func TestMaterializeLayerArtifactRejectsUnsupportedHost(t *testing.T) { p := paths.New(t.TempDir()) - m := &manager{paths: p} + store := newLayerStore(p, 1) + store.artifactsSupported = false + m := &manager{paths: p, layers: store} _, err := m.materializeLayerArtifact(context.Background(), layerDescriptor{ Digest: "sha256:" + strings.Repeat("ab", 32), MediaType: testTarGzMediaType, diff --git a/lib/images/layer_store.go b/lib/images/layer_store.go new file mode 100644 index 000000000..46d414439 --- /dev/null +++ b/lib/images/layer_store.go @@ -0,0 +1,35 @@ +package images + +import ( + "sync" + + "github.com/kernel/hypeman/lib/paths" + "golang.org/x/sync/singleflight" +) + +// layerStore owns the state shared by layer materialization and accounting. +// Keeping it separate from manager makes the layer-specific concurrency and +// cache lifecycle explicit. +type layerStore struct { + paths *paths.Paths + flights singleflight.Group + builds chan struct{} + artifactsSupported bool + diskUsageMu sync.RWMutex + diskUsageGeneration uint64 + activeBuilds int + diskUsageLoaded bool + readyImageBytes int64 + ociCacheBytes int64 +} + +func newLayerStore(p *paths.Paths, maxConcurrentBuilds int) *layerStore { + if maxConcurrentBuilds < 1 { + maxConcurrentBuilds = 1 + } + return &layerStore{ + paths: p, + builds: make(chan struct{}, maxConcurrentBuilds), + artifactsSupported: probeLayerArtifactSupport(p.ImageLayersDir()), + } +} diff --git a/lib/images/manager.go b/lib/images/manager.go index f7b53ba62..6ebc221f6 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -19,7 +19,6 @@ import ( "github.com/kernel/hypeman/lib/queue" "github.com/kernel/hypeman/lib/tags" "go.opentelemetry.io/otel/metric" - "golang.org/x/sync/singleflight" ) var errStaleBuild = errors.New("stale image build") @@ -77,17 +76,9 @@ type manager struct { queue *queue.Queue createMu sync.Mutex storeLock *imageStoreLock - layerFlights singleflight.Group - layerBuilds chan struct{} - layerArtifactsSupported bool - diskUsageMu sync.RWMutex - diskUsageGeneration uint64 - activeLayerBuilds int + layers *layerStore tagGenerations map[string]uint64 requestedTags map[string]string // newest pull's digest per requested tag - diskUsageLoaded bool - readyImageBytes int64 - ociCacheBytes int64 metrics *Metrics inflightPulls map[string]*inflightImagePull // keyed by digest borrowedCredentialsTimeout time.Duration @@ -112,19 +103,19 @@ func NewManager(p *paths.Paths, maxConcurrentBuilds int, meter metric.Meter) (Ma if maxConcurrentBuilds < 1 { maxConcurrentBuilds = 1 } + layers := newLayerStore(p, maxConcurrentBuilds) m := &manager{ paths: p, storeLock: storeLock, + layers: layers, ociClient: ociClient, queue: queue.New(maxConcurrentBuilds), - layerBuilds: make(chan struct{}, maxConcurrentBuilds), inflightPulls: make(map[string]*inflightImagePull), borrowedCredentialsTimeout: DefaultBorrowedCredentialsTimeout, readySubscribers: make(map[string][]chan StatusEvent), tagGenerations: make(map[string]uint64), requestedTags: make(map[string]string), } - m.layerArtifactsSupported = probeLayerArtifactSupport(p.ImageLayersDir()) // Initialize metrics if meter is provided if meter != nil { @@ -612,7 +603,7 @@ func (m *manager) finalizeImage(ref *ResolvedRef, result *pullResult, diskSize i if !m.claimRequestedTags(ref, meta) { m.cleanupUnclaimedImage(ref) } - m.refreshDiskUsageTotals() + m.layers.refreshDiskUsageTotals() return nil } @@ -825,7 +816,7 @@ func (m *manager) deleteDigestImage(repository, digestHex string) error { return err } m.clearRequestedDigest(digestHex) - m.refreshDiskUsageTotals() + m.layers.refreshDiskUsageTotals() return nil } @@ -859,13 +850,13 @@ func (m *manager) deleteTaggedImage(repository, tag string) error { if err := removeDigestIfUnreferenced(m.paths, repository, digestHex, true); err != nil { return fmt.Errorf("delete orphaned digest %s: %w", digestHex, err) } - m.refreshDiskUsageTotals() + m.layers.refreshDiskUsageTotals() return nil } // TotalImageBytes returns the total size of all ready images on disk. func (m *manager) TotalImageBytes(ctx context.Context) (int64, error) { - readyImageBytes, _, err := m.getDiskUsageTotals(ctx) + readyImageBytes, _, err := m.layers.getDiskUsageTotals(ctx) if err != nil { return 0, err } @@ -874,7 +865,7 @@ func (m *manager) TotalImageBytes(ctx context.Context) (int64, error) { // TotalOCICacheBytes returns the total size of the OCI and materialized layer caches. func (m *manager) TotalOCICacheBytes(ctx context.Context) (int64, error) { - _, ociCacheBytes, err := m.getDiskUsageTotals(ctx) + _, ociCacheBytes, err := m.layers.getDiskUsageTotals(ctx) if err != nil { return 0, err } diff --git a/lib/images/oci.go b/lib/images/oci.go index 88bd6de24..36684cacd 100644 --- a/lib/images/oci.go +++ b/lib/images/oci.go @@ -641,6 +641,8 @@ func convertToOCIMediaType(mediaType string) string { return v1.MediaTypeImageConfig case "application/vnd.docker.image.rootfs.diff.tar.gzip": return v1.MediaTypeImageLayerGzip + case "application/vnd.docker.image.rootfs.diff.tar.zstd": + return v1.MediaTypeImageLayerZstd case "application/vnd.docker.image.rootfs.diff.tar": return v1.MediaTypeImageLayer default: diff --git a/lib/images/tag.go b/lib/images/tag.go index 302263d7e..af7582476 100644 --- a/lib/images/tag.go +++ b/lib/images/tag.go @@ -92,7 +92,7 @@ func (m *manager) cleanupReplacedTag(ref *NormalizedRef, previousDigest, digestH if err := removeDigestIfUnreferenced(m.paths, ref.Repository(), previousDigest, true); err != nil { slog.Warn("failed to collect replaced image content", "repository", ref.Repository(), "digest", previousDigest, "error", err) } - m.refreshDiskUsageTotals() + m.layers.refreshDiskUsageTotals() } func parseTagReferences(source, target string) (*NormalizedRef, *NormalizedRef, error) { diff --git a/lib/images/tag_test.go b/lib/images/tag_test.go index e717d9eea..9a6d6a605 100644 --- a/lib/images/tag_test.go +++ b/lib/images/tag_test.go @@ -54,7 +54,12 @@ func seedImage(t *testing.T, p *paths.Paths, repository, tag, digestHex string, func newTagTestCase(t *testing.T) (*paths.Paths, *manager, string) { t.Helper() p := paths.New(t.TempDir()) - return p, &manager{paths: p, tagGenerations: make(map[string]uint64), requestedTags: make(map[string]string)}, "docker.io/library/alpine" + return p, &manager{ + paths: p, + layers: newLayerStore(p, 1), + tagGenerations: make(map[string]uint64), + requestedTags: make(map[string]string), + }, "docker.io/library/alpine" } func requireTagResolvesTo(t *testing.T, p *paths.Paths, repository, tag, digest string) { diff --git a/lib/resources/disk.go b/lib/resources/disk.go index 50e106828..9c15e4588 100644 --- a/lib/resources/disk.go +++ b/lib/resources/disk.go @@ -90,19 +90,21 @@ func (d *DiskResource) GetBreakdown(ctx context.Context) (*DiskBreakdown, error) // Get volume sizes if d.volumeLister != nil { volumeBytes, err := d.volumeLister.TotalVolumeBytes(ctx) - if err == nil { - breakdown.Volumes = volumeBytes + if err != nil { + return nil, fmt.Errorf("get volume disk usage: %w", err) } + breakdown.Volumes = volumeBytes } // Get overlay sizes from instances if d.instanceLister != nil { instances, err := d.instanceLister.ListInstanceAllocations(ctx) - if err == nil { - for _, inst := range instances { - if isActiveState(inst.State) { - breakdown.Overlays += inst.OverlayBytes + inst.VolumeOverlayBytes - } + if err != nil { + return nil, fmt.Errorf("get instance disk usage: %w", err) + } + for _, inst := range instances { + if isActiveState(inst.State) { + breakdown.Overlays += inst.OverlayBytes + inst.VolumeOverlayBytes } } } diff --git a/lib/resources/resource_test.go b/lib/resources/resource_test.go index 0ba2504af..227392643 100644 --- a/lib/resources/resource_test.go +++ b/lib/resources/resource_test.go @@ -2,6 +2,7 @@ package resources import ( "context" + "errors" "runtime" "testing" @@ -14,10 +15,11 @@ import ( // mockInstanceLister implements InstanceLister for testing type mockInstanceLister struct { allocations []InstanceAllocation + err error } func (m *mockInstanceLister) ListInstanceAllocations(ctx context.Context) ([]InstanceAllocation, error) { - return m.allocations, nil + return m.allocations, m.err } // mockImageLister implements ImageLister for testing @@ -37,10 +39,11 @@ func (m *mockImageLister) TotalOCICacheBytes(ctx context.Context) (int64, error) // mockVolumeLister implements VolumeLister for testing type mockVolumeLister struct { totalBytes int64 + err error } func (m *mockVolumeLister) TotalVolumeBytes(ctx context.Context) (int64, error) { - return m.totalBytes, nil + return m.totalBytes, m.err } func TestNewManager(t *testing.T) { @@ -487,6 +490,23 @@ func TestDiskBreakdown_IncludesOCICacheAndVolumeOverlays(t *testing.T) { assert.Equal(t, int64(25*1024*1024*1024), status.DiskDetail.Overlays) } +func TestDiskBreakdownPropagatesVolumeAndInstanceErrors(t *testing.T) { + cfg := &config.Config{DataDir: t.TempDir(), Capacity: config.CapacityConfig{Disk: "100GB"}} + p := paths.New(cfg.DataDir) + + volumeErr := errors.New("volume lookup failed") + disk, err := NewDiskResource(cfg, p, nil, nil, &mockVolumeLister{err: volumeErr}) + require.NoError(t, err) + _, err = disk.GetBreakdown(context.Background()) + require.ErrorIs(t, err, volumeErr) + + instanceErr := errors.New("instance lookup failed") + disk, err = NewDiskResource(cfg, p, &mockInstanceLister{err: instanceErr}, nil, nil) + require.NoError(t, err) + _, err = disk.GetBreakdown(context.Background()) + require.ErrorIs(t, err, instanceErr) +} + func TestValidateAllocation_Disk(t *testing.T) { cfg := &config.Config{ DataDir: t.TempDir(),