diff --git a/lib/images/README.md b/lib/images/README.md index eb2d422be..e1d346a7a 100644 --- a/lib/images/README.md +++ b/lib/images/README.md @@ -65,6 +65,10 @@ Content-addressable storage with tag symlinks (similar to Docker/Unikraft): rootfs.erofs latest -> abc123def456... # Tag symlink to digest 3.18 -> def456abc123... # Another tag + layers/ # Shared materialized layer artifacts + abc123def456.../ + layer.erofs + artifact.erofs.json system/ oci-cache/ # Shared OCI layout for all images index.json # Manifest index with digest-based tags @@ -82,6 +86,7 @@ Content-addressable storage with tag symlinks (similar to Docker/Unikraft): - Natural hierarchy: All versions of an image grouped under repository - Easy inspection: Clear which digest belongs to which image - Layer caching: All images share the same blob storage, layers deduplicated automatically +- Materialized layer artifacts are reference-protected and reconciled by the layer lifecycle manager; stale temporary trees are age-gated before removal. **Design:** - Images stored by manifest digest (content hash) @@ -92,6 +97,8 @@ Content-addressable storage with tag symlinks (similar to Docker/Unikraft): - Shared blob storage enables automatic layer deduplication across all images - Orphaned digests are automatically deleted when the last tag referencing them is removed - Symlinks only created after successful build (status: ready) +- One Hypeman process owns a data directory at a time; the image manager holds `images/.lock` for its lifetime and rejects a second owner. +- Disk accounting uses logical file sizes, matching image metadata and storage admission rather than filesystem block allocation. ## Reference Handling (reference.go) diff --git a/lib/images/disk.go b/lib/images/disk.go index 896d10f2a..68184b785 100644 --- a/lib/images/disk.go +++ b/lib/images/disk.go @@ -1,7 +1,9 @@ package images import ( + "context" "fmt" + "io" "os" "os/exec" "path/filepath" @@ -29,15 +31,20 @@ var DefaultImageFormat = func() ExportFormat { return FormatErofs }() -// ExportRootfs exports rootfs directory in specified format (public for system manager) +// ExportRootfs exports rootfs directory in specified format (public for system manager). func ExportRootfs(rootfsDir, outputPath string, format ExportFormat) (int64, error) { + return ExportRootfsWithContext(context.Background(), rootfsDir, outputPath, format) +} + +// ExportRootfsWithContext exports rootfs directory and cancels external formatters with ctx. +func ExportRootfsWithContext(ctx context.Context, rootfsDir, outputPath string, format ExportFormat) (int64, error) { switch format { case FormatExt4: - return convertToExt4(rootfsDir, outputPath) + return convertToExt4(ctx, rootfsDir, outputPath) case FormatErofs: - return convertToErofs(rootfsDir, outputPath) + return convertToErofs(ctx, rootfsDir, outputPath) case FormatCpio: - return convertToCpio(rootfsDir, outputPath) + return convertToCpio(ctx, rootfsDir, outputPath) default: return 0, fmt.Errorf("unsupported export format: %s", format) } @@ -45,7 +52,7 @@ func ExportRootfs(rootfsDir, outputPath string, format ExportFormat) (int64, err // convertToCpio packages directory as uncompressed cpio archive (initramfs format) // Uses uncompressed format for faster boot (kernel loads directly without decompression) -func convertToCpio(rootfsDir, outputPath string) (int64, error) { +func convertToCpio(ctx context.Context, rootfsDir, outputPath string) (int64, error) { // Ensure parent directory exists if err := os.MkdirAll(filepath.Dir(outputPath), 0755); err != nil { return 0, fmt.Errorf("create output dir: %w", err) @@ -56,7 +63,13 @@ func convertToCpio(rootfsDir, outputPath string) (int64, error) { if err != nil { return 0, fmt.Errorf("create output file: %w", err) } - defer outFile.Close() + keepOutput := false + defer func() { + _ = outFile.Close() + if !keepOutput { + _ = os.Remove(outputPath) + } + }() // Create newc format cpio writer (kernel-compatible format) cpioWriter := cpio.Newc.Writer(outFile) @@ -66,6 +79,9 @@ func convertToCpio(rootfsDir, outputPath string) (int64, error) { // Walk the rootfs directory and add all files err = filepath.Walk(rootfsDir, func(path string, info os.FileInfo, err error) error { + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } if err != nil { return err } @@ -89,6 +105,13 @@ func convertToCpio(rootfsDir, outputPath string) (int64, error) { // Set the name to be relative to root rec.Name = relPath + if rec.ReaderAt != nil { + rec.ReaderAt = &contextReaderAt{ + ctx: ctx, + reader: rec.ReaderAt, + closer: readerCloser(rec.ReaderAt), + } + } // Write the record to the archive if err := cpioWriter.WriteRecord(rec); err != nil { @@ -113,9 +136,35 @@ func convertToCpio(rootfsDir, outputPath string) (int64, error) { return 0, fmt.Errorf("stat output: %w", err) } + keepOutput = true return stat.Size(), nil } +type contextReaderAt struct { + ctx context.Context + reader io.ReaderAt + closer io.Closer +} + +func (r *contextReaderAt) ReadAt(p []byte, off int64) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.reader.ReadAt(p, off) +} + +func (r *contextReaderAt) Close() error { + if r.closer == nil { + return nil + } + return r.closer.Close() +} + +func readerCloser(reader io.ReaderAt) io.Closer { + closer, _ := reader.(io.Closer) + return closer +} + // sectorSize is the block size for disk images (required by Virtualization.framework) const sectorSize = 4096 @@ -128,9 +177,9 @@ func alignToSector(size int64) int64 { } // convertToExt4 converts a rootfs directory to an ext4 disk image using mkfs.ext4 -func convertToExt4(rootfsDir, diskPath string) (int64, error) { +func convertToExt4(ctx context.Context, rootfsDir, diskPath string) (int64, error) { // Calculate size of rootfs directory - sizeBytes, err := dirSize(rootfsDir) + sizeBytes, err := dirSizeWithContext(ctx, rootfsDir) if err != nil { return 0, fmt.Errorf("calculate dir size: %w", err) } @@ -168,7 +217,7 @@ func convertToExt4(rootfsDir, diskPath string) (int64, error) { // -O ^has_journal: Disable journal (not needed for read-only VM mounts) // -d: Copy directory contents into filesystem // -F: Force creation (file not block device) - cmd := exec.Command(mkfsExt4Binary(), "-b", "4096", "-O", "^has_journal", "-d", rootfsDir, "-F", diskPath) + cmd := exec.CommandContext(ctx, mkfsExt4Binary(), "-b", "4096", "-O", "^has_journal", "-d", rootfsDir, "-F", diskPath) output, err := cmd.CombinedOutput() if err != nil { return 0, fmt.Errorf("mkfs.ext4 failed: %w, output: %s", err, output) @@ -193,7 +242,7 @@ func convertToExt4(rootfsDir, diskPath string) (int64, error) { } // convertToErofs converts a rootfs directory to an erofs disk image using mkfs.erofs -func convertToErofs(rootfsDir, diskPath string) (int64, error) { +func convertToErofs(ctx context.Context, rootfsDir, diskPath string) (int64, error) { // Ensure parent directory exists if err := os.MkdirAll(filepath.Dir(diskPath), 0755); err != nil { return 0, fmt.Errorf("create disk parent dir: %w", err) @@ -202,7 +251,7 @@ func convertToErofs(rootfsDir, diskPath string) (int64, error) { // Create erofs image with LZ4 fast compression // -zlz4: LZ4 fast compression (~20-25% space savings, faster builds) // erofs doesn't need pre-allocation, creates file directly - cmd := exec.Command("mkfs.erofs", "-zlz4", diskPath, rootfsDir) + cmd := exec.CommandContext(ctx, "mkfs.erofs", "-zlz4", diskPath, rootfsDir) output, err := cmd.CombinedOutput() if err != nil { return 0, fmt.Errorf("mkfs.erofs failed: %w, output: %s", err, output) @@ -226,13 +275,20 @@ func convertToErofs(rootfsDir, diskPath string) (int64, error) { return stat.Size(), nil } -// dirSize calculates the total size of a directory +// dirSize is used by layer-store reconciliation. func dirSize(path string) (int64, error) { + return dirSizeWithContext(context.Background(), path) +} + +func dirSizeWithContext(ctx context.Context, path string) (int64, error) { var size int64 err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error { if err != nil { return err } + if err := ctx.Err(); err != nil { + return err + } if !info.IsDir() { size += info.Size() } diff --git a/lib/images/disk_usage.go b/lib/images/disk_usage.go index b65dba1ac..ca72c1994 100644 --- a/lib/images/disk_usage.go +++ b/lib/images/disk_usage.go @@ -1,6 +1,7 @@ package images import ( + "context" "encoding/json" "fmt" "os" @@ -8,43 +9,65 @@ import ( "syscall" ) +func walkWithContext(ctx context.Context, root string, walkFn filepath.WalkFunc) error { + return filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + return walkFn(path, info, err) + }) +} + // totalReadyImageBytesFromMetadata sums ready image sizes directly from metadata.json files. // This is conservative for admission control and disk accounting: if metadata says an // image is ready, we count its recorded size without re-validating the rootfs path. If // the metadata file is unreadable or malformed, we fall back to counting any rootfs disk // files found in the digest directory so we do not undercount host disk usage. -func totalReadyImageBytesFromMetadata(imagesDir string) (int64, error) { +func totalReadyImageBytesFromMetadataWithContext(ctx context.Context, imagesDir string) (int64, error) { var total int64 seenRootfs := make(map[rootfsIdentity]struct{}) + layersDir := filepath.Join(imagesDir, "layers") - err := filepath.Walk(imagesDir, func(path string, info os.FileInfo, err error) error { + err := walkWithContext(ctx, imagesDir, func(path string, info os.FileInfo, err error) error { if err != nil { if os.IsNotExist(err) { return nil } return err } - if info.IsDir() || info.Name() != "metadata.json" { + if info.IsDir() { + if filepath.Clean(path) == filepath.Clean(layersDir) { + return filepath.SkipDir + } + return nil + } + if info.Name() != "metadata.json" { return nil } data, err := os.ReadFile(path) if err != nil { - rootfsBytes, fallbackErr := totalUniqueRootfsBytesInDigestDir(filepath.Dir(path), seenRootfs) + rootfsBytes, fallbackErr := totalRootfsBytesInDigestDirWithContext(ctx, filepath.Dir(path), seenRootfs) if fallbackErr == nil { total += rootfsBytes return nil } + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } return fmt.Errorf("read image metadata %s: %w", path, err) } var meta imageMetadata if err := json.Unmarshal(data, &meta); err != nil { - rootfsBytes, fallbackErr := totalUniqueRootfsBytesInDigestDir(filepath.Dir(path), seenRootfs) + rootfsBytes, fallbackErr := totalRootfsBytesInDigestDirWithContext(ctx, filepath.Dir(path), seenRootfs) if fallbackErr == nil { total += rootfsBytes return nil } + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } return fmt.Errorf("unmarshal image metadata %s: %w", path, err) } if meta.Status == StatusReady { @@ -67,7 +90,7 @@ func totalReadyImageBytesFromMetadata(imagesDir string) (int64, error) { total += meta.SizeBytes return nil } - rootfsBytes, err := totalRootfsBytesInDigestDir(filepath.Dir(path)) + rootfsBytes, err := totalRootfsBytesInDigestDirWithContext(ctx, filepath.Dir(path), nil) if err != nil { return fmt.Errorf("stat ready image rootfs for %s: %w", path, err) } @@ -82,86 +105,138 @@ func totalReadyImageBytesFromMetadata(imagesDir string) (int64, error) { return total, nil } -// totalOCICacheBlobBytesFromFilesystem sums blob sizes directly from the OCI cache blob store. -// This counts the actual bytes on disk, including any blob files that are currently -// present but no longer referenced by the OCI layout index. -func totalOCICacheBlobBytesFromFilesystem(blobDir string) (int64, error) { +func totalFileBytesWithContext(ctx context.Context, dir, label string) (int64, error) { var total int64 - - err := filepath.Walk(blobDir, func(path string, info os.FileInfo, err error) error { + err := walkWithContext(ctx, dir, func(path string, info os.FileInfo, err error) error { if err != nil { if os.IsNotExist(err) { return nil } return err } - if info.IsDir() { - return nil + if !info.IsDir() { + total += info.Size() } - total += info.Size() return nil }) if err != nil && !os.IsNotExist(err) { - return 0, fmt.Errorf("walk OCI cache blobs: %w", err) + return 0, fmt.Errorf("walk %s: %w", label, err) } - return total, nil } -func (m *manager) getDiskUsageTotals() (int64, int64, error) { - m.diskUsageMu.RLock() - if m.diskUsageLoaded { - readyImageBytes := m.readyImageBytes - ociCacheBytes := m.ociCacheBytes - m.diskUsageMu.RUnlock() +// totalLayerArtifactBytesFromFilesystem includes records and in-progress trees. +func totalLayerArtifactBytesFromFilesystemWithContext(ctx context.Context, layersDir string) (int64, error) { + return totalFileBytesWithContext(ctx, layersDir, "layer artifacts") +} + +// totalOCICacheBlobBytesFromFilesystem includes unreferenced blobs still on disk. +func totalOCICacheBlobBytesFromFilesystemWithContext(ctx context.Context, blobDir string) (int64, error) { + return totalFileBytesWithContext(ctx, blobDir, "OCI cache blobs") +} + +func (s *layerStore) getDiskUsageTotals(ctx context.Context) (int64, int64, error) { + for attempt := 0; ; attempt++ { + if err := ctx.Err(); err != nil { + return 0, 0, err + } + s.diskUsageMu.RLock() + generation := s.diskUsageGeneration + if s.diskUsageLoaded { + readyImageBytes := s.readyImageBytes + ociCacheBytes := s.ociCacheBytes + s.diskUsageMu.RUnlock() + return readyImageBytes, ociCacheBytes, nil + } + s.diskUsageMu.RUnlock() + + readyImageBytes, ociCacheBytes, err := s.computeDiskUsageTotals(ctx) + if err != nil { + return 0, 0, err + } + + s.diskUsageMu.Lock() + if s.diskUsageGeneration != generation { + s.diskUsageMu.Unlock() + if attempt >= 2 { + return 0, 0, fmt.Errorf("disk usage changed during scan") + } + continue + } + if s.activeBuilds == 0 { + if !s.diskUsageLoaded { + s.readyImageBytes = readyImageBytes + s.ociCacheBytes = ociCacheBytes + s.diskUsageLoaded = true + } + readyImageBytes = s.readyImageBytes + ociCacheBytes = s.ociCacheBytes + } + s.diskUsageMu.Unlock() + return readyImageBytes, ociCacheBytes, nil } - m.diskUsageMu.RUnlock() +} - readyImageBytes, ociCacheBytes, err := m.computeDiskUsageTotals() - if err != nil { - return 0, 0, err - } +func (s *layerStore) invalidateDiskUsageTotals() { + s.diskUsageMu.Lock() + s.diskUsageGeneration++ + s.diskUsageLoaded = false + s.diskUsageMu.Unlock() +} - m.diskUsageMu.Lock() - if !m.diskUsageLoaded { - m.readyImageBytes = readyImageBytes - m.ociCacheBytes = ociCacheBytes - m.diskUsageLoaded = true - } - readyImageBytes = m.readyImageBytes - ociCacheBytes = m.ociCacheBytes - m.diskUsageMu.Unlock() +func (s *layerStore) beginLayerBuild() { + s.diskUsageMu.Lock() + s.activeBuilds++ + s.diskUsageGeneration++ + s.diskUsageLoaded = false + s.diskUsageMu.Unlock() +} - return readyImageBytes, ociCacheBytes, nil +func (s *layerStore) endLayerBuild() { + s.diskUsageMu.Lock() + s.activeBuilds-- + s.diskUsageGeneration++ + s.diskUsageLoaded = false + s.diskUsageMu.Unlock() } -func (m *manager) refreshDiskUsageTotals() { - readyImageBytes, ociCacheBytes, err := m.computeDiskUsageTotals() +func (s *layerStore) refreshDiskUsageTotals() { + s.diskUsageMu.RLock() + generation := s.diskUsageGeneration + s.diskUsageMu.RUnlock() + + readyImageBytes, ociCacheBytes, err := s.computeDiskUsageTotals(context.Background()) if err != nil { return } - m.diskUsageMu.Lock() - m.readyImageBytes = readyImageBytes - m.ociCacheBytes = ociCacheBytes - m.diskUsageLoaded = true - m.diskUsageMu.Unlock() + s.diskUsageMu.Lock() + if s.diskUsageGeneration == generation && s.activeBuilds == 0 { + s.readyImageBytes = readyImageBytes + s.ociCacheBytes = ociCacheBytes + s.diskUsageLoaded = true + } + s.diskUsageMu.Unlock() } -func (m *manager) computeDiskUsageTotals() (int64, int64, error) { - readyImageBytes, err := totalReadyImageBytesFromMetadata(m.paths.ImagesDir()) +func (s *layerStore) computeDiskUsageTotals(ctx context.Context) (int64, int64, error) { + readyImageBytes, err := totalReadyImageBytesFromMetadataWithContext(ctx, s.paths.ImagesDir()) if err != nil { return 0, 0, err } - ociCacheBytes, err := totalOCICacheBlobBytesFromFilesystem(m.paths.OCICacheBlobDir()) + ociCacheBytes, err := totalOCICacheBlobBytesFromFilesystemWithContext(ctx, s.paths.OCICacheBlobDir()) if err != nil { return 0, 0, err } - return readyImageBytes, ociCacheBytes, nil + layerArtifactBytes, err := totalLayerArtifactBytesFromFilesystemWithContext(ctx, s.paths.ImageLayersDir()) + if err != nil { + return 0, 0, err + } + return readyImageBytes, ociCacheBytes + layerArtifactBytes, nil } -func totalRootfsBytesInDigestDir(digestDir string) (int64, error) { +func totalRootfsBytesInDigestDirWithContext(ctx context.Context, digestDir string, seen map[rootfsIdentity]struct{}) (int64, error) { rootfsPaths, err := filepath.Glob(filepath.Join(digestDir, "rootfs.*")) if err != nil { return 0, err @@ -171,7 +246,11 @@ func totalRootfsBytesInDigestDir(digestDir string) (int64, error) { } var total int64 + found := false for _, rootfsPath := range rootfsPaths { + if err := ctx.Err(); err != nil { + return 0, err + } info, err := os.Stat(rootfsPath) if err != nil { if os.IsNotExist(err) { @@ -182,9 +261,13 @@ func totalRootfsBytesInDigestDir(digestDir string) (int64, error) { if info.IsDir() { continue } + found = true + if seen != nil && !markUniqueRootfs(info, seen) { + continue + } total += info.Size() } - if total == 0 { + if !found || seen == nil && total == 0 { return 0, os.ErrNotExist } return total, nil @@ -207,37 +290,3 @@ func markUniqueRootfs(info os.FileInfo, seen map[rootfsIdentity]struct{}) bool { seen[identity] = struct{}{} return true } - -func totalUniqueRootfsBytesInDigestDir(digestDir string, seen map[rootfsIdentity]struct{}) (int64, error) { - rootfsPaths, err := filepath.Glob(filepath.Join(digestDir, "rootfs.*")) - if err != nil { - return 0, err - } - if len(rootfsPaths) == 0 { - return 0, os.ErrNotExist - } - - var total int64 - found := false - for _, rootfsPath := range rootfsPaths { - info, err := os.Stat(rootfsPath) - if err != nil { - if os.IsNotExist(err) { - continue - } - return 0, err - } - if info.IsDir() { - continue - } - found = true - if !markUniqueRootfs(info, seen) { - continue - } - total += info.Size() - } - if !found { - return 0, os.ErrNotExist - } - return total, nil -} diff --git a/lib/images/disk_usage_test.go b/lib/images/disk_usage_test.go index bd6056fef..2b5404d74 100644 --- a/lib/images/disk_usage_test.go +++ b/lib/images/disk_usage_test.go @@ -1,6 +1,7 @@ package images import ( + "context" "os" "path/filepath" "testing" @@ -17,7 +18,7 @@ func TestTotalReadyImageBytesFromMetadata_UsesRootfsFallbackForMalformedMetadata require.NoError(t, os.WriteFile(filepath.Join(digestDir, "metadata.json"), []byte("{not-json"), 0o644)) require.NoError(t, os.WriteFile(filepath.Join(digestDir, "rootfs.erofs"), []byte("rootfs-data"), 0o644)) - total, err := totalReadyImageBytesFromMetadata(imagesDir) + total, err := totalReadyImageBytesFromMetadataWithContext(context.Background(), imagesDir) require.NoError(t, err) require.Equal(t, int64(len("rootfs-data")), total) } @@ -39,7 +40,7 @@ func TestTotalReadyImageBytesFromMetadata_DeduplicatesHardLinkedAliases(t *testi require.NoError(t, os.WriteFile(filepath.Join(sourceDir, "metadata.json"), metadata, 0o644)) require.NoError(t, os.WriteFile(filepath.Join(targetDir, "metadata.json"), metadata, 0o644)) - total, err := totalReadyImageBytesFromMetadata(imagesDir) + total, err := totalReadyImageBytesFromMetadataWithContext(context.Background(), imagesDir) require.NoError(t, err) require.Equal(t, int64(len("shared-rootfs")), total) } @@ -59,11 +60,51 @@ func TestTotalReadyImageBytesFromMetadata_DeduplicatesMalformedAliases(t *testin require.NoError(t, os.WriteFile(filepath.Join(malformedDir, "metadata.json"), []byte("{not-json"), 0o644)) require.NoError(t, os.WriteFile(filepath.Join(validDir, "metadata.json"), []byte(`{"status":"ready","size_bytes":13}`), 0o644)) - total, err := totalReadyImageBytesFromMetadata(imagesDir) + total, err := totalReadyImageBytesFromMetadataWithContext(context.Background(), imagesDir) require.NoError(t, err) require.Equal(t, int64(len("shared-rootfs")), total) } +func TestTotalFileBytesWithContextStopsWhenCanceled(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, err := totalFileBytesWithContext(ctx, t.TempDir(), "test files") + require.ErrorIs(t, err, context.Canceled) +} + +func TestExportRootfsWithContextStopsCpioWhenCanceled(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, err := ExportRootfsWithContext(ctx, t.TempDir(), filepath.Join(t.TempDir(), "rootfs.cpio"), FormatCpio) + require.ErrorIs(t, err, context.Canceled) +} + +func TestTotalLayerArtifactBytesFromFilesystem(t *testing.T) { + t.Parallel() + + layersDir := t.TempDir() + digestDir := filepath.Join(layersDir, "a", "b") + require.NoError(t, os.MkdirAll(digestDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(digestDir, "layer.erofs"), []byte("erofs"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(digestDir, "layer.ext4"), []byte("ext4"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(digestDir, "artifact.erofs.json"), []byte("record"), 0o644)) + + // In-progress temp dirs and records count toward actual layer-store usage. + unpackDir := filepath.Join(digestDir, ".unpack-tmp") + require.NoError(t, os.MkdirAll(unpackDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(unpackDir, "layer.bin"), []byte("unpacked-content"), 0o644)) + + total, err := totalLayerArtifactBytesFromFilesystemWithContext(context.Background(), layersDir) + require.NoError(t, err) + require.Equal(t, int64(len("erofs")+len("ext4")+len("record")+len("unpacked-content")), total) +} + func TestTotalReadyImageBytesFromMetadata_UsesRootfsFallbackForReadyImageWithoutSize(t *testing.T) { t.Parallel() @@ -73,7 +114,7 @@ func TestTotalReadyImageBytesFromMetadata_UsesRootfsFallbackForReadyImageWithout require.NoError(t, os.WriteFile(filepath.Join(digestDir, "metadata.json"), []byte(`{"status":"ready","size_bytes":0}`), 0o644)) require.NoError(t, os.WriteFile(filepath.Join(digestDir, "rootfs.erofs"), []byte("another-rootfs"), 0o644)) - total, err := totalReadyImageBytesFromMetadata(imagesDir) + total, err := totalReadyImageBytesFromMetadataWithContext(context.Background(), imagesDir) require.NoError(t, err) require.Equal(t, int64(len("another-rootfs")), total) } diff --git a/lib/images/image_store_lock_test.go b/lib/images/image_store_lock_test.go new file mode 100644 index 000000000..a2c130edc --- /dev/null +++ b/lib/images/image_store_lock_test.go @@ -0,0 +1,43 @@ +//go:build linux || darwin + +package images + +import ( + "os" + "os/exec" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestImageStoreLockIsExclusive(t *testing.T) { + imagesDir := t.TempDir() + first, err := acquireImageStoreLock(imagesDir) + require.NoError(t, err) + + cmd := exec.Command(os.Args[0], "-test.run=^TestImageStoreLockChild$") + cmd.Env = append(os.Environ(), "HYPEMAN_TEST_LOCK_DIR="+imagesDir) + require.NoError(t, cmd.Run()) + + second, err := acquireImageStoreLock(imagesDir) + require.NoError(t, err) + require.NoError(t, second.Close()) + require.NoError(t, first.Close()) + + third, err := acquireImageStoreLock(imagesDir) + require.NoError(t, err) + require.NoError(t, third.Close()) +} + +func TestImageStoreLockChild(t *testing.T) { + imagesDir := os.Getenv("HYPEMAN_TEST_LOCK_DIR") + if imagesDir == "" { + return + } + + lock, err := acquireImageStoreLock(imagesDir) + if err == nil { + _ = lock.Close() + t.Fatal("acquired image store lock held by another process") + } +} diff --git a/lib/images/image_store_lock_unix.go b/lib/images/image_store_lock_unix.go new file mode 100644 index 000000000..171996377 --- /dev/null +++ b/lib/images/image_store_lock_unix.go @@ -0,0 +1,89 @@ +//go:build linux || darwin + +package images + +import ( + "fmt" + "os" + "path/filepath" + "sync" + + "golang.org/x/sys/unix" +) + +type imageStoreLock struct { + file *os.File + path string + once sync.Once + err error +} + +type imageStoreLockState struct { + file *os.File + refs int +} + +var imageStoreLocks = struct { + sync.Mutex + byPath map[string]*imageStoreLockState +}{byPath: make(map[string]*imageStoreLockState)} + +func acquireImageStoreLock(imagesDir string) (*imageStoreLock, error) { + if err := os.MkdirAll(imagesDir, 0755); err != nil { + return nil, fmt.Errorf("create image store: %w", err) + } + lockPath, err := filepath.Abs(filepath.Join(imagesDir, ".lock")) + if err != nil { + return nil, fmt.Errorf("resolve image store lock: %w", err) + } + + imageStoreLocks.Lock() + defer imageStoreLocks.Unlock() + + state := imageStoreLocks.byPath[lockPath] + if state == nil { + file, err := os.OpenFile(lockPath, os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + return nil, fmt.Errorf("open image store lock: %w", err) + } + if err := unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB); err != nil { + _ = file.Close() + return nil, fmt.Errorf("image store is already in use: %w", err) + } + state = &imageStoreLockState{file: file} + imageStoreLocks.byPath[lockPath] = state + } + + fd, err := unix.FcntlInt(state.file.Fd(), unix.F_DUPFD_CLOEXEC, 0) + if err != nil { + if state.refs == 0 { + delete(imageStoreLocks.byPath, lockPath) + _ = state.file.Close() + } + return nil, fmt.Errorf("duplicate image store lock: %w", err) + } + state.refs++ + return &imageStoreLock{file: os.NewFile(uintptr(fd), lockPath), path: lockPath}, nil +} + +func (l *imageStoreLock) Close() error { + l.once.Do(func() { + if err := l.file.Close(); err != nil { + l.err = err + } + + imageStoreLocks.Lock() + state := imageStoreLocks.byPath[l.path] + if state != nil { + state.refs-- + if state.refs == 0 { + delete(imageStoreLocks.byPath, l.path) + if err := state.file.Close(); l.err == nil { + l.err = err + } + } + } + imageStoreLocks.Unlock() + }) + return l.err +} diff --git a/lib/images/layer_artifact.go b/lib/images/layer_artifact.go new file mode 100644 index 000000000..fb13f1b48 --- /dev/null +++ b/lib/images/layer_artifact.go @@ -0,0 +1,546 @@ +package images + +import ( + "compress/gzip" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "log/slog" + "os" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/kernel/hypeman/lib/paths" + "github.com/klauspost/compress/zstd" + v1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/opencontainers/umoci/oci/layer" + "golang.org/x/sys/unix" +) + +const ( + layerRecordSchemaVersion = 3 + maxLayerUnpackedBytes = 100 << 30 + + // layerBuildTimeout bounds a shared layer build end to end. Builds are + // detached from the initiating request's context, so this deadline is the + // only thing that can abort a hung unpack and free the singleflight key. + layerBuildTimeout = time.Hour +) + +var errCorruptLayerRecord = errors.New("corrupt layer record") + +// layerArtifact is the persisted record for one materialized layer artifact. +// The key is the compressed layer blob digest plus the artifact format. The +// encoding records the descriptor's decompression class so conflicting +// descriptors cannot overwrite one artifact path. +type layerArtifact struct { + SchemaVersion int `json:"schema_version"` + Digest string `json:"digest"` // compressed layer blob digest, sha256:... + Encoding string `json:"encoding"` + DiffID string `json:"diff_id,omitempty"` + Format string `json:"format"` + SizeBytes int64 `json:"size_bytes"` // artifact bytes on disk + ArtifactDigest string `json:"artifact_digest"` // sha256 of artifact bytes + UnpackedBytes int64 `json:"unpacked_bytes"` // decompressed tar stream bytes + CreatedAt time.Time `json:"created_at"` +} + +// validate checks a record read back from disk. The format fully determines +// the artifact options (erofs is always lz4-compressed, ext4 uncompressed), +// so only the format is stored. +func (a *layerArtifact) validate() error { + if a.SchemaVersion != layerRecordSchemaVersion { + return fmt.Errorf("unsupported schema version: %d", a.SchemaVersion) + } + if _, err := parseSHA256Digest(a.Digest); err != nil { + return fmt.Errorf("invalid digest: %w", err) + } + if _, err := parseSHA256Digest(a.ArtifactDigest); err != nil { + return fmt.Errorf("invalid artifact digest: %w", err) + } + if a.Format != string(FormatErofs) && a.Format != string(FormatExt4) { + return fmt.Errorf("invalid format: %s", a.Format) + } + if a.Encoding != "raw" && a.Encoding != "gzip" && a.Encoding != "zstd" { + return fmt.Errorf("invalid encoding: %s", a.Encoding) + } + if a.SizeBytes < 0 || a.UnpackedBytes < 0 { + return fmt.Errorf("invalid size") + } + return nil +} + +// matches decides whether the stored record satisfies a lookup. A descriptor +// without a DiffID (a manifest lookup that did not consult the image config) +// matches any record: the compressed digest content-addresses the blob, and +// unpackCachedLayer re-verifies the diff ID whenever one is supplied. +func (a *layerArtifact) matches(desc layerDescriptor) bool { + if a.Digest != desc.Digest || a.Format != layerArtifactFormat() || a.Encoding != layerEncoding(desc.MediaType) { + return false + } + return desc.DiffID == "" || a.DiffID == desc.DiffID +} + +func layerEncoding(mediaType string) string { + normalized := convertToOCIMediaType(mediaType) + switch { + case normalized == v1.MediaTypeImageLayerZstd, strings.HasSuffix(normalized, ".tar.zstd"): + return "zstd" + case normalized == v1.MediaTypeImageLayerGzip, strings.HasSuffix(normalized, ".tar.gzip"): + return "gzip" + default: + return "raw" + } +} + +func layerArtifactFormat() string { + switch DefaultImageFormat { + case FormatErofs, FormatExt4: + return string(DefaultImageFormat) + default: + return "" + } +} + +func layerArtifactPath(p *paths.Paths, layerHex string) string { + return p.ImageLayerArtifactForFormat(layerHex, layerArtifactFormat()) +} + +func layerArtifactRecordPath(p *paths.Paths, layerHex string) string { + return p.ImageLayerRecordForFormat(layerHex, layerArtifactFormat()) +} + +func layerDigestHex(value string) (string, error) { + digest, err := parseSHA256Digest(value) + if err != nil || digest.String() != value { + return "", fmt.Errorf("invalid layer digest %q", value) + } + return digest.Encoded(), nil +} + +// layerMapOptions preserves tar ownership when running as root. Otherwise +// umoci's rootless mode skips chown and stands in empty files for device nodes. +// Unlike unpackLayers in oci.go, which maps container root to the current +// user, this deliberately leaves ownership untouched as root: artifacts must +// keep the layer's on-disk ownership for later stacking. +func layerMapOptions() layer.MapOptions { + return layer.MapOptions{Rootless: os.Geteuid() != 0} +} + +// supportsLayerArtifacts reports whether the host can create the native +// overlayfs representation used by per-layer artifacts. Other platforms and +// rootless processes must compose the OCI layers into one rootfs instead. +func supportsLayerArtifacts() bool { + return runtime.GOOS == "linux" && os.Geteuid() == 0 +} + +func probeLayerArtifactSupport(layersDir string) bool { + if !supportsLayerArtifacts() || os.MkdirAll(layersDir, 0755) != nil { + return false + } + probeDir, err := os.MkdirTemp(layersDir, ".probe-*") + if err != nil { + return false + } + defer os.RemoveAll(probeDir) + + whiteout := filepath.Join(probeDir, "whiteout") + if err := unix.Mknod(whiteout, unix.S_IFCHR|0600, int(unix.Mkdev(0, 0))); err != nil { + return false + } + if err := unix.Lsetxattr(probeDir, "trusted.overlay.opaque", []byte("y"), 0); err != nil { + return false + } + value := make([]byte, 1) + n, err := unix.Lgetxattr(probeDir, "trusted.overlay.opaque", value) + return err == nil && n == 1 && value[0] == 'y' +} + +func (m *manager) layerArtifactSupport() bool { + return m.layers.artifactsSupported +} + +func (m *manager) materializeLayerArtifact(ctx context.Context, desc layerDescriptor) (*layerArtifact, error) { + return m.layers.materializeLayerArtifact(ctx, desc) +} + +// layerArtifactOnDiskFormat is the extraction format for per-layer artifacts. +// Whiteouts become overlayfs whiteout inodes and opaque xattrs, which an +// overlayfs mount of stacked layers understands, so the artifact retains the +// layer's deletions without a private marker format. Callers must check +// supportsLayerArtifacts before materializing this format. +func layerArtifactOnDiskFormat() layer.OnDiskFormat { + return layer.OverlayfsRootfs{MapOptions: layerMapOptions()} +} + +// readLayerRecord loads the artifact record for a layer digest, if present. +// A missing record returns (nil, nil): the layer simply was never +// materialized. +func readLayerRecord(p *paths.Paths, layerHex string) (*layerArtifact, error) { + recordPath := layerArtifactRecordPath(p, layerHex) + info, err := os.Lstat(recordPath) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("stat layer record: %w", err) + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("%w: layer record is not a regular file", errCorruptLayerRecord) + } + data, err := os.ReadFile(recordPath) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("read layer record: %w", err) + } + var record layerArtifact + if err := json.Unmarshal(data, &record); err != nil { + return nil, fmt.Errorf("%w: unmarshal layer record: %v", errCorruptLayerRecord, err) + } + if err := record.validate(); err != nil { + return nil, fmt.Errorf("%w: invalid layer record: %v", errCorruptLayerRecord, err) + } + return &record, nil +} + +func discardLayerCache(p *paths.Paths, layerHex string) error { + for _, path := range []string{ + layerArtifactRecordPath(p, layerHex), + layerArtifactPath(p, layerHex), + } { + if err := removePath(path); err != nil && !os.IsNotExist(err) { + return err + } + } + return nil +} + +func (s *layerStore) clearLayerCache(layerHex string) error { + err := discardLayerCache(s.paths, layerHex) + s.invalidateDiskUsageTotals() + return err +} + +// materializeLayerArtifact ensures a layer has a materialized artifact keyed +// by its blob digest, building it from the shared OCI cache blob when absent. +// The layer is unpacked into an isolated temp directory, converted to the +// default image format, and installed atomically. Normal failures remove the +// temp directory; a crash mid-build can leave a stale .unpack-* directory +// behind, which reconciliation landing with the pull integration is expected +// to sweep. No production caller yet: pull integration and +// composition land in later changes. +// +// Concurrent callers share one build. The build itself is detached from the +// initiating caller's cancellation so one cancelled pull cannot fail every +// other pull waiting on the same layer; each caller still returns as soon as +// its own context is done. +func (s *layerStore) materializeLayerArtifact(ctx context.Context, desc layerDescriptor) (*layerArtifact, error) { + layerHex, err := layerDigestHex(desc.Digest) + if err != nil { + return nil, err + } + key := desc.Digest + "\x00" + layerArtifactFormat() + result := s.flights.DoChan(key, func() (any, error) { + // The build outlives the initiating request, so this deadline is its + // only bound: without it a hung cache-blob read would wedge the + // singleflight key, and every future caller for the layer, forever. + buildCtx, cancel := context.WithTimeout(context.Background(), layerBuildTimeout) + defer cancel() + if s.builds != nil { + select { + case s.builds <- struct{}{}: + defer func() { <-s.builds }() + case <-buildCtx.Done(): + return nil, buildCtx.Err() + } + } + return s.materializeLayerArtifactOnce(buildCtx, desc, layerHex) + }) + select { + case <-ctx.Done(): + return nil, ctx.Err() + case shared := <-result: + if shared.Err != nil { + return nil, shared.Err + } + artifact := shared.Val.(*layerArtifact) + if !artifact.matches(desc) { + return nil, fmt.Errorf("layer artifact metadata does not match descriptor %s", desc.Digest) + } + return artifact, nil + } +} + +func (s *layerStore) materializeLayerArtifactOnce(ctx context.Context, desc layerDescriptor, layerHex string) (*layerArtifact, error) { + if layerArtifactFormat() == "" { + return nil, fmt.Errorf("unsupported layer artifact format: %s", DefaultImageFormat) + } + if !s.artifactsSupported { + return nil, fmt.Errorf("layer artifacts are unsupported on this host") + } + + if record, err := readLayerRecord(s.paths, layerHex); err != nil { + if !errors.Is(err, errCorruptLayerRecord) { + return nil, err + } + if discardErr := s.clearLayerCache(layerHex); discardErr != nil { + return nil, fmt.Errorf("discard corrupt layer cache: %w", discardErr) + } + } else if record != nil && record.Digest == desc.Digest && record.Format == layerArtifactFormat() && record.Encoding != layerEncoding(desc.MediaType) { + return nil, fmt.Errorf("layer artifact encoding does not match descriptor %s", desc.Digest) + } else if record != nil && record.matches(desc) { + artifactInfo, statErr := os.Lstat(layerArtifactPath(s.paths, layerHex)) + if statErr == nil && artifactInfo.Mode().IsRegular() && record.SizeBytes > 0 && artifactInfo.Size() == record.SizeBytes { + artifactDigest, digestErr := fileSHA256(ctx, layerArtifactPath(s.paths, layerHex)) + if digestErr == nil && artifactDigest == record.ArtifactDigest { + return record, nil + } + } + if discardErr := s.clearLayerCache(layerHex); discardErr != nil { + return nil, fmt.Errorf("discard invalid layer cache: %w", discardErr) + } + } + + artifactPath := layerArtifactPath(s.paths, layerHex) + if artifactInfo, statErr := os.Lstat(artifactPath); statErr == nil && !artifactInfo.Mode().IsRegular() { + if discardErr := s.clearLayerCache(layerHex); discardErr != nil { + return nil, fmt.Errorf("discard invalid layer artifact: %w", discardErr) + } + } + + layerDir := s.paths.ImageLayerDir(layerHex) + if err := os.MkdirAll(layerDir, 0755); err != nil { + return nil, fmt.Errorf("create layer directory: %w", err) + } + unpackDir, err := os.MkdirTemp(layerDir, ".unpack-*") + if err != nil { + return nil, fmt.Errorf("create unpack directory: %w", err) + } + s.beginLayerBuild() + defer func() { + if err := removePath(unpackDir); err != nil { + slog.Warn("failed to remove layer unpack directory", "dir", unpackDir, "error", err) + } + s.endLayerBuild() + }() + + stats, err := unpackCachedLayer(ctx, s.paths, desc, unpackDir, layerArtifactOnDiskFormat()) + if err != nil { + return nil, err + } + record, err := s.installLayerArtifact(ctx, desc, layerHex, unpackDir, stats) + if err != nil { + s.invalidateDiskUsageTotals() + return nil, err + } + s.invalidateDiskUsageTotals() + return record, nil +} + +func (s *layerStore) installLayerArtifact(ctx context.Context, desc layerDescriptor, layerHex, unpackDir string, stats *unpackStats) (*layerArtifact, error) { + record := &layerArtifact{ + SchemaVersion: layerRecordSchemaVersion, + Digest: desc.Digest, + Encoding: layerEncoding(desc.MediaType), + DiffID: stats.diffID, + Format: layerArtifactFormat(), + UnpackedBytes: stats.unpackedBytes, + CreatedAt: time.Now(), + } + + if err := installAtomically(layerArtifactPath(s.paths, layerHex), func(path string) error { + size, err := ExportRootfsWithContext(ctx, unpackDir, path, DefaultImageFormat) + if err != nil { + return err + } + record.SizeBytes = size + record.ArtifactDigest, err = fileSHA256(ctx, path) + return err + }); err != nil { + return nil, fmt.Errorf("install layer artifact %s: %w", desc.Digest, err) + } + + data, err := json.MarshalIndent(record, "", " ") + if err != nil { + return nil, fmt.Errorf("marshal layer record: %w", err) + } + if err := writeJSONAtomic(layerArtifactRecordPath(s.paths, layerHex), data); err != nil { + _ = os.Remove(layerArtifactPath(s.paths, layerHex)) + return nil, fmt.Errorf("write layer record: %w", err) + } + return record, nil +} + +type unpackStats struct { + unpackedBytes int64 + diffID string + blobDigest string // sha256 of the compressed bytes as read +} + +func fileSHA256(ctx context.Context, path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", err + } + defer file.Close() + + hash := sha256.New() + if _, err := io.Copy(hash, contextReader{ctx: ctx, reader: file}); err != nil { + return "", err + } + return fmt.Sprintf("sha256:%x", hash.Sum(nil)), nil +} + +// contextReader fails reads once ctx is done so a cancelled caller stops a +// long extraction instead of running it to completion. +type contextReader struct { + ctx context.Context + reader io.Reader +} + +func (r contextReader) Read(p []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.reader.Read(p) +} + +// unpackCachedLayer locates desc's blob in the shared OCI cache, unpacks it +// into dest, and verifies both the blob digest and the diff ID when the +// descriptor carries one. The caller must have validated desc.Digest. +func unpackCachedLayer(ctx context.Context, p *paths.Paths, desc layerDescriptor, dest string, onDisk layer.OnDiskFormat) (*unpackStats, error) { + blobPath := p.OCICacheBlob(strings.TrimPrefix(desc.Digest, "sha256:")) + if _, err := os.Stat(blobPath); err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("layer blob missing from oci cache: %s", desc.Digest) + } + return nil, fmt.Errorf("stat layer blob: %w", err) + } + stats, err := unpackLayerBlob(ctx, blobPath, desc.MediaType, dest, onDisk) + if err != nil { + return nil, fmt.Errorf("unpack layer %s: %w", desc.Digest, err) + } + if desc.Digest != "" && stats.blobDigest != desc.Digest { + return nil, fmt.Errorf("layer blob digest mismatch: got %s, want %s", stats.blobDigest, desc.Digest) + } + if desc.DiffID != "" && stats.diffID != desc.DiffID { + return nil, fmt.Errorf("layer %s diff id mismatch: got %s, want %s", desc.Digest, stats.diffID, desc.DiffID) + } + return stats, nil +} + +// unpackLayerBlob extracts one compressed layer blob into dest with umoci, +// which confines every entry to dest and interprets whiteouts per onDisk. +// The compressed stream is hashed to verify the blob digest, the decompressed +// stream for the diff ID, and capped in size. +func unpackLayerBlob(ctx context.Context, blobPath, mediaType, dest string, onDisk layer.OnDiskFormat) (*unpackStats, error) { + if err := os.MkdirAll(dest, 0755); err != nil { + return nil, fmt.Errorf("create extraction root: %w", err) + } + blob, err := os.Open(blobPath) + if err != nil { + return nil, fmt.Errorf("open blob: %w", err) + } + defer blob.Close() + + blobHash := sha256.New() + compressed := io.TeeReader(contextReader{ctx: ctx, reader: blob}, blobHash) + reader, closer, err := decompressLayer(compressed, mediaType) + if err != nil { + return nil, err + } + defer closer.Close() + + hash := sha256.New() + limited := &io.LimitedReader{R: contextReader{ctx: ctx, reader: reader}, N: maxLayerUnpackedBytes + 1} + hashed := io.TeeReader(limited, hash) + if err := layer.UnpackLayer(dest, hashed, &layer.UnpackOptions{OnDiskFormat: onDisk}); err != nil { + return nil, err + } + if _, err := io.Copy(io.Discard, hashed); err != nil { + return nil, fmt.Errorf("drain layer: %w", err) + } + if _, err := io.Copy(io.Discard, compressed); err != nil { + return nil, fmt.Errorf("drain blob: %w", err) + } + if limited.N == 0 { + return nil, fmt.Errorf("layer exceeds maximum unpacked size of %d bytes", maxLayerUnpackedBytes) + } + return &unpackStats{ + unpackedBytes: maxLayerUnpackedBytes + 1 - limited.N, + diffID: fmt.Sprintf("sha256:%x", hash.Sum(nil)), + blobDigest: fmt.Sprintf("sha256:%x", blobHash.Sum(nil)), + }, nil +} + +// decompressLayer wraps the blob in the reader for its layer media type. Both +// OCI-style suffixes (+gzip, +zstd) and docker-style media types (tar.gzip, +// tar.zstd) are matched so neither encoding falls through to the raw path. +func decompressLayer(r io.Reader, mediaType string) (io.Reader, io.Closer, error) { + switch layerEncoding(mediaType) { + case "zstd": + decoder, err := zstd.NewReader(r) + if err != nil { + return nil, nil, fmt.Errorf("zstd reader: %w", err) + } + decodeCloser := decoder.IOReadCloser() + return decodeCloser, decodeCloser, nil + case "gzip": + gz, err := gzip.NewReader(r) + if err != nil { + return nil, nil, fmt.Errorf("gzip reader: %w", err) + } + return gz, gz, nil + default: + return r, io.NopCloser(r), nil + } +} + +// removePath removes a tree that may contain read-only directories restored +// from layer metadata. +func removePath(path string) error { + if err := os.RemoveAll(path); err == nil || os.IsNotExist(err) { + return nil + } + if err := makeTreeWritable(path); err != nil && !os.IsNotExist(err) { + return err + } + if err := os.RemoveAll(path); err != nil && !os.IsNotExist(err) { + return err + } + return nil +} + +func makeTreeWritable(path string) error { + info, err := os.Lstat(path) + if err != nil { + return err + } + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return nil + } + if err := os.Chmod(path, info.Mode().Perm()|0700); err != nil { + return err + } + return filepath.WalkDir(path, func(p string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() && entry.Type()&os.ModeSymlink == 0 { + info, err := entry.Info() + if err != nil { + return err + } + return os.Chmod(p, info.Mode().Perm()|0700) + } + return nil + }) +} diff --git a/lib/images/layer_artifact_smoke_test.go b/lib/images/layer_artifact_smoke_test.go new file mode 100644 index 000000000..2ab96ddff --- /dev/null +++ b/lib/images/layer_artifact_smoke_test.go @@ -0,0 +1,172 @@ +package images + +import ( + "archive/tar" + "context" + "os" + "os/exec" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +// Smoke tests for layer-artifact edge cases beyond the standard suite. +// Media types, path validation, cache-miss rebuilds, singleflight sharing, +// device/fifo entries (root only), and disk-usage accounting. + +const ( + ociZstdMediaType = "application/vnd.oci.image.layer.v1.tar+zstd" + dockerZstdMediaType = "application/vnd.docker.image.rootfs.diff.tar.zstd" + dockerGzipMediaType = "application/vnd.docker.image.rootfs.diff.tar.gzip" + ociPlainTarMediaType = "application/vnd.oci.image.layer.v1.tar" +) + +func TestSmokeUnpackZstdMediaTypes(t *testing.T) { + root := t.TempDir() + for name, tc := range map[string]struct { + mediaType string + blob string + }{ + "oci_suffix_zstd": {ociZstdMediaType, writeZstdTarLayer(t, root, "oci.tar.zst", fileEntry("f.txt", "zstd"))}, + "docker_tar_zstd": {dockerZstdMediaType, writeZstdTarLayer(t, root, "docker.tar.zstd", fileEntry("f.txt", "zstd"))}, + "docker_tar_gzip": {dockerGzipMediaType, writeLayerBlob(t, root, "docker.tar.gz", fileEntry("f.txt", "gzip"))}, + "plain_tar": {ociPlainTarMediaType, writeRawTarLayer(t, root, "plain.tar", fileEntry("f.txt", "raw"))}, + "unknown_media_raw": {"application/vnd.custom.tar", writeRawTarLayer(t, root, "custom.tar", fileEntry("f.txt", "raw"))}, + } { + t.Run(name, func(t *testing.T) { + dest := filepath.Join(root, t.Name()) + stats, err := unpackLayerBlob(context.Background(), tc.blob, tc.mediaType, dest, composeOnDiskFormat()) + require.NoError(t, err) + require.Greater(t, stats.unpackedBytes, int64(0)) + require.FileExists(t, filepath.Join(dest, "f.txt")) + }) + } +} + +func TestSmokeRejectsBadLayerHex(t *testing.T) { + p := paths.New(t.TempDir()) + m := testLayerArtifactManager(p) + for _, digest := range []string{ + "sha256:..", + "sha256:../escape", + "sha256:a/b", + "sha256:", + } { + _, err := m.materializeLayerArtifact(context.Background(), layerDescriptor{ + Digest: digest, + MediaType: testTarGzMediaType, + }) + require.ErrorContains(t, err, "invalid layer digest", "digest %q must be rejected", digest) + } + require.NoDirExists(t, filepath.Join(p.ImageLayersDir(), "..", "layers-escape")) + + _, err := m.materializeLayerArtifact(context.Background(), layerDescriptor{ + Digest: "notadigest", + MediaType: testTarGzMediaType, + }) + require.ErrorContains(t, err, "invalid layer digest") +} + +func TestSmokeRebuildWhenArtifactFileMissing(t *testing.T) { + if _, err := exec.LookPath("mkfs.ext4"); err != nil { + t.Skip("mkfs.ext4 not available") + } + originalFormat := DefaultImageFormat + DefaultImageFormat = FormatExt4 + t.Cleanup(func() { DefaultImageFormat = originalFormat }) + + p := paths.New(t.TempDir()) + img, err := mutate.AppendLayers(empty.Image, syntheticLayer(t, "f.txt", "content")) + require.NoError(t, err) + writeLayerTestLayout(t, p, img) + desc := layerDescFromImage(t, img, 0) + m := testLayerArtifactManager(p) + + first, err := m.materializeLayerArtifact(context.Background(), desc) + require.NoError(t, err) + layerHex := desc.Digest[len("sha256:"):] + + // Record survives but the artifact file is gone: must rebuild, not reuse. + require.NoError(t, os.Remove(p.ImageLayerArtifactForFormat(layerHex, layerArtifactFormat()))) + second, err := m.materializeLayerArtifact(context.Background(), desc) + require.NoError(t, err) + require.True(t, second.CreatedAt.After(first.CreatedAt), "missing artifact must trigger a rebuild") + require.FileExists(t, p.ImageLayerArtifactForFormat(layerHex, layerArtifactFormat())) +} + +func TestSmokeConcurrentMaterializeSharesFlight(t *testing.T) { + if _, err := exec.LookPath("mkfs.ext4"); err != nil { + t.Skip("mkfs.ext4 not available") + } + originalFormat := DefaultImageFormat + DefaultImageFormat = FormatExt4 + t.Cleanup(func() { DefaultImageFormat = originalFormat }) + + p := paths.New(t.TempDir()) + img, err := mutate.AppendLayers(empty.Image, syntheticLayer(t, "f.txt", "shared")) + require.NoError(t, err) + writeLayerTestLayout(t, p, img) + desc := layerDescFromImage(t, img, 0) + m := testLayerArtifactManager(p) + + const callers = 8 + var wg sync.WaitGroup + records := make([]*layerArtifact, callers) + errs := make([]error, callers) + start := time.Now() + for i := 0; i < callers; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + records[i], errs[i] = m.materializeLayerArtifact(context.Background(), desc) + }(i) + } + wg.Wait() + require.NotEmpty(t, errs) + for i := range errs { + require.NoError(t, errs[i]) + require.Equal(t, records[0], records[i], "all callers must share one build result") + } + t.Logf("8 callers finished in %s sharing one flight", time.Since(start)) + layerHex := desc.Digest[len("sha256:"):] + require.FileExists(t, p.ImageLayerArtifactForFormat(layerHex, layerArtifactFormat())) +} + +func TestSmokeDeviceAndFifoEntries(t *testing.T) { + if !probeLayerArtifactSupport(t.TempDir()) { + t.Skip("device and fifo tar entries need mknod") + } + root := t.TempDir() + blob := writeRawTarLayer(t, root, "devices.tar", + fileEntry("regular.txt", "x"), + func(t *testing.T, tw *tar.Writer) { + require.NoError(t, tw.WriteHeader(&tar.Header{ + Name: "dev/null", Typeflag: tar.TypeChar, + Devmajor: 1, Devminor: 3, Mode: 0666, + })) + }, + func(t *testing.T, tw *tar.Writer) { + require.NoError(t, tw.WriteHeader(&tar.Header{ + Name: "pipes/fifo", Typeflag: tar.TypeFifo, Mode: 0644, + })) + }, + ) + dest := filepath.Join(root, "dest") + _, err := unpackLayerBlob(context.Background(), blob, ociPlainTarMediaType, dest, composeOnDiskFormat()) + require.NoError(t, err) + + require.FileExists(t, filepath.Join(dest, "regular.txt")) + var stat unix.Stat_t + require.NoError(t, unix.Lstat(filepath.Join(dest, "dev", "null"), &stat)) + require.Equal(t, uint32(unix.S_IFCHR), stat.Mode&unix.S_IFMT) + require.Equal(t, uint64(0x103), uint64(stat.Rdev), "char device 1:3") + require.NoError(t, unix.Lstat(filepath.Join(dest, "pipes", "fifo"), &stat)) + require.Equal(t, uint32(unix.S_IFIFO), stat.Mode&unix.S_IFMT) +} diff --git a/lib/images/layer_artifact_test.go b/lib/images/layer_artifact_test.go new file mode 100644 index 000000000..596c44196 --- /dev/null +++ b/lib/images/layer_artifact_test.go @@ -0,0 +1,537 @@ +package images + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + gcr "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/layout" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/kernel/hypeman/lib/paths" + "github.com/klauspost/compress/zstd" + "github.com/opencontainers/umoci/oci/layer" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +// whiteoutPrefix marks OCI whiteout entries (".wh." and ".wh..wh..opq"). +// umoci interprets them during extraction: composeOnDiskFormat applies them +// against the tree being composed, layerArtifactOnDiskFormat converts them to +// overlayfs whiteout inodes and opaque xattrs so per-layer artifacts can +// later be stacked. +const whiteoutPrefix = ".wh." + +// composeOnDiskFormat applies whiteouts against the tree being composed. It +// belongs to the composition flow and moves to production with that change. +func composeOnDiskFormat() layer.OnDiskFormat { + return layer.DirRootfs{MapOptions: layerMapOptions()} +} + +const testTarGzMediaType = "application/vnd.oci.image.layer.v1.tar+gzip" + +// writeLayerTestLayout writes img into the shared OCI cache of p tagged with +// the image's digest, mirroring pullToOCILayout. +func testLayerArtifactManager(p *paths.Paths) *manager { + store := newLayerStore(p, 1) + store.artifactsSupported = true + return &manager{paths: p, layers: store} +} + +func writeLayerTestLayout(t *testing.T, p *paths.Paths, img gcr.Image) { + t.Helper() + digest, err := img.Digest() + require.NoError(t, err) + layoutPath, err := layout.Write(p.SystemOCICache(), empty.Index) + require.NoError(t, err) + require.NoError(t, layoutPath.AppendImage(img, layout.WithAnnotations(map[string]string{ + "org.opencontainers.image.ref.name": digestToLayoutTag(digest.String()), + }))) +} + +func layerDescFromImage(t *testing.T, img gcr.Image, index int) layerDescriptor { + t.Helper() + manifest, err := img.Manifest() + require.NoError(t, err) + configFile, err := img.ConfigFile() + require.NoError(t, err) + layer := manifest.Layers[index] + return layerDescriptor{ + Digest: layer.Digest.String(), + Size: layer.Size, + MediaType: string(layer.MediaType), + DiffID: configFile.RootFS.DiffIDs[index].String(), + } +} + +// tarGz builds a gzipped tar from the entries written by fn. +func tarGz(t *testing.T, fn func(tw *tar.Writer)) []byte { + t.Helper() + var buf bytes.Buffer + gzw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gzw) + fn(tw) + require.NoError(t, tw.Close()) + require.NoError(t, gzw.Close()) + return buf.Bytes() +} + +func writeTarEntry(t *testing.T, tw *tar.Writer, header *tar.Header, content string) { + t.Helper() + if header.Typeflag == tar.TypeReg { + header.Size = int64(len(content)) + } + require.NoError(t, tw.WriteHeader(header)) + if content != "" { + _, err := tw.Write([]byte(content)) + require.NoError(t, err) + } +} + +func fileEntry(name, content string) func(*testing.T, *tar.Writer) { + return func(t *testing.T, tw *tar.Writer) { + writeTarEntry(t, tw, &tar.Header{Name: name, Typeflag: tar.TypeReg, Mode: 0644}, content) + } +} + +func dirEntry(name string) func(*testing.T, *tar.Writer) { + return func(t *testing.T, tw *tar.Writer) { + writeTarEntry(t, tw, &tar.Header{Name: name, Typeflag: tar.TypeDir, Mode: 0755}, "") + } +} + +func symlinkEntry(name, target string) func(*testing.T, *tar.Writer) { + return func(t *testing.T, tw *tar.Writer) { + writeTarEntry(t, tw, &tar.Header{Name: name, Typeflag: tar.TypeSymlink, Linkname: target}, "") + } +} + +func hardlinkEntry(name, target string) func(*testing.T, *tar.Writer) { + return func(t *testing.T, tw *tar.Writer) { + writeTarEntry(t, tw, &tar.Header{Name: name, Typeflag: tar.TypeLink, Linkname: target}, "") + } +} + +// writeLayerBlob writes a gzipped tar layer to a file and returns its path. +func writeLayerBlob(t *testing.T, dir, name string, entries ...func(*testing.T, *tar.Writer)) string { + t.Helper() + data := tarGz(t, func(tw *tar.Writer) { + for _, entry := range entries { + entry(t, tw) + } + }) + path := filepath.Join(dir, name) + require.NoError(t, os.WriteFile(path, data, 0644)) + return path +} + +func writeRawTarLayer(t *testing.T, dir, name string, entries ...func(*testing.T, *tar.Writer)) string { + t.Helper() + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + for _, entry := range entries { + entry(t, tw) + } + require.NoError(t, tw.Close()) + path := filepath.Join(dir, name) + require.NoError(t, os.WriteFile(path, buf.Bytes(), 0644)) + return path +} + +func writeZstdTarLayer(t *testing.T, dir, name string, entries ...func(*testing.T, *tar.Writer)) string { + t.Helper() + var tarBuf bytes.Buffer + tw := tar.NewWriter(&tarBuf) + for _, entry := range entries { + entry(t, tw) + } + require.NoError(t, tw.Close()) + var zstBuf bytes.Buffer + zw, err := zstd.NewWriter(&zstBuf) + require.NoError(t, err) + _, err = zw.Write(tarBuf.Bytes()) + require.NoError(t, err) + require.NoError(t, zw.Close()) + path := filepath.Join(dir, name) + require.NoError(t, os.WriteFile(path, zstBuf.Bytes(), 0644)) + return path +} + +// unpackInto applies a layer blob onto dest with compose semantics. +func unpackInto(t *testing.T, blobPath, dest string) *unpackStats { + t.Helper() + stats, err := unpackLayerBlob(context.Background(), blobPath, testTarGzMediaType, dest, composeOnDiskFormat()) + require.NoError(t, err) + return stats +} + +func requireNoWhiteoutMarkers(t *testing.T, root string) { + t.Helper() + require.NoError(t, filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { + require.NoError(t, err) + require.NotContains(t, entry.Name(), whiteoutPrefix, "whiteout marker leaked into tree") + return nil + })) +} + +func TestMaterializeLayerArtifact(t *testing.T) { + if _, err := exec.LookPath("mkfs.erofs"); err != nil { + t.Skip("mkfs.erofs not available") + } + + p := paths.New(t.TempDir()) + img, err := mutate.AppendLayers(empty.Image, syntheticLayer(t, "base.txt", "base layer content")) + require.NoError(t, err) + writeLayerTestLayout(t, p, img) + + desc := layerDescFromImage(t, img, 0) + m := testLayerArtifactManager(p) + + record, err := m.materializeLayerArtifact(context.Background(), desc) + require.NoError(t, err) + require.Equal(t, desc.Digest, record.Digest) + require.Equal(t, desc.DiffID, record.DiffID) + require.Equal(t, layerArtifactFormat(), record.Format) + require.Greater(t, record.SizeBytes, int64(0)) + require.Greater(t, record.UnpackedBytes, int64(0)) + + layerHex := desc.Digest[len("sha256:"):] + artifactPath := p.ImageLayerArtifactForFormat(layerHex, layerArtifactFormat()) + artifactInfo, err := os.Stat(artifactPath) + require.NoError(t, err, "layer.erofs must be installed") + + // A second materialization reuses the existing artifact. + reused, err := m.materializeLayerArtifact(context.Background(), desc) + require.NoError(t, err) + require.True(t, record.CreatedAt.Equal(reused.CreatedAt), "reuse must return the stored record") + artifactInfoAfter, err := os.Stat(p.ImageLayerArtifactForFormat(layerHex, layerArtifactFormat())) + require.NoError(t, err) + require.Equal(t, artifactInfo.ModTime(), artifactInfoAfter.ModTime(), "reuse must not rebuild") + + // A truncated artifact must not be reused just because its record is valid. + require.NoError(t, os.Truncate(artifactPath, 0)) + rebuilt, err := m.materializeLayerArtifact(context.Background(), desc) + require.NoError(t, err) + require.NotEqual(t, reused.CreatedAt, rebuilt.CreatedAt, "invalid artifact must rebuild") + rebuiltInfo, err := os.Stat(artifactPath) + require.NoError(t, err) + require.Equal(t, rebuilt.SizeBytes, rebuiltInfo.Size()) + + // A same-size mutation must not be reused. + file, err := os.OpenFile(artifactPath, os.O_WRONLY, 0) + require.NoError(t, err) + _, err = file.WriteAt([]byte{0}, 0) + require.NoError(t, err) + require.NoError(t, file.Close()) + rebuiltAgain, err := m.materializeLayerArtifact(context.Background(), desc) + require.NoError(t, err) + require.NotEqual(t, rebuilt.CreatedAt, rebuiltAgain.CreatedAt, "checksum mismatch must rebuild") +} + +func TestMaterializeLayerArtifactRecoversCorruptRecord(t *testing.T) { + if _, err := exec.LookPath("mkfs.ext4"); err != nil { + t.Skip("mkfs.ext4 not available") + } + originalFormat := DefaultImageFormat + DefaultImageFormat = FormatExt4 + t.Cleanup(func() { DefaultImageFormat = originalFormat }) + + p := paths.New(t.TempDir()) + img, err := mutate.AppendLayers(empty.Image, syntheticLayer(t, "base.txt", "base layer content")) + require.NoError(t, err) + writeLayerTestLayout(t, p, img) + + desc := layerDescFromImage(t, img, 0) + m := testLayerArtifactManager(p) + first, err := m.materializeLayerArtifact(context.Background(), desc) + require.NoError(t, err) + + layerHex := desc.Digest[len("sha256:"):] + require.NoError(t, os.WriteFile( + p.ImageLayerRecordForFormat(layerHex, layerArtifactFormat()), + []byte("{not valid json"), + 0600, + )) + + second, err := m.materializeLayerArtifact(context.Background(), desc) + require.NoError(t, err) + require.NotEqual(t, first.CreatedAt, second.CreatedAt, "corrupt record must trigger a rebuild") + require.FileExists(t, p.ImageLayerArtifactForFormat(layerHex, layerArtifactFormat())) + + data, err := os.ReadFile(p.ImageLayerRecordForFormat(layerHex, layerArtifactFormat())) + require.NoError(t, err) + var record layerArtifact + require.NoError(t, json.Unmarshal(data, &record)) + require.NoError(t, record.validate()) +} + +// TestMaterializeLayerArtifactOutlivesCancelledCaller checks that a caller +// abandoning a shared build does not abort the build for everyone else: the +// artifact still lands even though the initiating context was cancelled. +func TestMaterializeLayerArtifactOutlivesCancelledCaller(t *testing.T) { + if _, err := exec.LookPath("mkfs.ext4"); err != nil { + t.Skip("mkfs.ext4 not available") + } + originalFormat := DefaultImageFormat + DefaultImageFormat = FormatExt4 + t.Cleanup(func() { DefaultImageFormat = originalFormat }) + + p := paths.New(t.TempDir()) + img, err := mutate.AppendLayers(empty.Image, syntheticLayer(t, "base.txt", "base layer content")) + require.NoError(t, err) + writeLayerTestLayout(t, p, img) + desc := layerDescFromImage(t, img, 0) + m := testLayerArtifactManager(p) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + // The build usually loses the race against the cancelled context, but a + // fast build can finish first; either outcome is valid when detached. + if _, buildErr := m.materializeLayerArtifact(ctx, desc); buildErr != nil { + require.ErrorIs(t, buildErr, context.Canceled) + } + + layerHex := desc.Digest[len("sha256:"):] + require.Eventually(t, func() bool { + record, err := readLayerRecord(p, layerHex) + return err == nil && record != nil + }, 30*time.Second, 50*time.Millisecond, "detached build must still install the artifact") + require.FileExists(t, p.ImageLayerArtifactForFormat(layerHex, layerArtifactFormat())) +} + +func TestMaterializeLayerArtifactRejectsUnsupportedHost(t *testing.T) { + p := paths.New(t.TempDir()) + store := newLayerStore(p, 1) + store.artifactsSupported = false + m := &manager{paths: p, layers: store} + _, err := m.materializeLayerArtifact(context.Background(), layerDescriptor{ + Digest: "sha256:" + strings.Repeat("ab", 32), + MediaType: testTarGzMediaType, + }) + require.ErrorContains(t, err, "unsupported on this host") +} + +func TestMaterializeLayerArtifactMissingBlob(t *testing.T) { + p := paths.New(t.TempDir()) + m := testLayerArtifactManager(p) + + _, err := m.materializeLayerArtifact(context.Background(), layerDescriptor{ + Digest: "sha256:abababababababababababababababababababababababababababababababab", + MediaType: testTarGzMediaType, + }) + require.ErrorContains(t, err, "missing from oci cache") +} + +// TestUnpackLayerBlobArtifactFormatKeepsWhiteouts checks that the artifact +// extraction format records deletions in overlayfs form: a 0:0 character +// device for a whiteout and the opaque xattr for an opaque directory. +func TestUnpackLayerBlobArtifactFormatKeepsWhiteouts(t *testing.T) { + if !probeLayerArtifactSupport(t.TempDir()) { + t.Skip("overlayfs whiteouts need mknod and trusted xattrs") + } + root := t.TempDir() + blob := writeLayerBlob(t, root, "layer.tar.gz", + fileEntry("keep.txt", "keep"), + dirEntry("gone/"), + fileEntry("gone/.wh.deleted.txt", ""), + dirEntry("opq/"), + fileEntry("opq/.wh..wh..opq", ""), + fileEntry("opq/fresh.txt", "fresh"), + ) + dest := filepath.Join(root, "dest") + _, err := unpackLayerBlob(context.Background(), blob, testTarGzMediaType, dest, layerArtifactOnDiskFormat()) + require.NoError(t, err) + + var stat unix.Stat_t + require.NoError(t, unix.Lstat(filepath.Join(dest, "gone", "deleted.txt"), &stat)) + require.Equal(t, uint32(unix.S_IFCHR), stat.Mode&unix.S_IFMT, "whiteout must be a character device") + require.Equal(t, uint64(0), uint64(stat.Rdev), "whiteout device must be 0:0") + + value := make([]byte, 8) + n, err := unix.Lgetxattr(filepath.Join(dest, "opq"), "trusted.overlay.opaque", value) + require.NoError(t, err) + require.Equal(t, "y", string(value[:n])) + requireNoWhiteoutMarkers(t, dest) +} + +func TestExportedLayerArtifactPreservesWhiteouts(t *testing.T) { + if !probeLayerArtifactSupport(t.TempDir()) { + t.Skip("exported whiteouts need mknod and trusted xattrs") + } + if DefaultImageFormat != FormatErofs { + t.Skip("round-trip test requires erofs") + } + if _, err := exec.LookPath("mkfs.erofs"); err != nil { + t.Skip("mkfs.erofs not available") + } + if _, err := exec.LookPath("fsck.erofs"); err != nil { + t.Skip("fsck.erofs not available") + } + + root := t.TempDir() + blob := writeLayerBlob(t, root, "layer.tar.gz", + dirEntry("gone/"), + fileEntry("gone/.wh.deleted.txt", ""), + dirEntry("opq/"), + fileEntry("opq/.wh..wh..opq", ""), + ) + dest := filepath.Join(root, "dest") + _, err := unpackLayerBlob(context.Background(), blob, testTarGzMediaType, dest, layerArtifactOnDiskFormat()) + require.NoError(t, err) + + artifact := filepath.Join(root, "layer.erofs") + _, err = ExportRootfs(dest, artifact, FormatErofs) + require.NoError(t, err) + + extracted := filepath.Join(root, "extracted") + output, err := exec.Command("fsck.erofs", "--xattrs", "--extract="+extracted, artifact).CombinedOutput() + require.NoError(t, err, string(output)) + + var stat unix.Stat_t + require.NoError(t, unix.Lstat(filepath.Join(extracted, "gone", "deleted.txt"), &stat)) + require.Equal(t, uint32(unix.S_IFCHR), stat.Mode&unix.S_IFMT) + require.Equal(t, uint64(0), uint64(stat.Rdev)) + value := make([]byte, 8) + n, err := unix.Lgetxattr(filepath.Join(extracted, "opq"), "trusted.overlay.opaque", value) + require.NoError(t, err) + require.Equal(t, "y", string(value[:n])) +} + +func TestUnpackLayerBlobAppliesWhiteoutsAcrossLayers(t *testing.T) { + root := t.TempDir() + base := writeLayerBlob(t, root, "base.tar.gz", + dirEntry("etc/"), + fileEntry("etc/config.txt", "original"), + dirEntry("data/"), + fileEntry("data/old.txt", "stale"), + dirEntry("replacedir/"), + fileEntry("replacedir/inner.txt", "inner"), + fileEntry("added/foo", "old"), + ) + top := writeLayerBlob(t, root, "top.tar.gz", + fileEntry("etc/.wh.config.txt", ""), + fileEntry("data/.wh..wh..opq", ""), + fileEntry("data/new.txt", "new"), + fileEntry("replacedir", "now a file"), + fileEntry("added/.wh.foo", ""), + fileEntry("added/foo", "new"), + ) + dest := filepath.Join(root, "dest") + unpackInto(t, base, dest) + unpackInto(t, top, dest) + + _, err := os.Lstat(filepath.Join(dest, "etc", "config.txt")) + require.True(t, os.IsNotExist(err), "whiteout must delete the lower entry") + _, err = os.Lstat(filepath.Join(dest, "data", "old.txt")) + require.True(t, os.IsNotExist(err), "opaque marker must mask lower contents") + data, err := os.ReadFile(filepath.Join(dest, "data", "new.txt")) + require.NoError(t, err) + require.Equal(t, "new", string(data)) + info, err := os.Lstat(filepath.Join(dest, "replacedir")) + require.NoError(t, err) + require.False(t, info.IsDir(), "file must replace lower directory") + data, err = os.ReadFile(filepath.Join(dest, "added", "foo")) + require.NoError(t, err) + require.Equal(t, "new", string(data), "whiteout then recreate in one layer keeps the new file") + requireNoWhiteoutMarkers(t, dest) +} + +// TestUnpackLayerBlobReplacesDirectorySymlink checks the standard runtime +// behavior: a directory in an upper layer replaces a symlink from a lower one +// rather than writing through it. +func TestUnpackLayerBlobReplacesDirectorySymlink(t *testing.T) { + root := t.TempDir() + base := writeLayerBlob(t, root, "base.tar.gz", + dirEntry("usr/"), + dirEntry("usr/bin/"), + fileEntry("usr/bin/sh", "sh"), + symlinkEntry("bin", "usr/bin"), + ) + top := writeLayerBlob(t, root, "top.tar.gz", + dirEntry("bin/"), + fileEntry("bin/tool", "tool"), + ) + dest := filepath.Join(root, "dest") + unpackInto(t, base, dest) + unpackInto(t, top, dest) + + info, err := os.Lstat(filepath.Join(dest, "bin")) + require.NoError(t, err) + require.True(t, info.IsDir(), "upper directory must replace the lower symlink") + require.FileExists(t, filepath.Join(dest, "bin", "tool")) + require.FileExists(t, filepath.Join(dest, "usr", "bin", "sh")) + require.NoFileExists(t, filepath.Join(dest, "usr", "bin", "tool")) +} + +func TestUnpackLayerBlobConfinesSymlinkTraversal(t *testing.T) { + root := t.TempDir() + blob := writeLayerBlob(t, root, "layer.tar.gz", + symlinkEntry("link", "../../escape"), + fileEntry("link/pwned", "x"), + ) + dest := filepath.Join(root, "dest") + unpackInto(t, blob, dest) + + require.NoFileExists(t, filepath.Join(root, "escape", "pwned")) + require.FileExists(t, filepath.Join(dest, "escape", "pwned"), "escaping link must be resolved inside the root") +} + +func TestUnpackLayerBlobPreservesHardlinks(t *testing.T) { + root := t.TempDir() + blob := writeLayerBlob(t, root, "layer.tar.gz", + fileEntry("a", "shared"), + hardlinkEntry("b", "a"), + ) + dest := filepath.Join(root, "dest") + unpackInto(t, blob, dest) + + a, err := os.Stat(filepath.Join(dest, "a")) + require.NoError(t, err) + b, err := os.Stat(filepath.Join(dest, "b")) + require.NoError(t, err) + require.True(t, os.SameFile(a, b)) +} + +func TestUnpackLayerBlobContextHonorsCancellation(t *testing.T) { + root := t.TempDir() + blob := writeLayerBlob(t, root, "layer.tar.gz", fileEntry("file", "x")) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := unpackLayerBlob(ctx, blob, testTarGzMediaType, filepath.Join(root, "dest"), composeOnDiskFormat()) + require.ErrorIs(t, err, context.Canceled) +} + +func TestUnpackLayerBlobIncludesTrailingTarPaddingInDiffID(t *testing.T) { + root := t.TempDir() + var tarData bytes.Buffer + tw := tar.NewWriter(&tarData) + writeTarEntry(t, tw, &tar.Header{Name: "file", Typeflag: tar.TypeReg, Mode: 0644}, "x") + require.NoError(t, tw.Close()) + _, err := tarData.Write(make([]byte, 512)) + require.NoError(t, err) + + var compressed bytes.Buffer + gzw := gzip.NewWriter(&compressed) + _, err = gzw.Write(tarData.Bytes()) + require.NoError(t, err) + require.NoError(t, gzw.Close()) + blob := filepath.Join(root, "layer.tar.gz") + require.NoError(t, os.WriteFile(blob, compressed.Bytes(), 0644)) + + stats := unpackInto(t, blob, filepath.Join(root, "dest")) + want := sha256.Sum256(tarData.Bytes()) + require.Equal(t, "sha256:"+fmt.Sprintf("%x", want), stats.diffID) + require.Equal(t, int64(tarData.Len()), stats.unpackedBytes) +} diff --git a/lib/images/layer_store.go b/lib/images/layer_store.go new file mode 100644 index 000000000..46d414439 --- /dev/null +++ b/lib/images/layer_store.go @@ -0,0 +1,35 @@ +package images + +import ( + "sync" + + "github.com/kernel/hypeman/lib/paths" + "golang.org/x/sync/singleflight" +) + +// layerStore owns the state shared by layer materialization and accounting. +// Keeping it separate from manager makes the layer-specific concurrency and +// cache lifecycle explicit. +type layerStore struct { + paths *paths.Paths + flights singleflight.Group + builds chan struct{} + artifactsSupported bool + diskUsageMu sync.RWMutex + diskUsageGeneration uint64 + activeBuilds int + diskUsageLoaded bool + readyImageBytes int64 + ociCacheBytes int64 +} + +func newLayerStore(p *paths.Paths, maxConcurrentBuilds int) *layerStore { + if maxConcurrentBuilds < 1 { + maxConcurrentBuilds = 1 + } + return &layerStore{ + paths: p, + builds: make(chan struct{}, maxConcurrentBuilds), + artifactsSupported: probeLayerArtifactSupport(p.ImageLayersDir()), + } +} diff --git a/lib/images/manager.go b/lib/images/manager.go index 8b905a417..6ebc221f6 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -55,7 +55,7 @@ type Manager interface { // TotalImageBytes returns the total size of all ready images on disk. // Used by the resource manager for disk capacity tracking. TotalImageBytes(ctx context.Context) (int64, error) - // TotalOCICacheBytes returns the total size of the OCI layer cache. + // TotalOCICacheBytes returns the total size of the OCI and materialized layer caches. // Used by the resource manager for disk capacity tracking. TotalOCICacheBytes(ctx context.Context) (int64, error) // WaitForReady blocks until the image identified by name reaches a terminal @@ -75,12 +75,10 @@ type manager struct { ociClient *ociClient queue *queue.Queue createMu sync.Mutex - diskUsageMu sync.RWMutex + storeLock *imageStoreLock + layers *layerStore tagGenerations map[string]uint64 requestedTags map[string]string // newest pull's digest per requested tag - diskUsageLoaded bool - readyImageBytes int64 - ociCacheBytes int64 metrics *Metrics inflightPulls map[string]*inflightImagePull // keyed by digest borrowedCredentialsTimeout time.Duration @@ -91,15 +89,25 @@ type manager struct { // NewManager creates a new image manager. // If meter is nil, metrics are disabled. func NewManager(p *paths.Paths, maxConcurrentBuilds int, meter metric.Meter) (Manager, error) { - // Create cache directory under dataDir for OCI layouts + storeLock, err := acquireImageStoreLock(p.ImagesDir()) + if err != nil { + return nil, err + } cacheDir := p.SystemOCICache() ociClient, err := newOCIClient(cacheDir) if err != nil { + _ = storeLock.Close() return nil, fmt.Errorf("create oci client: %w", err) } + if maxConcurrentBuilds < 1 { + maxConcurrentBuilds = 1 + } + layers := newLayerStore(p, maxConcurrentBuilds) m := &manager{ paths: p, + storeLock: storeLock, + layers: layers, ociClient: ociClient, queue: queue.New(maxConcurrentBuilds), inflightPulls: make(map[string]*inflightImagePull), @@ -113,6 +121,7 @@ func NewManager(p *paths.Paths, maxConcurrentBuilds int, meter metric.Meter) (Ma if meter != nil { metrics, err := newMetrics(meter, m) if err != nil { + _ = storeLock.Close() return nil, fmt.Errorf("create metrics: %w", err) } m.metrics = metrics @@ -594,7 +603,7 @@ func (m *manager) finalizeImage(ref *ResolvedRef, result *pullResult, diskSize i if !m.claimRequestedTags(ref, meta) { m.cleanupUnclaimedImage(ref) } - m.refreshDiskUsageTotals() + m.layers.refreshDiskUsageTotals() return nil } @@ -807,7 +816,7 @@ func (m *manager) deleteDigestImage(repository, digestHex string) error { return err } m.clearRequestedDigest(digestHex) - m.refreshDiskUsageTotals() + m.layers.refreshDiskUsageTotals() return nil } @@ -841,22 +850,22 @@ func (m *manager) deleteTaggedImage(repository, tag string) error { if err := removeDigestIfUnreferenced(m.paths, repository, digestHex, true); err != nil { return fmt.Errorf("delete orphaned digest %s: %w", digestHex, err) } - m.refreshDiskUsageTotals() + m.layers.refreshDiskUsageTotals() return nil } // TotalImageBytes returns the total size of all ready images on disk. func (m *manager) TotalImageBytes(ctx context.Context) (int64, error) { - readyImageBytes, _, err := m.getDiskUsageTotals() + readyImageBytes, _, err := m.layers.getDiskUsageTotals(ctx) if err != nil { return 0, err } return readyImageBytes, nil } -// TotalOCICacheBytes returns the total size of the OCI layer cache. +// TotalOCICacheBytes returns the total size of the OCI and materialized layer caches. func (m *manager) TotalOCICacheBytes(ctx context.Context) (int64, error) { - _, ociCacheBytes, err := m.getDiskUsageTotals() + _, ociCacheBytes, err := m.layers.getDiskUsageTotals(ctx) if err != nil { return 0, err } diff --git a/lib/images/oci.go b/lib/images/oci.go index 88bd6de24..36684cacd 100644 --- a/lib/images/oci.go +++ b/lib/images/oci.go @@ -641,6 +641,8 @@ func convertToOCIMediaType(mediaType string) string { return v1.MediaTypeImageConfig case "application/vnd.docker.image.rootfs.diff.tar.gzip": return v1.MediaTypeImageLayerGzip + case "application/vnd.docker.image.rootfs.diff.tar.zstd": + return v1.MediaTypeImageLayerZstd case "application/vnd.docker.image.rootfs.diff.tar": return v1.MediaTypeImageLayer default: diff --git a/lib/images/tag.go b/lib/images/tag.go index 302263d7e..af7582476 100644 --- a/lib/images/tag.go +++ b/lib/images/tag.go @@ -92,7 +92,7 @@ func (m *manager) cleanupReplacedTag(ref *NormalizedRef, previousDigest, digestH if err := removeDigestIfUnreferenced(m.paths, ref.Repository(), previousDigest, true); err != nil { slog.Warn("failed to collect replaced image content", "repository", ref.Repository(), "digest", previousDigest, "error", err) } - m.refreshDiskUsageTotals() + m.layers.refreshDiskUsageTotals() } func parseTagReferences(source, target string) (*NormalizedRef, *NormalizedRef, error) { diff --git a/lib/images/tag_test.go b/lib/images/tag_test.go index e717d9eea..9a6d6a605 100644 --- a/lib/images/tag_test.go +++ b/lib/images/tag_test.go @@ -54,7 +54,12 @@ func seedImage(t *testing.T, p *paths.Paths, repository, tag, digestHex string, func newTagTestCase(t *testing.T) (*paths.Paths, *manager, string) { t.Helper() p := paths.New(t.TempDir()) - return p, &manager{paths: p, tagGenerations: make(map[string]uint64), requestedTags: make(map[string]string)}, "docker.io/library/alpine" + return p, &manager{ + paths: p, + layers: newLayerStore(p, 1), + tagGenerations: make(map[string]uint64), + requestedTags: make(map[string]string), + }, "docker.io/library/alpine" } func requireTagResolvesTo(t *testing.T, p *paths.Paths, repository, tag, digest string) { diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index b1d2067b8..ad6135826 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -934,7 +934,7 @@ type DiskBreakdown struct { // ImagesBytes Disk used by exported rootfs images ImagesBytes *int64 `json:"images_bytes,omitempty"` - // OciCacheBytes Disk used by OCI layer cache (shared blobs) + // OciCacheBytes Disk used by OCI layer cache (shared blobs and materialized layers) OciCacheBytes *int64 `json:"oci_cache_bytes,omitempty"` // OverlaysBytes Disk used by instance overlays (rootfs + volume overlays) @@ -19367,269 +19367,269 @@ var swaggerSpec = []string{ "sUHvPvc5fHd8tGNNkZu3RuP64mjMYVZ2VEbro41cEi8TlYtgjL4XCt8Qg3/r0Po7ipQvcSdXvWtW4ly/", "eRcw0yEgLItUeXMg6DrDXIs26k1u2aCzyMD8UJ4ST5C3WRURDaYcH1F5+aMg+BLg85fvbQBgGJn7LBxo", "BygZ4wWgQoGRS3CuJhIV4Cml8Xt779ne892ne88HgwCQ8TLB84iOIG+m1QDeHB6jBC8KTL8N8N/HaJzw", - "cZXQn+w+ff5s8GJ7p+04jMe63ToU+rf7Cm3YFfmLK3HmnlQGtbPz7Onu7u7g6dOdvVajsm6DVoNyLoaK", - "SPJs99ne9vOdvVarENLrXjpg6boAH6pycGAKA0Hep8xIRCc0QgDIhvQHkDI/o4wUztnqmRzj2NWCCN8d", - "CtNErgxrM53ZN429Nc0TRbOEmGewIa1cEjDzI2gpWFyLsaL2xs1asnDcawOe3FyKV0Ccick4n04t1l2x", - "dCc2jb0UnihJ4n1UqGwr+RzsZjmwD010YOfQkhpea9XJos55RGCuLgPRIAgq6MRsWmVWLtaBsiwPkkTj", - "Uv6UC4M+AY0iPOa5Ml51Wyyl7MQAuTDwMucsbmfu+ImLy7WJDfomLmrCtDQYuAx3kwFfy233hb7C+21i", - "BOxzid6aL4yhsPw5y6sF8brQkzUoMpeWHxd2Y9tMW+kyLLeAzdyFy5j+St55T6F8vYmJfvqyGraYEqiF", - "pNZKLJpSTDzSGbzeOk9Kf7jWkNJi3Rm5uo9Fh0y/nibbnmQ4u5sVXxUxXNgavDx9buPs+whOFwT5uQT9", - "2kk7c1ATdsn7Q2YTjTy8AnCkAZaACROfESoQF3RKqx1X7az3GLN7E8p0xHVr6vQ/bIrBk4UbMMQDDNLy", - "xITAmgIgPtCr3ZNOt3NWFI2yjKm6NG+LwltLK1JG1i9j7py+u2nkbSb4hIaAcCASyD61ipqLSX29Nzjr", - "bf/VRM1r8gOJjTITPZTyuFbjyb7f7iJ6dfrutGlMRdUl5I9uaU4r4H2Wo10mOTN1nzybCrb1GC3imsHj", - "MZ310Y9Eqh6ZTLhQ+zaQBRxSXs1XvECCxLmmBx+ajDMk6TiBM1r0qgV+/YueEACBjPPJhIgq6sCLkCzt", - "vT1KA8a9n/RzZF4oYA9OfqzK01pub6u1n1aoAdT2CY4om2623u6AQbA2jXW4gq9O3721ZYOaEJf1Uhal", - "hQzYch/9WhTW0kstS1SlfsBSWKMnE0CtXxjJhIeiUX4ShCB4Vhb01GcPPNgGo7eIGsAxYDRx5soKog2I", - "8oqJRL/nWGCmKCMxev+T/B4NrMv8/U/IZNFYBkSlH9hY1RSfhba0MRvudLaQNMKJWRZT2oMy30oJR7q1", - "mnFafmjtuQFlI1xUzrEPtDGfZjks4Nnb3vGb91tpTObdypggGnHGE6LHvenx2LmDZChT9yqsdd5kLnJb", - "VdQ4HJV1FesHAdCgrR2zTh16bwZoTCLsAEXqmxcVZYjHEHsBVq8rQZXSkiu2RmeInikbMaB5Ek3yJOlb", - "BuiGbMzME+DDHoKpuao0+cDzDIMmVYhR1wo6JUjmUURIXAtfqY/ao7d9lLMUCznDSX1y+0WodzTDAkf6", - "Pvzu+rtOuI6QHpxsy+g96ixumtZk6d0rAXr0zl3TEX//U/V4mqveJAdbAzllVAHYGSBcbxSDNJjcfXSC", - "F9aOksGdaeml3DE8mRjIz+IuESQhWEL9b4ne/9RfG5ipuMJJ0xzO9UPLpzb0hPQp0sPsoqzCBoB3eVhb", - "frdPg1dGOZ+Amwk6rPss+uiYhdmeif4BtDwDfCUlnTKt20kbABRhVqzk0t5V786gWaiOLtipLlxlOt3A", - "BRCimNBtZbIZl9Wc5Xqk/LLKpvjl+hqYppHmfg9dwmXNr2V1SXduIS/TRSIh86nlGMY6KkmGBVYkWdRC", - "oqs1DchyUAm5JtENMj5f6tc/mTo2uSAjNRNEznhSDUXa7S7XQpYQ9j8ntvybmZPne1McpVhcwilzujTK", - "mVmBatbI7jqIp5lS2Q0m9fP5+akxsCki5jip5x3JpSCbI5LgBRoTdUUIc1PBEmE/7Lyety0bSnAJNcqI", - "oLy6hp3dQL9nJhUBTQWOCDJfuUrqBVvTR6/tUtpeAqChUUSkbNjf7VX7az+d5Em7PQ4Na3vdDqvoJht8", - "fnjqSkkVpbrdMu8sr/IpET1z5FzN7tVbuyNXFzVzXTGDBb0ks48JVDmzyV5+LrYLMYQqbvrzSv6zxxyk", - "n7Fr+4FTYJaqa875h1YKXv24h2JpUsziUMlzk+Nj0DGmAJkHiQYiB+2LxkZSMneydz/bVCUtaVFGpKxB", - "C0S5SDrdTm9iZ7W/taX5fQKooXu728+3VkdyrwzhtxGLo5iuMvG4uEYT+eYS2Q3kIky6ShJbOMtaGMHN", - "Oq65H4A9LYcMQ5lzfbd5SpbLIRkMllASrnGkXGFGsIpXoh6wf2wB27sqyOgGU1NrSWu7L/zzOQiGoWRY", - "zarkv7VE+xCaBmHFmkaMxbG2jobIPwalUy5C1fe4UDbpdkxc3HFxH7qoXodiXfGPD577s3z65Mnuk3V8", - "CJhN7ZjbcxeYqnm7CitAvOW259c2ABGSVZnDHenVBYT1uqyhKc0Rl0hq9YLyjLAbreeTvd2dm61n24kc", - "u8jMGl8KoTIdnhwZmUjr8pgyIlBKFI6xwlUmA+ZkzWWgvBMmKWTmTb5fzVoaQph8mKXb1qb7UgEwDWUq", - "3zpU9hQzOgElybzp9yxneOfJ031TTDcmk70nT/v9/k3BZ16WaDOttmLLxMl6ODR9Ofu8fbgDjJk2c/mj", - "c3pw/rNmZLkU5tLakmPK9r1/F/8sH8Af5p9jysLYNG3qL9PJUt3lakhobsG/SbyPyhL7Tu5pE6LX4A+C", - "BAEAxAoiPVYCpe8O0rGgcVqt3HGD9PwV6epaXHnDkkXj2ty6lDIrbKHKK6Hs2xValFOmH1eHuDiLM7xj", - "+zSWjqLS9HJwy61qhcuV5VSXqu1lhBUFVJPE/BVxBtjaoWqqlSvSPWtRjA+uEVt1r+jS/7Ho3fvx0B+I", - "97sr5uf9ZMuqfrhhVNpKgfRvy3Loei7kxNE1hzls/i9uhbYlrC0UZDAd44HvwttEblZ7fzP9r9//jzx9", - "9vft31+/f//f81f/dfQr/e/3yembz4IZWo0C+qBQnl8MvRPCFSsQnm1J6QSrKGCj0+pfwwrbJ8bioKIZ", - "FN1FY7I/ZD30mioiTAnHWv7xsIM2CGhK8JUWd6E+lUn93NQfn5qgAv3xH04M/lRvI7a4EMJuSAH3I/Nx", - "zFNM2eaQDZltC7mJSNAL9F8xinBm6jBShrT+u0BjAUUzrZe37LyL/sBZ9mlzyMAqS64NoH2Goe7gpPBN", - "MBejYUdlItHt66RAfjFJwUNW3NYFDKZx9ffLmhmUJPW0vYZFWa2/Wc3p+SAEGAopZXojoS4UqCAFZWsy", - "KnLd0PPB5rI+t0bHKGhoBfnZWBiTc3yQh8zFTXnKRySmEfAVl6o7s8ncRZa0oTRrxMsEv17A3rw1+aMx", - "wrmaaV4UWWyLiPNLSrqwpV3wSEPwFXxpQmpmPOuNF70ZzwqcEyxMwBk2QSlVJfv/9OxEe++JoBPbUxCu", - "QpNIQOiEI2NnZrJ+C+vC0sTOTUkdpkWfObGvm9pM0pTjMdkUKhfMVYwhUDcWAHUK6iMhmfx7FCUUrE5y", - "Bj68GWBeKt1MCLayMyiy1vA4ismk/u9qVNHOk6egwbp/7+60Tho3S7eKyvIkoNOmjvW14NiGTcIAjHgw", - "cobwNXGA+ga0oRRgp1Ac/nuGXEPliSsYifH0mTRWaQuQJNJLYN0MZhraY2BRe0bYnqY299HSKawkLrZo", - "wQQIwWdJC7yglyZh+vz1GVJEpA7CYiPSuwOnxIDD9KiUua2Fd3B48nKz3wlinVVcWrBVKxMbq4MOwJ3Y", - "gKGmOKjSRoNT0kXHR5Cwbq+VUheDDKOfuECJuRXLy2gf8HKq5h5sqmgeH1kBNFmUUUdGbBl2Nl2LWf16", - "20dvCxUQF0MpUo9L2nJNlpcJNGtjUE3601LrtUx18I9Z9c/ex5DsBCVGDS8GvNbG+6u9zdGhv+mLqmYh", - "u/GF5AeCNdq/vL3/0sjmX15G372ZjG690KNshmWIume+VxNeWtp335FdZfeiOWyv0u9I0uDZ+psrtORd", - "Q4roe67yeQjK90lve/t8e+/m5rubglJX0eg8pMoCl7o9oPRdADMHYJapGjXmdyD92GZzOLvI+xM0w5J9", - "p+BhzTqyvfusjVECem2bGeHnRPCJGVLBpRy0XRHRb0D+LmmSGAFG0inDCXqBNs6OX/1y/Pr1JuqhN29O", - "6lux6ovg/twCnxpuAVhHExwVQDeroHKgIn33/Pw1HK6EQAaUkcMvb49avda02ALF2g3u1ek7cPxjOXKx", - "083pwrhMuSfXVCq5DGzYKgXhc1CzzaelYazNJE0bNqhyLfT2zxVs6CBS5eYdYGa7/JGl5XwAOOmHzNP9", - "+qCsV4JPfy6CtLUz3BGAdOOVFgJfroGSPGm63W4PBX0nw6lgOoXYli/hOBCFW2Mvdzs0kEB+YOP40PFp", - "WWytdEa45mtzerHT3376HOoFbw/aMPYURyv6Pjk4bN/5YMfcMvt4vB/F+6Cw39ZnZQnbqCA4ucILqLdp", - "lnbYMRemp916x9Yqkq3ia5Yhrm+HaF0X4xowq0GcdYFLcpSuLPfTIkO4jluY5haVNaVJQiWJOItlVUae", - "YYlkZsCITdmbQoIfMhhgFxXVx0FKQTiKRF6aHq10beX9PLN0D6V3M860DgC1N34hC4lSCk7QonsIfZSo", - "SESLh2xDuKTFIjsRqu7G+gdIAera5JK4C1HDUN5HfzBkcpYrzcQ2++iQM5mnRFirLBpT8BhtIpkblRbG", - "C6ux0AxT0piIIdOvBeCO/yjUk/2ng8Fg0O0Umtyu/vcgRE136vzsWzhvk3YPgJvMAntDtLvIGcpZTERR", - "gp8YcqiHyN3QcfqZON7u83bilf28lKvCB3Md7Hc7PO/PBVGGoTbo5xAdegvl/MntRfRWaX9OfrUJf/ar", - "0U0iGGxKhtb4xvq2MwY5ElszpCQ216PIvXlnyt9Wp25DjxVHv+dELND7k5NK2IPN0Gg3ceASDfvAsxtt", - "w84aG8na0dzEvexBTt8HzHRdUvEkxC8OKu17GB0OgaHQimGrojivsq9prTKYA0WZ2SdNNCsmWKuAbFBb", - "ysBIv1W5kBMLsWpFaZdTYSHfCwSW0qkvF3JrnMutLKJbNntsC+BxngM8zl4QvyAm81Geh1Qj/cjhIb17", - "d3yENuAXgHc2CTKV7jF+uv188PxF7/l4+2lvLx5s9/D27tPezhM8mOxGz3a3d3ZXJBW1SGi9fY5qUGMO", - "xDEXUesjFz0fCmpuyl2oySY2HvuKsphfVa6/YICs37sNvl3X/XJofeshBFOCEiyVMV80cLITuORJpNs2", - "Aek2abqodhY2dD49H2x/rvUHBtdwR5yLnBm3qoH5KFwIqTdgf7Oq47wdy4cBucSXdavld95+0Qb7T17s", - "P/ncRXPJG+vGWCene9zcpogwB4Neyw5xScKeHckZKDtWJjJWfZtM0ul2inwX+BuEgVosdfG4VRJX04Ht", - "htnIqmulAb/guKKvQKSKgcGM97Wk4vQRKKpSoGRoEegw4XmMPFucQQUEP9yxp7voZsAtZk10JnHZJGNA", - "ZiSVNp+PMs2Iwf+oG7FgB/voFbwLj3Bq1Do7CFNCyHe94Xhh4mX0+XJdGyVr9ZDPrH4F32hlC+l/wbT1", - "MliT7eomjHS2j37l8E2h7TFet/2a10HNWn69bifesIj5DrwGOrOi5j76qRAvCwHVCqQbktg/R5ZhlZhR", - "mxXkDrvjHU0t5c55KBTdjlnRTrfjFgrQKpZxK96VVL90/nxSDAWSEZzAWS7T9HNFE4uUDzOhUtFI2uQR", - "vblNYo/NzSTxyChPTTGpJvXVKljFR06qen+CNgAM9S/IGrb1vzaL+NXKXbfzYu/F02c7L562gjwrB7he", - "ND4EZILlwa2Vk6MsH1nbSNPUD0/fGdtHZKwKRezL+xMfYSYTXLMePXPXoN/5i/4LH+kt5vk48RyLFhbS", - "AEvDhgXBDAte1BAH+TtN5nQyYb9/jC53/i5oun39VO6MtxsQrE1HYbPbsR9csGSjJuOeqVQWBuMCghKy", - "Ea/uLZEwA3RGFAL66SEcgXpT5FNbknOodnbFg4S1t7u7+/zZk51WdGVH5x2cERjhApeyHYF3xOBNtPH2", - "7AxteQRn2nSwLlkA5qB2zpAtMz6oCqT97cFuiEoaLu6Samzb87Rxyd9b9dFOyi46JG8XquXSKQ+u9u7u", - "4Nnek+dP2h1jax4eievVHMalLJnlsbUw/J3fAGny/OAUQULwBEdV246LELvRqNSNRgV1XEz9hRsM7Pmz", - "p0/2dne22wEvhoJOLKRo5cBWeVfg0AWIIrAbgaVYZr3dptsiJE4ZAntLogTT9CByKRa128fUWRgJ81q5", - "CW0uBquBL11cLb5tZdwqTFYmQceIBlygnBXVffrrXbJfxLPazLXN9bCeq4fScphePYsQZqoY3mIpM0Hm", - "lOfyCzTElcmZnSScixt926SwvCUyT5Sx2VCJ3p98BzxF0xqSimRVHcpS4woctVtO7kbnuUIiYSJvWqxW", - "u9Fm61dNuNtwarurADUq3KARvTDWnCtn64M/D3ES5VDPChf7qWcFMHwACZBlycLE9icJ5wxFM8zASSI8", - "0DE040ncD0bC6iejSTCqgl+hhBvc9UtCMlvqyQxCf6ZFGDonaMMvcmhIqVZ6+ElqmIwt5lOlxidpuIZq", - "GFOpSIXX64kV9yDBzScVS2jCpxKUQgVZC/16JYoMC5OMgJkpXTZPjS4ZCLgODLHGzEM3qrlJ+cQquFbk", - "gERzs5I4Elx6cGDvT2r5yyty3oos5vUBndXBtiBd49AMXGUGkK51hcPQ/RjI5/mcGxJoGHIGV4RKOuNk", - "ilkOxZ88QraG+H7rcMgZl2pUwFzdcLBSjaCiSy5ICVhZZN0X9iD3TvBedKztNstl445v9fUSVYWbahpg", - "M08Nrmh4tboFDYbIeBlXbiWUXQnwVwezugleZFkChEpolXrIgWgDcl48tuShQG62CZIJq6y6nyVt1Rbo", - "fb03OGuLrLgaSPEUq9kxm/AA9scNPKfOEm2jVTMiHLhhTBglsdMlCxeqNXVBwngiCYpzYlfOyKc+/B2E", - "OGA1A3skfEjZtMbr6x22MQ+bMawu+AL92hfbhDvJcELtuchhrUy8okS4TK1tFQRK5SjszlpuWJBpnmCB", - "LCZqmyHLRZpQdtmmdblIxzyhEdIf1P3iE54k/GqkH8kfYC6brWanPxg1VQc7M4OzeYFmQ2r9llP4Qc9y", - "s5aVDJaYLfP9FjhG20SPBSPFf6IJsdiU7xi99gi9WhFhb2fQlC3f0GglT34ZnPWmnNuSbPDE5zKQW7hS", - "ynF1zUhsy1QYsSfLpSmx1OJWcjjIzgV4O49ONXHk86BJDg2/rgGToDGBvB83tWWu0YIttplKsLpLLmfo", - "73xcNYi2DfsN1AzcYCVEhiCTYHw/7OhKg7R5Y2lNvN29CQYFsFU9UfjohtAO66orlvFVTfzk7VKhwRmx", - "S0bdHE3RwRZFdFz8RwFfYHttj2NQLwkZiFcGlBqpFlBcGSpaLbw6pxKNuRAAAq8lHM7cbAB2Rcs8eq0d", - "7hU6n5EFEiTFlA0ZZYWRFMDUCGJkToSXJcuFVrKmJO6jv3kqHsDmp5la2HoMYDz/TiJ+xYoxDpk/SN14", - "LnU7B8xYFkWeqUrFVt0saH2aUCBrGZxgSkAJU6pmaCKInPlzD5Wt1TLeFRdxYz2wBXKvQJkp8LEixS8J", - "81lZ0UxQNTQNjcxXy1F8puY0PLX6J6qUgUb1Ms+r+8slEWEhsZhS8Uqr0BXvqHjKiQGBAUQUKO9p/zIs", - "vkBBaYF5Ujb/V9dk+dNp0Xj1t9prHq6JQ/o+MGbboAk2Mmk8tWCfqidtbagKpMGtQrNZ9iWgDRdC7Wok", - "VSUBr1ZRq3uyXSZePVnAjWZLkqja+97zJ8+etiwW9VnOOoPe9aVdc/N0hUuuYadO2vh9nj95/uLF7t6T", - "Fzs38rC4vJKG/WnKLfH3B22Qa6UPa/Kvf/zz/UnN6/MEYrAHNxqUySwJD6khu6Q6oPcn//rHP92obj2g", - "EKNZBulv8Ns3Rukk/k66QIGqC6+dk2yFfn9QMRLggs2gDQJQ3HRORmbdeuVgajAg7aRgnOGIqkWAkeMr", - "E+1evFLD2m7jDqoONiTymrYtKqrmXDIfl0mnG65z9J/GN1yjheeta87JfNzkh35T79V4oUuvhR/j0CLE", - "wFBE2MBdzOcKy0pAt/47grwLl2G2nG1j3liNultPhYAoFlta0QsFDNUaqMmT9iN/+2vb6fktK2ad+op/", - "WHEOm4/gjay+gRs5YPSN1qfW1viDvQBv99Vo7FeDXFlus1I6srx1b95vi+zh5VIlxQ128/68hMmbfFjH", - "BAZ6tGOwS1623a2QRAM1ebkwAQMaT0jPK15gSm3J3HgE9Zm3MPOBDM7okk8mVazbJ83Y6AD7A8lerhes", - "lNZMuohcO5tFHVjbYPwMO0/ksKNVgGFnOx12am6rYPpkiq9HtoMqtstgFVh5mf5eG6R0MxgnPLo0hQ6h", - "fn4fDVBKMJMoZ3D4a1617cFq71C3k3l7U0CDExPitMS2YExjMsNzCvVUrE9lWgnEJNdUSQgYhXb2UcwN", - "2lOlyrOdoX7NJDful5OGSwezhW1YN6jf48xFtJbvgoFvArWl2UcieNeCFWiO/ebNSdcEMEDooRlYJb7R", - "TdSMQDPIootaeYXy93D88DghIxh3Ha4/XV5HPycdPKuCSKKkxe8uyaFGBCjiOVN1HP+0nSJXTStbvpJy", - "BsF+NvwDcNls74ZAUEwiOJFy+SxWCf0WxF3LG7ArHUoc2A2RMByKFfV33lqHcH0AxtjgFWg37fhx3cZL", - "OJKK24p+xakekeuIkLgO+Bl+pW2svP0yGCv/GluMoKJ2un0b4p2XZ9e/uwQvGGvTavsx/YyzHqCTuC21", - "SCIGGtBi1VQJrQI97kFajELwqqEX2mRck+vVa/0ruVaAjx7niQG9C5OuZVX2Mlq34rfObGw60FyQtRUy", - "76BypIk3v1XtSBuq/hDlI+1bd1Iycml3zohy755ZMmrcoWqhl4pLywX8u1eqMTaGlLrIXvBoO92skeDe", - "LGwVsaC8LXM0GU7JKBNkQq9XEI95wSjGVViT8iAVGQwGX3Qjxddo7xmU/pK1sTM6nalkUQ3A2QtgKX1W", - "XVVBFGHOUNhm58vddB8uR7vZ7fRbDwnHZx400FJJEyuSjlbhZh+W3jZrnc/wAqw4jU7CZ7t7g8HuzuBW", - "wNluWDdYrsPyE1uFtNpOU0qd95119FeiVP0WiiTr5dLWplyd55SUShCc7kPiTYYjghIyAZC8IqF1vWex", - "3vXqwVuBymbRFvTvNsrum/PBV0vmFF1ZzHE3jY5zLlYxiPznaxyiDWwmWoLUC+Tc7fYGT8+3d/efPN3f", - "3r4LsOtikZqyPZ593L56luzgyV7yfPHs9+3Zs+lOuhvUwy6pqQzUhlZ/0e82RtmUl2QVy6jC0tCGnUNG", - "RL1meb3WvyQJZaQniwyp9WmKK3iB8b+vPf83s/ObGayUHc6qk/RFCKzKxalQ1sPgb9nJrPRd1GdzfLR6", - "FrfKQKoPJExv9aEAebUbDFSo2O58JjJDzlpeQ++8F1tfRCuz4tZdRSEPO5z04C43rHiIvGvADN6sV13g", - "y5dcwHY65YKqWbr6tiheK2DEIW76o1RxFe+pj46njGvtyf+5CJPzlSj9cafbST7uVc+M/b098pdFIC4I", - "0G61LxW0CCNLyJwkq1cBXikVD2Ei2bWursf8w3Zv+wXEISQf934Y9F5UIw66ZrX85dt2b1d+HbRZQ78E", - "oCsdtf3iRhHXbj1XUdAvNFTArryXLTaxpfGyPLy7OlzCbWWDy8dLe1xD8mkUQD9X0rOX28gXmmKS4EUI", - "m94z1Mqa9ugTGRqTKWWyjd12d1AYbp+kw04fHViAcNBlFS/68ZvXtOLTCU1TElMtYxrVvzmDYaelLa6u", - "S9ysNon7KiCt9cPi2ov1EAnrEq7WXZP9z8jH/Sztt53Guwq9A+xqTkUFDDF4sYvoBGFWK1DqilUbpQMS", - "IyFebd8BtZUka3mALOVAZyfpoilXqEyhb2lvy1mzXbAYP7kGe+sKzAxDEDtfBBClABCjq9jX8RHKBI/z", - "qMwfTWDQJeKHyGsQbSuE/PUhuXdp34DE7AkXaL19o8mg0c4+2bTfNdukJtjmrd4erN/qOzGKdDt5Fq/n", - "YealdhzsRsjta1IQAyaa6rLXJEFvMh9acPS3/gou67zGlhxpkSjPnINF09QyJQXcLeBiCMX1HpGE6Gtq", - "uRHEk7jMkqCy5KLrWer20+ezJhcneKSWB/ILIZnWVQD/CPpLMVsEB+bKjhZ3ycbAoX1L4/DqmXJFdrWq", - "g3u2VhJr3CrfhNtUQsFw+ZrN2+ClXHrm7wLj2xfNlhFQHMOvCGlvm0sA2C9d2Fuj/fguzHIPKaS9sa6H", - "GmyrAxUu0NFd/2UssBbrqsS7F3LPh8jiHE+hVt4aP0ojqYTqSoKTAyOFbXErKLCp/4mgiqh0gC1OQIDP", - "XUavLTaIqEJ4iuuVI5rKUhpP9PoYouZCL+fWRNCEyVtPh/XN7we9/zHmdjTq72/98Jf/u/fhP4Nm95oB", - "QRLRi8kEIq4uyaJnqjApPK1eoX+YEhBaq5jaM0NwCsY0QHu3XMkf75NBwT0Xv+J0aQoQquaVUNpeO6G/", - "/EdzoJe3jO/gwlh7dj+7QspdVJJV3N3LGykRUxdU7zLqNvtDBkl6l2QhkVeYzcp27sR+J4tPvFB8dGHI", - "vU/Y/AKNKVS6lEOm1XscRSTTapWt9UNNuXYObFgQnPjt2AJx7rxYz6wJrCDo/ckSnPGbd+c/vnn369Ho", - "zenLXw+OR7+8/G+IdbnqmR7inqa9vSdPbZF2fyW3g4VCbl7voo9ObL6CjXmY5KDZA2CZRGmucoiOIddR", - "kks6d55Sldy+ssVy1vLtK0V8JhSyUkkoPMNCdid0QiDAAe5VG11EpSNGKqG6vbXyUIaWRRdDOMMOXCmK", - "O0kiWFdEb0V4tcuNrS7601m7u8SAxAYOO6T+6sMWsKK+phKAO1wQjPcy2oAUGleC12UQb94MtPagaDAY", - "gvmFKy0NXnyJaqjvVpY/nfOkp/W8hpIRQbO6WYtgCgE0ZVIzOk3el+k4oMxYG/eUTnHA4RJyrHyRqqVu", - "QGtTx5b2v7F8Wzih46heT8McS7NUtfoPNWuJVL3mfI9Ui/cNCMQAsWySeKkXZFjN2E2Z2rLVhUPAITEH", - "VPdVadvlKXMwkT34aH028koF05uZN5LmvTlxelRN01uxQKd6aa5mRBBvI+CDsk7BDZfMJii1gKMx1Rkz", - "IsrgXZfdpMVz8LtLtFGYwNwSFGnXy36B1XUoTvB10QP4lLBccsTCPMo6WNuvfoSaAW9d7U86cU3AMGpa", - "bhghv0pFq9bEUdXyZvhUtTxv837w4FletYL7NZ2tGnGWfVRIM0SPf8NU/cQF6MXN4C93DrQPl39MBIDh", - "1WH0W2HQ05TEI56r1edfv6alR3PlF/Vhy/rCzgaAgYijSl5zEy9w8CTlGJZXWi8HiXJB1eJMr5eNaod8", - "UFfUFxYSOoKfy46hkOqnT2A9nwQyZ14RRgSNoEytPo8pZqAxofcnXrVCU7hyCbgWRKA3h8fW7uKwj0GP", - "pgpIzwWgHpwed7qdORHG9tAZ9Hf7AzjMGWE4o539zm5/uz/ogFY1gylujXOaxDaR3GrUhQZ/HFtJ6Ef3", - "kv5S4JQo+OK3ACQCBGDa10EFwVNPicwwFVaLzBKAajAEQ/XXUHfBXaj75lbummVvbTyGfGtIAyLZG7u5", - "H0BQhrMD09wZDCzCu7LXLyQxmcyJrb/bMNqy31ZSnV2iQBmCJTXPyZbF0n/qdvYG2zca06qhwNkNdfyO", - "YZvNTEA7f3LDhbhVp8fM5CfabHMbF+afOCAk/6z99kHvmczTFIuFWzB/tTIumwRjIhF27xo9TkkUaVYB", - "xZL66A0j5jnCCmETwi1yBjWm3YeaQqunwLTtNrlAa/qRx4svtoSVPpyN4lOVnenj8mmJnr8c7RRkvLyR", - "9pGDGjdUew8E9CMuCqQ/2EnZG7y4+04POZskNFKoVxCwDcymEmKfEgBOdyBMXKDfc64wKvIaHtGRtjLr", - "uCC3bnkVbf1B40/meCck5A84JSLFzGSJmHfWHPql42x8M+VxXnmrOcI/PurYm8qhEZmLCgS56hH1r626", - "MLh8He0FoChsn2Z68QMS/t49nHA72aJG7kMeOahMinJJHtNxsr7GcSmEBGW5V0R9LTQ/uM8ry1ZT+BOe", - "osdCwK9IIeGVu7V0KWxlImdGAQ5KgG/LzE373XdV4e+8fOKFC4FfQzcNdT2UcTDjeNFHbk2N0q8WgDUl", - "CMwzXr5WTvXwvpYTtnMfJwxmXHiKvl1T366pVafcUIubAhxM75S3sEHcyALx57M/3Nj68M320N720Mry", - "wMiVtS78nY/7yIbmRjwmSM54nsRoTJABfnJBOAqL/vQjwiKa0TkBdD+oVpcnimZYQIhNimKssPGhNxom", - "Vpoliua2dHM9F5BZLnAd0EOSEYTAjJqAOMtQTMoYiZH+xEbNlLiKS3XVzdkPGtiLBsurEV3NuCQFsCFT", - "3m0Oed7SaMfQbH/Izi3irV5AiCp3vEaSBHB7V9h/OEN4yOwH3zsW4iLiJE5LzoUFgCdSA9FptmU5x0+P", - "dCQjHgIdOicMM9WTGYnohEZ2WpdkYQNbgw22KkClB+zG+f6kyFxBO5th4DoIXQqjFB8Vz5ClpKr/hkE0", - "eJTkcenkclhKWIxxkgQrlEwTPsbJyKzPJQn4BF/BG3ZRSodL6U1iPCamln62UDPOzN/5OGcqN3+PBb+S", - "RAw7m/0hg4wUu9Yk7pYCIrqCinZpxvU5Ezw1fW6ZIW79cUkWn/pDdhCnlDmKgE9wIjki1/AdxI0BeIjh", - "Xg30YE5T2A9+mEvFUx8C1tGdGSbPVZYrm1ojieqG4E+HTHH0hwO5/LT1R9njJ3AWExxrOvFeMVMC2bpp", - "1HKE9exH8GrA3U5gAYYdfZGaMI+pwEwZ/NICpRNN/S3dKMpEQOnY+gpHmKGMZ6bEBhDVDGuSq7QBoBU4", - "SZCCo+S+1YI77GTDfCwGYTpuBCA0iHG1Y0QZOvnRO0yDvefh8yRJJEgoouS/zt78iuBW1ntgXivDtUxu", - "C9MCA4pzcJ06nvYSRzNkHFVQVXHYofGwU7hz400Yay5tuEyvBz7FH/TQfjDddGn8Q7+vmzLuyn302x+m", - "lX19lrLUAKIOO5+6yHswpWqWj4tnH8IL2oTjdlZhBGjDXHObwEkwBcgd78Y3VyRmMeL2FkgWCKOSA/mB", - "K2PKsFisyqgMLL1dQT4xkYzeYvwxhMjFYWd/6GIXh53usEPYHH6zAY7DzqfwClivZXMJP7jPCudmQURP", - "B4PN9ZDgdn0DPssWjoEvrAM2akVF/VG9gxaP9s/lH/i31j8L1w9muvMSo8ko/s74/ggdEJ7E7muiARdE", - "TezGLCKJE7vXG3ru33mgNysiSXLfBPpQ5Fm4x4qSBY+KHGGzymO00nz/wBQ3uK9LpWK2fxj6fXT284D1", - "3NrOydyFOocLtgAYj1WlkXkZYYnOYEy9M618v4Rf+/a/TvcDcMmLhE8v9o3qjhI+RQllNh/AC1TW4oFd", - "S/jI4PEU31l4Hlctb8NIEv/6xz9hUJRN//WPf1qQ+3/9459w3LcMzhwU276YESzUmGB1sY9+ISTr4YTO", - "iZsMlMMlcyIWaHdgbf7wCHk1/62UJodsyN4SlQvm5U2YwnXSNmhdBXo+lOVEWjwjSBOa2Ko6JrYxYLdx", - "Z9ks5b2e6G4AFxJm4E1A34qOBgBUj5qK41YT7YRNpmbOFaNpPUxzKVhvPX9R5FoZ6u2ZAd6QwcASh84d", - "PLCTRhtnZy83+wi0LUMVUDkJdIeyGatG9L/xpPU8yXCUKkOBVTa8KcIZHtOEOpNjQ9kXcwRTHM0oI2V8", - "cQG67prYdyPVPObg9BjZQMguvDpkb862wMSqSKRyQbqWEwgLtVrWheM2zwV6AP5FFUSH9ey7QzYhGPKE", - "jo8ME/DQyIvEyKJhBogmEONKVaUEXXfIDKSuhXDWBy/lMUngI+h/ihW5wosuKor+ujIxCVZaIZZd/fKQ", - "mVRDuwY9wGxB3jD7wM/MkHouktfmbAkySbRqDBH4pv459L0x4QLZCOdumVfqujPZpmZYetFSHL050/Ob", - "gibIjT0QWnpz5nZjs4skR1FCgRoizIZsCoFADsWYs8quFgllMyziXsT1JeCjWl0yfpWQeNrEYw99IrtD", - "SabST+A4/Vwn18cmXMyWJ6APsUHqW+25O7LvtHPd2Rb/TL47WxHzBs47Y8Elht+Y1f3myGvhyAuvm3Pq", - "hTxrRw6K8u4ifk0XDxTw62hvec3NE2/JHsKihzYcxg94RbhAp4fHCMexIFJu/nvb+/RMDZWW8p++HzUr", - "fojQEzsWLiz6obW3VAnksbCDt3bUCLt51QsN+/fbVqUKUeNNVxQkKq+8u789ap3e5Bophd6S1r7dJGuD", - "bamMONRbLKmlB6JRQgrxpTinPhWtsyqbMN7iylkpLln2fHzkDuT92Zdt1zmr3w33wBSPagzxARlhNdXa", - "Lx/+mKj5XbGLDnZ7hfn56yLNwf1JQfdtig6R+WNSF+PasmkuaIBOGi/QV0QZeJO71NNtD4GJnxHhTrUZ", - "6MLMupiW+RQZnBaYEFhiVuu+x+aVdqqvae/PpPnC8txEYrFL/k1EaaHslmu1SsE9trWw706/rSC53XPY", - "iiWwwCKDFXXs3E5gWd3AcsGizW+RK1+cok1cY6nECjdvEheWbIOmVOhZ9yXXHTC/8LqW6axeSxmaJHQ6", - "s06AmE4gVk/5hcxhlDv3MMqiYLjAitgQxceY93uqF9l6gedEeEiN/pW69QcEra5XlRzzWnm7vnv7ukdY", - "xOPCedIsk9onX1hhMvRfyeW9/1P3CPNZqRMPmgTGz9h/E0yOCujO/7XzkwXv/F87P+Eko4z8r92DBCsi", - "1eadEcvgvm66+1ZgHjHxaf2FVhdtiTVtKTz181CrlOnga29CmRXYWWfsLhJpvhj1fXkhr47V20rCuze6", - "15rUfSOWHLuyBGZLuXDoVTVI4n9voe+sQtD3bq+rdE8ljAAEu0cYpcyFQ13FFraaOuK2vIlNAdd7jTGi", - "eKulPcK9/6cySZhJ38goUazrN7tEG7uEv1wrTRN2K+7UOGH6eCDve0FsodWGR9/gtu7BoWMp0oPbqni4", - "S8CtGZcKHj0+7AV7ldCC4vxro6VnsjyQK68PR7rHR11YSCj7D2WHbGrzPfkp3TjuXfG2/d6/0HOQjuk0", - "57n0s6ZTrKIZkRZRICFVBvzYTALl9dxoFPiKqXRwn1fHvev83+j+jqwR9Q01zNsEG6yT+d1bbWV++76W", - "+Q3askVdsLWRuq5u3mZDEojDW25LxhVY6uXklNC4QroIeqcVlVJdQKBB7A/Z/9b6x2+K4PTDDy69Ox8M", - "dp7C74TNP/zgMrzZiSMVwpSgtszpwa9HEOExhSB+qIRagknUx4HSXCpDeq7kyb+dglQGubTXkBwVftOQ", - "WmlI3nKt1pDsXtytilQtm3TvOpKjt9CC23oHf04t6U/uuq1ocDKfTGhECYPiU2D8k0uxykaT++a1vSVY", - "ArOxEl6gY0USaa1GFlxrjYReFv7/kpGE3cYaFBxhpUiaKTQVOCKTPDFVW5Cc5SrmV8yVpIAJuupmtJxP", - "6Hp3TY1cI+EE2XCJ9raablGW8b5VXdvxI81Q5ZmtMG6Vy1K0adYuH5Z471anbHHV3r9W+ZhJzKhvy0uX", - "aQ0hUGLNuHnS3KTzFl+W6Ix9dH7+2qXuavVEuIJ9irsqfa6S85D5Vfr66GVZ/tC84FrQ6gOJbao/JDTb", - "uncxwXFCGYFcByJDWbbV2poPeiy+vAQcLhx63y7uNsfS1sR+OAn4wVjBvciahU9fWYdscTS9mqLFaXHy", - "JpyaR8WvLAMKMJ6QrLeFc8V7Fgxga8YNQmQYJPc0wRFg5OrXDHyjxV8xeK1+UwCqIniSEGFgObNcOXFr", - "yIrBUaZ4UXjaSmYXuvlRzhRNLrom1BCwlSTCbGGx6Yas0pmV+QAjAfA/YISCZGbEtSq6etCU5xLeAjgD", - "v0uEkyu8kENmURXM51B6XZDIINgmSR/9zAHQxlST9hivKeX6nRyyCxonZGTxaC4QlUjOuFCEkRilfE5k", - "tV+CRUKJgEkcYr1yEqV4AcCQBiPXrA/PiAFfrKDecP1vzGIKRUF1z8WU94cMo53BAKUEM2kxLCSewIVj", - "20AwiMqAvkcY7Q1e2K9q+wbg5W75N/RpEoLMeYTHyQIRTcWm2PYmbGBqi/Saqu96+yZUSLNfhX3TVl+s", - "bCyVruZs3EU5K1E6wNafswJUQ2+XygWDeVovIKGiuAYtMNGYRFivJ+PVfgASlkdRLkIXpN5qr1r0v6Pg", - "6E3vDJYqjIGRgMkgIjHsOeNqBmeaw1Ha/L6Bqkqi+nNcNMFDwgXCyKPr0qJBohxY4wZAqF6UpU+ZK2V+", - "sfm9Ozv6+FpG4I6/ATF9LPcTEBGfTCoHcP3VZA7wqtyzZRL+s57TQ1fz2mdxMcVTxqWikWOGDqHaV5q/", - "KYStFMLVKxuk5gkXl80Bxz9xcdlWA3OBkY9LEfNn+BU6IvTwAAT/4f0RYA03yoommntX0ur0VZxSELqo", - "kkVIMEo4m+pTVFrl791t4Gt1GwbQUl+mwji7C/gxrYSM7I+mbLYX1g0uhsi2+tC8SPd+D86oX7lCNM0S", - "khIoq90zxKY3u4SqGy8slF4B2HYzXqlPlY+rYHRBaeIPuk4cArpyG7YB0vvydgWZasKn6wFRi84d+mcA", - "EXXI3klTquDCuJ4uUMGDtUBryo+gqxmNZoCOCnqrbt+Ap+IsuyiA4Tf30Ss4yD4+PnS+YYqOaFqTPCEG", - "9HSephf7y4Wj35+cwEcGGNWUiL7YR65YdHF/SP2Wj3aqZ5FgqdCvFsN1o1DGYUcvFNb6ZjG/TYuDWgL3", - "D1kIE5WRK9sgnaALDx71ogG7z/Hb13wqvxpXUVluxcxFcWRVR6BNwuJOU5AHTcKOn+3BIFQFoCVKqxnG", - "HYO0Lg3mNZ8WpV4qpIyzrC352mECFc/TdAUNow0P8lGqmOfqL1LFRAj42FJ3E3GjDRzZMn/4UhOqBfh0", - "B3sTyC8YymRqLwSXSjPVTrdDWJ529n+z/5qnaafbsePR312pLPVqN9xAyF+DeltvcDn0Ru+QB237TTy/", - "CWhtlel7qLW1G8Sq1c2S+Vvzwp/ea+hsdw9IhiAn1Iy5X5Mo6o23avhhvEDfhZG9uI+RAUQvihIuScXR", - "83gA/qzBqyY7NhuM3Br39PDi3FVEaxPJcmY/PXNffgU6+LqYETdm5KZ778EjyyN4zGAFcmk2Ey7qqHDr", - "okq+ekL6cluyNNU2FPKNNm9ubWxFmFpfWGYR9oPYVMjEueIpVjSC6mzRjHPpkX0B4W7qKFojckGZYGIx", - "2q7NJLjQpHphzdEXVp3Yt6YzhP1Hto8+fG7zD8JfuEflFz951oGC43edCgAVTCTCaCwomaAM55JoqS5P", - "CYoWkeaKphwfwdEMRThTuSBQaZSglDKa5qmPza93bI4BR+hiO73oonGuUILFFLQz89AF3UQ8TQmLCdjp", - "hmxG8Jxq1VKgBCvCokVPEqhQPifoiovLhOMYTA1ZjMHjAxVOBdEUCIUOUqJwjBUGQedCn/iRSWa6KIqW", - "G/WekeuSGuIhEzn73lRd0c1euIFeIAJlBaicFcVtIxwTFgXh9s++bjb25W3SZ0TVJ/pAEUK34qUPGTLk", - "217dcL6OaKJHCw3Rgs2vEHplswpbzQJxZPTveaTNXN0cH8jRVCzxqlP8dXiYCqL7arxMD+9G4gLFuenO", - "O5VA5n9W31DBUPygK8gwNdt4WwdRUcWzWOYb8bytP9yfx7ew5X0lnLDbqNg31YsrJ/01sFy7qrfiuQ9k", - "xLS2JN8m93As2EV2PZj4xIXH5R6LsbWC0FbwbZ87KYFB++LsG9uus20b+HBbtu1ss0uufY+RU9aDWNEw", - "B7dm3EZWbU0H/6ZZKbXZeSzzwVlk6bm4d7hFxxozvEg4jv8MwcIr/EcRF8LAYACwxmOCiPashn6aANjm", - "ykKUXZe1+f7kZLOJSwi1kkcI9Yg5hJeaoz9L42UD7ps5EYLGDgbz8OTIhu1SiUTO+uhNShVSHF0SkpWZ", - "LZBd2Nfzc4AgtWHXkT+6HcKUWGScMrV2FOWrdzOY8gducVG+QlHS1hz45g5v7Q4Hy/7jY2fAZSB3w0xg", - "tWaqsFpbC5myCRepkcvwmOe6dc2D9DLp/TSIBROaELmQiqQmOnGSJ3DcoH6NrVFuvzO73IXYXH1yTNpc", - "RkRKpaScySGzOSMZEbpv/blu3wu0CjoEFC7466lhkl9HEJ8ejIlbw6pp1QC6CWofd/Y7WzjLtmKscEOg", - "mB3eZwzpJ4jKQ3KRjnlCI5RQdinRRkIvjXqC5hIl+o/NlWF9I/juS1dgv/3J0it9zCY8WN/S0GxBzH+q", - "7C7L1pxj8tGxtVfEPyyO/8BGh9na+hrvguCkBzXTHYAPyhVN6EfD6nQjVCoamdQjXKzd+5OCqfaH7IQo", - "od/BkOKWJAbZALTLrUzwaGuYDwa7UUYBBW6XwOCA4TU/TqHHw9N3Jh2VpFwsukOm/wENnx+cGu/uBFtr", - "gjdQW9wdHW+9WRPofAbL9G8cIWgmuBLFILjh31yCN8caaTxDsuGI8myVqsSzP30Iq5XgvtkVHqddAcCe", - "itlsFABfDpUrbEOY8yRP9T/MH8fr8M0Ujmbv4dWvRto1w1nbjZvgoziUdk4xMfV3H8TpYRbsscas6oVz", - "UwAhphINGLwFDtSfkbq/vPneX8ev0N1pV9TVtv5qztZ933x2DA5pw1+Px3LMDaW5mSi+2vp0hWmz9enH", - "hEeX0kKy+GZDrbcBzrr+scTFti5CEBMgQxRZKCMDmEVkd8hqBkiD/CMRRoqIlDKcbMGcTSOA8O2sWHjO", - "KSRqR5Cn0pM0BuykBGC8AQZPzwYMVa4Bz6MrbfU//x3fGak4GpOIp8Shnm+GVLe/Yap+4qIKYf618MVz", - "b/0BGhBTsLevQW1v7vGzUNxP8DWESse5dSi7EW284uWPxhTURbA3w87uQA47XTTs7KTDjt6BQwwmVKzQ", - "E5RSlisi++jI2LcgFffpAEkScRZLB77uLHi7A9mUmGvIsiHL8yl8d59ij6UqWMq3tpMQe9DvIf09JO2g", - "Df/A2TMZd+HQxYjnypj77bmyb8VEgXlk8959td4Z+abbt+Hkf7PHt8KjYJc1u/S23nD2LJcz0mxye20K", - "GuVqDKDergCynKG/87HsIkaujDVcSNVf4nv661PTwX0UHNBd3aTYgJ37t0oDLSoNlGsVBm00AZb6SnbU", - "YZAbyXXGhQI0R5tzb2gINAlAkIAyhW8Oj4cs0qzIQAwKknLgThYX3dzCB387Qy8P33bRERTjRT/n480+", - "esOSha1hb300Q2YkMcO8IszQ2FAtiUPXsxk7UM9dBovrDh6our05GQHPitsrFyTe7cwIjkEi+aPzmpvO", - "AujDb1/rAwQAwObLYts7K4WPzluixKJ3MFFELDd7YvOkWIGdYS9pB0VnBTcDgKk7lA6BrezTyAYGImN3", - "pxNAzPj0rfjD3Rdxvh8vmYkTMWX3xrl6tPVb4SAWzDHEAv3ruiif0JQlbHnZSgUDumyK/P6KTO4reVcF", - "Y/7f9XTBTB+toymr7JMm4qLsylpPr0sOnhlYZOuoinCGI6oWXYSTxN5R9iYoIlJ6hfg7FgRfxvyK9Yfs", - "bVHwxSb0osPTd13nqEUxlZemBeuL7aM3cyJkPi4Gh+CgGa8xrDmJh0xxFOEkyhMtbpDJhESQiwt1XGSD", - "L7cYSucOz07ZSbDojBfVnj+6WndhmoDdK8miTnFbZqu3BIkSTNNmEHIrqEHAIYQajHWjnCHKJokNqYoE", - "lxLZpnokoVM6TmyAkOyj8xlBEqdkyLIEM0YEyqWJitdD72WCSJmbBG/dAID1GorqohJgMBNc2dCEhHMh", - "TTSBpvD3J0gqkq0gs7em5ROY8x3JtqZx29MDGalrY2g2hdhXkN4QQylmwTUd5YkLYLzXUHQzoIeWEh/L", - "wT8XdDolQp8KbJisCcczx9otpzn0lYzlxrqXZ8Vb7epeFq16WYlext5KgLhRibkdd24W9Rfo/JI2Ygja", - "RzfLIv5Ff9Sy72q2angQ9tFnzjJUwvPfsVrmmZck2NaAVVL4YzMneSOvHNVKou16WK3WmbV3menaGj/r", - "wWCzHjNaFq6kzzYpvF8fIQzuF+XhvoutPW7aqqBdVXTThpT/9aj6XwUF3g2c/gOjnNwCTv+ryrsHvPOH", - "wz8JHtSHyqOv+J5d0d0/PSL+XaXPG1h8gGNrSp83XM8Gr65UlN7bd9qpSbbFP5MEb+MdbyC/u2X/pvW3", - "UBm8xVrngtYET9JMLVxAm/VVlkFnkn4k/QZHcBG3eneu4FuEdH458nB02hjQ+eeskf8gMaO2hCCV6Pgo", - "UHz+kWEM+meucrFs6Vunh0U0o3PSbHSvnmC7RJkgvYxn4FyJzYLZ9XB3mcKiP/2IbPMWc9X+C2pQAlQ/", - "iVFMBYlUsjD1QDVHMH18J5HgWhOA51wsmqNEzBH5SfD0wM5mzX1oz5Q1hpVxhumiF2OFe3PHbVaY0D4j", - "utPFU2qGhyhDr35EG+RaCVPpAk205oPopFhSch0REkugyU1/wNuDBssm/UhG03GbUa6oWfLG1oRBUS4V", - "T93eHx+hDaiBNiVM74UW9ScgyWaCz2lM4soYO3OemFXdbljQm9pdtVBRFLBzyoUZ3IPIMG0upOlHmlXZ", - "QhESM6YMw+DWVgWpnimTxK/7w5S5ABy7R24U364wq/ltOGVHUyLU47SLqDg3EM+b3665x3zN+clQ7k6r", - "3HYuPGe18bpdflTLtKW7KPxQ5M7dr9n6/deT0kPlo8zmsabzeaGQNpnNvy4SHNzf/XDf5vL3jzgF9BVx", - "yrdnKocGdIshgnkNMd0xmZOEZynURYd3O91OLpLOfmemVLa/tQWx3zMu1f7ei2e7nU8fPv3/AQAA//8A", - "qZIc0fkBAA==", + "sckmrER/wqvVA/Bk9+nzZ4MX2zttx2c82e3Wp9DL3Vdow67UX1zpM/ekMqidnWdPd3d3B0+f7uy1GpV1", + "J7QalHM9VESVZ7vP9raf7+y1WoWQvvfSAU7XBftQ9YMDUzAI8kFlRiI6oRECoDakP4BU+hllpHDaVs/q", + "GMeuRkT4TlGYJnJluJvpzL5p7LBpniiaJcQ8gw1p5aqAmR9BS8GiW4wVNTlu1pKF6V4bCOXmUrwCYk5M", + "xvl0ajHwiqU7sentpVBFSRLvo0KVW8n/YDfLgX1oogM7h5bU8FqrVBaNziMCc6UZ6AZBUEEnZtMqs3Ix", + "EJRleZAkGpfyp1wYVApoFOExz5XxttsiKmUnBuCFgfc5Z3E7M8hPXFyuTXjQN3RRK6alIcFlvpvM+FrO", + "uy8MFl5xEztgn0v01nxhDIjlz1leLZTXhZ6soZG5dP24sCfbZtpKnWF5BmzpLozG9FfyznsK8etNTFTU", + "l9W8xZRAjSS1VpLRlGLilM7g9db5U/rDtQaWFuvOyNV9LDpkAPY02fYkw9ndrPiqSOLCBuHl73Mbf99H", + "cLog+M8l7tdO2pmDoLBL3h8ym4Dk4RiAgw0wBkz4+IxQgbigU1rtuGp/vcdY3ptQpiOuW1On/2FTbJ4s", + "3IMhHmAQmCcmNNYUBvEBYO2edLqds6KYlGVM1aV5WxTkWlqRMuJ+GYvn9N1NI3IzwSc0BJADEUL2qVXg", + "XKzq673BWW/7ryaaXpMfSGyUmaiilMe12k/2/XYX0avTd6dNYyqqMSF/dEtzWgH7sxwFM8mZqQfl2Vqw", + "rdNokdgMTo/prI9+JFL1yGTChdq3AS7gqPJqweIFEiTONT34kGWcIUnHCZzRoletCOhf9IQAIGScTyZE", + "VNEIXoRkae/tURow+v2knyPzQgGHcPJjVZ7Wcntbbf60Qg2gzk9wRNl0s/V2BwyFtWmswxt8dfrurS0n", + "1ITErJeyKDlkQJj76Nei4JZealmiLfUDFsQaPZnAav3CSCY8FKXykyAEwbOy0Kc+e+DZNti9RTQBjgG7", + "iTNXbhBtQPRXTCT6PccCM0UZidH7n+T3aGBd6e9/Qia7xjIgKv2Ax6qm+Cy0pY1ZcqezhaQRTsyymJIf", + "lPnWSzjSrdWM0/JDa+cNKBvhYnOOfaCN+TTLYQHP3vaO37zfSmMy71bGBFGKM54QPe5Nj8fOHVRDmdJX", + "Ya3zJjOS26qi9uGorLdYPwiAEm3tm3Xq0HszQGMSYQc0Ut+8qChPPIaYDLCGXQmqlJZcsTVGQ1RN2YgB", + "05NokidJ3zJAN2Rjfp4AH/aQTc1VpckHnmcYNKlCjLpW0ClBMo8iQuJaWEt91B697aOcpVjIGU7qk9sv", + "QsCjGRY40vfhd9ffdcL1hfTgZFtG71FncdO0JkvvXgnQo3fumo74+5+qx9Nc9SZp2BrOKaMKQNAA+Xqj", + "GKTB6u6jE7ywdpQM7kxLL+WO4cnEQIEWd4kgCcES6oJL9P6n/tqATcUVTprmcK4fWj61oSekT5EeZhdl", + "FTYAvMvD4PK7fRq8Msr5BNxP0GHdl9FHxyzM9kxUEKDoGUAsKemUad1O2sCgCLNiJZf2rnp3Bs1CddTB", + "TnXhKtPpBi6AEMWEbiuT5bis5izXKeWXVTbFL9fXxjSNNPd76BIxa/4uq0u6cwv5mi5CCZlPLccwVlNJ", + "MiywIsmiFipdrXVAloNNyDWJbpAJ+lK//snUt8kFGamZIHLGk2qI0m53uUayhHSAObFl4cycPJ+c4ijF", + "4hJOmdOlUc7MClSzSXbXQT/NlMpuMKmfz89PjYFNETHHST0fSS4F3xyRBC/QmKgrQpibCpYI++Ho9Xxu", + "2VCaS6hRRgTl1TXs7Ab6PTMpCmgqcESQ+cpVWC/Ymj56bZfS9hIAE40iImXD/m6v2l/76SRP2u1xaFjb", + "63ZYRTfZ4PPDU1diqijh7ZZ5Z3mVT4nomSPnanmv3todubrYmeuKGYzoJZl9TKD6mU0C83O0XeghVHfT", + "n1fyoj3mIP1MXtsPnAKzVF1zzj+0UvDqxz0UY5NiFodKoZvcH4OaMQUoPUhAEDloXzQ2kpK5k7372aYw", + "aUmLMiJlDXIgykXS6XZ6Ezur/a0tze8TQBPd291+vrU6wntlaL+NZBzFdJWJx8U7mog4l+BuoBhh0lWS", + "2MJZ1sIIbtZxzf0A7Gk5lBjKn+u7zVOyXG7JYLCEnnCNI+UKNoJVvBINgf1jC5jfVUFGN5iaGkxa233h", + "n89BMDwlw2pWJf+tJdqHkDUIN9Y0YiyOtXU0RP4xKJ1yEarKx4Wyybhj4uKRi/vQRfs6dOuK33zw3J/l", + "0ydPdp+s40PAbGrH3J67wFTN21W4AeIttz2/tgGInKzKHO5Iry4srNdlDU1pjrhEUqsXlGeE3Wg9n+zt", + "7txsPdtO5NhFbNb4Ugit6fDkyMhEWpfHlBGBUqJwjBWuMhkwJ2suA2WfMEkhY2/y/WrW0hDa5MMv3bZm", + "3ZcKjGkoX/nWobWnmNEJKEnmTb9nOcM7T57umyK7MZnsPXna7/dvCkrzskShabUVWyZ+1sOn6cvZ5+3D", + "HWDPtJnLH53Tg/OfNSPLpTCX1pYcU7bv/bv4Z/kA/jD/HFMWxqxpU5eZTpbqMVdDRXMLCk7ifVSW3ndy", + "T5vQvQZ/ECQOQKhEEAGyEkB9d1CPBY3TakWPG6Ttr0hj1+LKG5YsGtfm1iWWWWELVV5pZd+u0KLMMv24", + "OvTFWZzhHdunsXQUFaiXg1tuVUNcriyzulSFLyOsKKyaJOaviDPA3A5VWa1cke5ZiyJ9cI3YanxFl/6P", + "Re/ej4f+QLzfXZE/7ydbbvXDDaPVVgqkf1uWQ9dzISeOrjnMYfN/cSu0LW1tISKDaRoPfBfeJqKz2vub", + "6X/9/n/k6bO/b//++v37/56/+q+jX+l/v09O33wW/NBqdNAHhfj8YqieEMZYgfZsS0onWEUBG51W/xpW", + "2D4xFgcVzaAYLxqT/SHroddUEWFKO9bykocdtEFAU4KvtLgLdatMSuim/vjUBBXoj/9wYvCnehuxxYsQ", + "dkMKGCCZj2OeYso2h2zIbFvITUSCXqD/ilGEM1OfkTKk9d8FGgsopmm9vGXnXfQHzrJPm0MGVllybYDu", + "Mwz1CCeFb4K5GA07KhOhbl8nBSKMSRYesuK2LuAxjau/X9bSoCSpp/M1LMpq/c1qTs8HISBRSDXTGwn1", + "okAFKShbk1GRA4eeDzaX9bk1OkZBQyvIz8bCmFzkgzxkLm7KXz4iMY2Ar7gU3plN8i6ypw2lWSNeJvj1", + "AvbmrckrjRHO1UzzoshiXkScX1LShS3tgkcagq/gSxNSM+NZb7zozXhW4J9gYQLOsAlKqSrZ/6dnJ9p7", + "TwSd2J6CMBaaRAJCJxwZOzOTDVxYF5Ymdm5K7TAt+syJfd3UbJKmTI/JslC5YK6SDIF6sgC0U1AfCcnk", + "36MooWB1kjPw4c0AC1PpZkJwlp1Bkc2Gx1FMJvV/V6OKdp48BQ3W/Xt3p3UyuVm6VVSWJwGdNnWsrwXH", + "NmwSBmDEg5EzhK+JA9Q3oA2lADuF4vDfM+QaKk9cwUiMp8+kt0pbmCSRXmLrZjAD0R4Di+YzwvY0tbmP", + "lk5hJaGxRQsmQAg+S1rgCL00idTnr8+QIiJ10BYbkd4dOCUGNKZHpcxtjbyDw5OXm/1OEAOt4tKCrVqZ", + "8FgddAAGxQYMNcVBlTYanJIuOj6CRHZ7rZS6GGQe/cQFSsytWF5G+4CjUzX3YFNd8/jICqDJoow6MmLL", + "sLPpWszq19s+eluogLgYSpGSXNKWa7K8TKBZG4Nq0qKWWq9lsIN/zKp/9j6GJCgoPWp4MeC4Nt5f7W2O", + "DhVOX1Q1C9mNLyQ/EKzR/uXt/ZdGPP/yMvruzWR064UeZTMsQ9Q9872a8NLSvvuO7Cq7F81he5V+R5IG", + "z9bfXAEm7xpSRN9zlc9DEL9Petvb59t7Nzff3RSsuopS5yFYFnjV7YGm7wKwOQC/TNWoMb8D6cc2m8PZ", + "Rd6foBmW7DsFD2vWke3dZ22MEtBr28wIPyeCT8yQCi7lIO+KiH4D/ndJk8QIMJJOGU7QC7Rxdvzql+PX", + "rzdRD715c1LfilVfBPfnFrjVcAvAOprgqADqWQWtAxVpvefnr+FwJQQyo4wcfnl7NOu1psUW6NZucK9O", + "34HjH8uRi51uTiPGZSo+uaZSyWXAw1YpCJ+Dpm0+LQ1jbSZp2rBBlWshuX+uYEYHESw37wBL2+WPLC3n", + "A8BMP2T+7tcHcb0SlPpzkaWtneGOgKUbr7QQKHMNrORJ0+12e4joOxlOBespxLZ8CceBK9wak7nboYHE", + "8gMbx4eOT8sibKUzwjVfm9OLnf720+dQR3h70Iaxpzha0ffJwWH7zgc75pbZx+P9KN4Hhf22PitL2EYF", + "wckVXkAdTrO0w465MD3t1ju2VpFsFV+zDH19O6TruhjXgGUN4qwLXJKjdGUZoBYZwnU8wzS3aK0pTRIq", + "ScRZLKsy8gxLJDMDUmzK4RQS/JDBALuoqEoOUgrCUSTy0vRopWsr7+eZpXsoyZtxpnUAqMnxC1lIlFJw", + "ghbdQ+ijREUiWjxkG8IlLRbZiVCNN9Y/QApQ1yaXxF2IGobEb/3BkMlZrjQT2+yjQ85knhJhrbJoTMFj", + "tIlkblRaGC+sxkIzTEljIoZMvxaAQf6jUE/2nw4Gg0G3U2hyu/rfgxA13anzs29hvk06PgBxMgv4DdHu", + "ImcoZzERRWl+YsihHiJ3Q8fpZ+J7u8/biVf281KuCh/MdXDg7XC+PxdcGYbaoJ9DdOgtlPMntxfRW6X9", + "OfnVJvzZr0Y3iWCwKRla4xvr284Y5EhszZCS2FyPIvfmnSmLW526DT1WHP2eE7FA709OKmEPNkOj3cSB", + "SzTsA89utA07a2wka0dzE/eyB0V9H/DTdUnFkxC/ONi072F0OASGQiuGrYrivMq+prXKYA4UZWafNNGs", + "mGCtMrJBcykDI/1W5UJOLPSqFaVdToWFgi+QWUqnvlzIrXEut7KIbtnssS2AzXkOsDl7QfyCmMxHeR5S", + "jfQjh5P07t3xEdqAXwD22STIVLrH+On288HzF73n4+2nvb14sN3D27tPeztP8GCyGz3b3d7ZXZFU1CKh", + "9fY5qkGNORDHXEStj1z0fCiouSl3oSab2HjsK8piflW5/oIBsn7vNvh2XffLofWthxBMCUqwVMZ80cDJ", + "TuCSJ5Fu2wSk26Tpogpa2ND59Hyw/bnWHxhcwx1xLnJm3KoG5qNwIaTegP3Nqo7zdiwfBuQSX9atlt95", + "+0Ub7D95sf/kcxfNJW+sG2OdnO5xc5siwhw8ei07xCUJe3YkZ6DsWJnIWPVtMkmn2ynyXeBvEAZqsdTF", + "41ZJXE0HthtmI6uulQb8guOKvgKRKgYeM97XkorTR6DYSoGSoUWgw4TnMfJscQYtEPxwx57uopsBt5g1", + "0ZnEZZOMAZmRVNp8Pso0Iwb/o27Egh3so1fwLjzCqVHr7CBMaSHf9YbjhYmX0efLdW2UrNVDPrP6FXyj", + "lS2k/wXT1stgTbarmzDS2T76lcM3hbbHeN32a14HNWv59bqdeMMi6TvwGujMipr76KdCvCwEVCuQbkhi", + "/xxZhlViRm1WkDvsjnc0tZQ756FQdDtmRTvdjlsoQKtYxq14V1L90vnzSTEUSEZwAme5TNPPFU0sgj7M", + "hEpFI2mTR/TmNok9NjeTxCOjPDXFpJrUV6tgFR85qer9CdoAkNS/IGvY1v/aLOJXK3fdzou9F0+f7bx4", + "2gryrBzgetH4EJAJlge3Vk6OsnxkbSNNUz88fWdsH5GxKhSxL+9PfISZTHDNevTMXYN+5y/6L3ykt5jn", + "48RzLFq4SAM4DRsWBDkseFFDHOTvNJnTyYT9/jG63Pm7oOn29VO5M95uQLY2HYXNbsd+cMGSjZqMe6aC", + "WRiMCwhKyEa8urdEwgzQGVEI6KeHcATqTZFPbUnOodrZFQ8S1t7u7u7zZ092WtGVHZ13cEZghAtcynYE", + "3hGDN9HG27MztOURnGnTwbpkAZiD2jlDtvz4oCqQ9rcHuyEqabi4S6qxbc/TxiV/b9VHOym76JC8XaiW", + "S6c8uNq7u4Nne0+eP2l3jK15eCSuV3MYl7JklsfWyPB3fgOkyfODUwQJwRMcVW07LkLsRqNSNxoV1Hcx", + "dRluMLDnz54+2dvd2W4HvBgKOrFQo5UDW+VdgUMXIIrAbgSWYpn1dptui5A4ZQjsLYkSTNODyKVY1G4f", + "U39hJMxr5Sa0uRisBr50cbX4tpVxqzBZmQQdIxpwgXJWVP3pr3fJfhHPajPXNtfDeq4eSsthevUsQpip", + "bniLpcwEmVOeyy/QEFcmZ3aScC5u9G2TwvKWyDxRxmZDJXp/8h3wFE1rSCqSVXUoS40rcNRuObkbnecK", + "iYSJvGmxWu1Gm61fNeFuw6ntrgLUqHCDRvTCWHOunK0P/jzESZRDnStc7KeeFcDwASRAliULE9ufJJwz", + "FM0wAyeJ8EDH0IwncT8YCaufjCbBqAp+hRJu8NgvCclsCSgzCP2ZFmHonKANv/ihIaVaSeInqWEytshP", + "lRqfpOHaqmFMpSIVXq8nVtyDCjefVCyhCZ8a2GQFWQv9eoWKDAuTjICZKWk2T40uGQi4DgyxxsxDN6q5", + "SfnEKrhW5IBEc7OSOBJcenBg709q+csrct6KLOb1AZ3VwbYgXePQDFxlBpCudeXD0P0YyOf5nBsSaBhy", + "BleESjrjZIpZDkWhPEK2hvh+63DIGZdqVMBc3XCwUo2g0ksuSAlYWWTdF/Yg907wXnSs7TbLZeOOb/X1", + "ElWFm2oaYDNPDa5oeLW6BQ2GyHgZV24llF0J8FcHs7oJXmRZGoRKaJV6yIFoA3JePLbkoUButgmSCaus", + "up8lbdUW7n29Nzhri6y4GkjxFKvZMZvwAPbHDTynzhJto1UzIhy4YUwYJbHTJQsXqjV1QcJ4IgmKc2JX", + "zsinPvwdhDhgNQN7JHxI2bTG6+sdtjEPmzGsLgQD/doX24Q7yXBC7bnIYa1MvKJEuEytbRUESuUo7M5a", + "bliQaZ5ggSwmapshy0WaUHbZpnW5SMc8oRHSH9T94hOeJPxqpB/JH2Aum61mpz8YNVUNOzODs3mBZkNq", + "/ZZT+EHPcrOWlQyWmC3z/RY4RttEjwUjxX+iCbHYlO8YvfYIvVoRYW9n0JQt39BoJU9+GZz1ppzbkmzw", + "xOcykFu4Uspx9c5cTQor9mS5NKWXWtxKDgfZuQBv59GpJo58HjTJoeHXNWASNCaQ9+Omtsw1WrDFNlMJ", + "Vn3J5Qz9nY+rBtG2Yb+BWoIbrITIEGQSjO83VUZWGaTNG0tr4u3uTTAogK3qicJHN4R2WFd1sYyvauIn", + "b5cKEM6IXTLq5miKEbYoruPiPwr4AttrexyDeqnIQLwyoNRItYCiy1DpauHVP5VozIUAEHgt4XDmZgOw", + "K1rm0WvtcK/Q+YwskCAppmzIKCuMpACmRhAjcyK8LFkutJI1JXEf/c1T8QA2P83UwtZjAOP5dxLxK1aM", + "ccj8QerGc6nbOWDGsijyTFUquepmQevThAJZy+AEUwJKm1I1QxNB5Myfe6icrZbxrriIG+uELZB7BcpP", + "gY8VKX5JmM/KimaCqqFpaGS+Wo7iM7Wo4anVP1GlPDSql39e3V8uiQgLicWUildaha54R8VTTgwIDCCi", + "QNlP+5dh8QUKSgvMk7L5v7omy59Oi8arv9Ve83BNHNL3gTHbBk2wkUnjqQX7VD1pa0NVIA1uFZrNsi8B", + "bbgQalcjqSoJeLWKWt2T7TLx6skCbjRbkkTV3veeP3n2tGWxqM9y1hn0ri/tmpunK1xyDTt10sbv8/zJ", + "8xcvdveevNi5kYfF5ZU07E9Tbom/P2iDXCt9WJN//eOf709qXp8nEIM9uNGgTGZJeEgN2SXVAb0/+dc/", + "/ulGdesBhRjNMkh/g9++MUon8XfSBQpUXXjtnGQr9PuDipEAF2wGbRCA4qZzMjLr1isHU4MBaScF4wxH", + "VC0CjBxfmWj34pUa1nYbd1B1sCGR17RtUVE155L5uEw63XCdo/80vuEaLTxvXXNO5uMmP/Sbeq/GC116", + "LfwYhxYhBoYiwgbuYj5XWFYCuvXfEeRduAyz5Wwb88Zq1N16KgREsdiSi14oYKjWQE2etB/521/bTs9v", + "WTHr1Ff8w4pz2HwEb2T1DdzIAaNvtD61tsYf7AV4u69GY79K5MoynJWSkuWte/N+W2QPL5cqKW6wm/fn", + "JUze5MM6JjDQox2DXfKy7W6FJBqoycuFCRjQeEJ6XvECU2pL5sYjqM+8hZkPZHBGl3wyqWLdPmnGRgfY", + "H0j2cr1gpbRm0kXk2tks6sDaBuNn2Hkihx2tAgw72+mwU3NbBdMnU3w9sh1UsV0Gq8DKy/T32iClm8E4", + "4dGlKXQIdfX7aIBSgplEOYPDX/OqbQ9We4e6nczbmwIanJgQpyW2BWMakxmeU6inYn0q00ogJrmmSkLA", + "KLSzj2Ju0J4q1Z/tDPVrJrlxv5w0XDqYLWzDukH9HmcuorV8Fwx8E6g5zT4SwbsWrEBz7DdvTromgAFC", + "D83AKvGNbqJmBJpBFl3UyiuUv4fjh8cJGcG463D96fI6+jnp4FkVRBIlLX53SQ41IkARz5mq4/in7RS5", + "alrZ8pWUMwj2s+EfgMtmezcEgmISwYmUy2exSui3IO5a3oBd6VDiwG6IhOFQrKi/89Y6hOsDMMYGr3C7", + "aceP6zZewpFU3Fb0K071iFxHhMR1wM/wK21j5e2XwVj519hiBBU11e3bEO+8PLv+3SV4wVibVtuP6Wec", + "9QCdxG2pRRIx0IAWq6ZKaBXocQ/SYhSCVw290CbjmlyvXutfybUCfPQ4TwzoXZh0Lauyl9G6Fb91ZmPT", + "geaCrK2QeQeVI028+a1qR9pQ9YcoH2nfupOSkUu7c0aUe/fMklHjDlULvVRcWi7g371SjbExpNRF9oJH", + "2+lmjQT3ZmGriAXlbZmjyXBKRpkgE3q9gnjMC0YxrsKalAepyGAw+KIbKb5Ge8+g9JesjZ3R6Uwli2oA", + "zl4AS+mz6qoKoghzhsI2O1/upvtwOdrNbqffekg4PvOggZZKmliRdLQKN/uw9LZZ63yGF2DFaXQSPtvd", + "Gwx2dwa3As52w7rBch2Wn9gqpNV2mlLqvO+so78Speq3UCRZL5e2NuXqPKekVILgdB8SbzIcEZSQCYDk", + "FQmt6z2L9a5XD94KVDaLtqB/t1F235wPvloyp+jKYo67aXScc7GKQeQ/X+MQbWAz0RKkXiDnbrc3eHq+", + "vbv/5On+9vZdgF0Xi9SU7fHs4/bVs2QHT/aS54tnv2/Pnk130t2gHnZJTWWgNrT6i363McqmvCSrWEYV", + "loY27BwyIuo1y+u1/iVJKCM9WWRIrU9TXMELjP997fm/mZ3fzGCl7HBWnaQvQmBVLk6Fsh4Gf8tOZqXv", + "oj6b46PVs7hVBlJ9IGF6qw8FyKvdYKBCxXbnM5EZctbyGnrnvdj6IlqZFbfuKgp52OGkB3e5YcVD5F0D", + "ZvBmveoCX77kArbTKRdUzdLVt0XxWgEjDnHTH6WKq3hPfXQ8ZVxrT/7PRZicr0TpjzvdTvJxr3pm7O/t", + "kb8sAnFBgHarfamgRRhZQuYkWb0K8EqpeAgTya51dT3mH7Z72y8gDiH5uPfDoPeiGnHQNavlL9+2e7vy", + "66DNGvolAF3pqO0XN4q4duu5ioJ+oaECduW9bLGJLY2X5eHd1eESbisbXD5e2uMakk+jAPq5kp693Ea+", + "0BSTBC9C2PSeoVbWtEefyNCYTCmTbey2u4PCcPskHXb66MAChIMuq3jRj9+8phWfTmiakphqGdOo/s0Z", + "DDstbXF1XeJmtUncVwFprR8W116sh0hYl3C17prsf0Y+7mdpv+003lXoHWBXcyoqYIjBi11EJwizWoFS", + "V6zaKB2QGAnxavsOqK0kWcsDZCkHOjtJF025QmUKfUt7W86a7YLF+Mk12FtXYGYYgtj5IoAoBYAYXcW+", + "jo9QJnicR2X+aAKDLhE/RF6DaFsh5K8Pyb1L+wYkZk+4QOvtG00GjXb2yab9rtkmNcE2b/X2YP1W34lR", + "pNvJs3g9DzMvteNgN0JuX5OCGDDRVJe9Jgl6k/nQgqO/9VdwWec1tuRIi0R55hwsmqaWKSngbgEXQyiu", + "94gkRF9Ty40gnsRllgSVJRddz1K3nz6fNbk4wSO1PJBfCMm0rgL4R9BfitkiODBXdrS4SzYGDu1bGodX", + "z5QrsqtVHdyztZJY41b5JtymEgqGy9ds3gYv5dIzfxcY375otoyA4hh+RUh721wCwH7pwt4a7cd3YZZ7", + "SCHtjXU91GBbHahwgY7u+i9jgbVYVyXevZB7PkQW53gKtfLW+FEaSSVUVxKcHBgpbItbQYFN/U8EVUSl", + "A2xxAgJ87jJ6bbFBRBXCU1yvHNFUltJ4otfHEDUXejm3JoImTN56Oqxvfj/o/Y8xt6NRf3/rh7/8370P", + "/xk0u9cMCJKIXkwmEHF1SRY9U4VJ4Wn1Cv3DlIDQWsXUnhmCUzCmAdq75Ur+eJ8MCu65+BWnS1OAUDWv", + "hNL22gn95T+aA728ZXwHF8bas/vZFVLuopKs4u5e3kiJmLqgepdRt9kfMkjSuyQLibzCbFa2cyf2O1l8", + "4oXiowtD7n3C5hdoTKHSpRwyrd7jKCKZVqtsrR9qyrVzYMOC4MRvxxaIc+fFemZNYAVB70+W4IzfvDv/", + "8c27X49Gb05f/npwPPrl5X9DrMtVz/QQ9zTt7T15aou0+yu5HSwUcvN6F310YvMVbMzDJAfNHgDLJEpz", + "lUN0DLmOklzSufOUquT2lS2Ws5ZvXyniM6GQlUpC4RkWsjuhEwIBDnCv2ugiKh0xUgnV7a2VhzK0LLoY", + "whl24EpR3EkSwboieivCq11ubHXRn87a3SUGJDZw2CH1Vx+2gBX1NZUA3OGCYLyX0Qak0LgSvC6DePNm", + "oLUHRYPBEMwvXGlp8OJLVEN9t7L86ZwnPa3nNZSMCJrVzVoEUwigKZOa0WnyvkzHAWXG2rindIoDDpeQ", + "Y+WLVC11A1qbOra0/43l28IJHUf1ehrmWJqlqtV/qFlLpOo153ukWrxvQCAGiGWTxEu9IMNqxm7K1Jat", + "LhwCDok5oLqvStsuT5mDiezBR+uzkVcqmN7MvJE0782J06Nqmt6KBTrVS3M1I4J4GwEflHUKbrhkNkGp", + "BRyNqc6YEVEG77rsJi2eg99doo3CBOaWoEi7XvYLrK5DcYKvix7Ap4TlkiMW5lHWwdp+9SPUDHjran/S", + "iWsChlHTcsMI+VUqWrUmjqqWN8OnquV5m/eDB8/yqhXcr+ls1Yiz7KNCmiF6/Bum6icuQC9uBn+5c6B9", + "uPxjIgAMrw6j3wqDnqYkHvFcrT7/+jUtPZorv6gPW9YXdjYADEQcVfKam3iBgycpx7C80no5SJQLqhZn", + "er1sVDvkg7qivrCQ0BH8XHYMhVQ/fQLr+SSQOfOKMCJoBGVq9XlMMQONCb0/8aoVmsKVS8C1IAK9OTy2", + "dheHfQx6NFVAei4A9eD0uNPtzIkwtofOoL/bH8BhzgjDGe3sd3b72/1BB7SqGUxxa5zTJLaJ5FajLjT4", + "49hKQj+6l/SXAqdEwRe/BSARIADTvg4qCJ56SmSGqbBaZJYAVIMhGKq/hroL7kLdN7dy1yx7a+Mx5FtD", + "GhDJ3tjN/QCCMpwdmObOYGAR3pW9fiGJyWRObP3dhtGW/baS6uwSBcoQLKl5TrYslv5Tt7M32L7RmFYN", + "Bc5uqON3DNtsZgLa+ZMbLsStOj1mJj/RZpvbuDD/xAEh+Wfttw96z2Seplgs3IL5q5Vx2SQYE4mwe9fo", + "cUqiSLMKKJbUR28YMc8RVgibEG6RM6gx7T7UFFo9BaZtt8kFWtOPPF58sSWs9OFsFJ+q7Ewfl09L9Pzl", + "aKcg4+WNtI8c1Lih2nsgoB9xUSD9wU7K3uDF3Xd6yNkkoZFCvYKAbWA2lRD7lABwugNh4gL9nnOFUZHX", + "8IiOtJVZxwW5dcuraOsPGn8yxzshIX/AKREpZiZLxLyz5tAvHWfjmymP88pbzRH+8VHH3lQOjchcVCDI", + "VY+of23VhcHl62gvAEVh+zTTix+Q8Pfu4YTbyRY1ch/yyEFlUpRL8piOk/U1jkshJCjLvSLqa6H5wX1e", + "Wbaawp/wFD0WAn5FCgmv3K2lS2ErEzkzCnBQAnxbZm7a776rCn/n5RMvXAj8GrppqOuhjIMZx4s+cmtq", + "lH61AKwpQWCe8fK1cqqH97WcsJ37OGEw48JT9O2a+nZNrTrlhlrcFOBgeqe8hQ3iRhaIP5/94cbWh2+2", + "h/a2h1aWB0aurHXh73zcRzY0N+IxQXLG8yRGY4IM8JMLwlFY9KcfERbRjM4JoPtBtbo8UTTDAkJsUhRj", + "hY0PvdEwsdIsUTS3pZvruYDMcoHrgB6SjCAEZtQExFmGYlLGSIz0JzZqpsRVXKqrbs5+0MBeNFhejehq", + "xiUpgA2Z8m5zyPOWRjuGZvtDdm4Rb/UCQlS54zWSJIDbu8L+wxnCQ2Y/+N6xEBcRJ3Faci4sADyRGohO", + "sy3LOX56pCMZ8RDo0DlhmKmezEhEJzSy07okCxvYGmywVQEqPWA3zvcnReYK2tkMA9dB6FIYpfioeIYs", + "JVX9NwyiwaMkj0snl8NSwmKMkyRYoWSa8DFORmZ9LknAJ/gK3rCLUjpcSm8S4zExtfSzhZpxZv7OxzlT", + "ufl7LPiVJGLY2ewPGWSk2LUmcbcUENEVVLRLM67PmeCp6XPLDHHrj0uy+NQfsoM4pcxRBHyCE8kRuYbv", + "IG4MwEMM92qgB3Oawn7ww1wqnvoQsI7uzDB5rrJc2dQaSVQ3BH86ZIqjPxzI5aetP8oeP4GzmOBY04n3", + "ipkSyNZNo5YjrGc/glcD7nYCCzDs6IvUhHlMBWbK4JcWKJ1o6m/pRlEmAkrH1lc4wgxlPDMlNoCoZliT", + "XKUNAK3ASYIUHCX3rRbcYScb5mMxCNNxIwChQYyrHSPK0MmP3mEa7D0PnydJIkFCESX/dfbmVwS3st4D", + "81oZrmVyW5gWGFCcg+vU8bSXOJoh46iCqorDDo2HncKdG2/CWHNpw2V6PfAp/qCH9oPppkvjH/p93ZRx", + "V+6j3/4wrezrs5SlBhB12PnURd6DKVWzfFw8+xBe0CYct7MKI0Ab5prbBE6CKUDueDe+uSIxixG3t0Cy", + "QBiVHMgPXBlThsViVUZlYOntCvKJiWT0FuOPIUQuDjv7Qxe7OOx0hx3C5vCbDXAcdj6FV8B6LZtL+MF9", + "Vjg3CyJ6OhhsrocEt+sb8Fm2cAx8YR2wUSsq6o/qHbR4tH8u/8C/tf5ZuH4w052XGE1G8XfG90fogPAk", + "dl8TDbggamI3ZhFJnNi93tBz/84DvVkRSZL7JtCHIs/CPVaULHhU5AibVR6jleb7B6a4wX1dKhWz/cPQ", + "76Oznwes59Z2TuYu1DlcsAXAeKwqjczLCEt0BmPqnWnl+yX82rf/dbofgEteJHx6sW9Ud5TwKUoos/kA", + "XqCyFg/sWsJHBo+n+M7C87hqeRtGkvjXP/4Jg6Js+q9//NOC3P/rH/+E475lcOag2PbFjGChxgSri330", + "CyFZDyd0TtxkoBwumROxQLsDa/OHR8ir+W+lNDlkQ/aWqFwwL2/CFK6TtkHrKtDzoSwn0uIZQZrQxFbV", + "MbGNAbuNO8tmKe/1RHcDuJAwA28C+lZ0NACgetRUHLeaaCdsMjVzrhhN62GaS8F66/mLItfKUG/PDPCG", + "DAaWOHTu4IGdNNo4O3u52UegbRmqgMpJoDuUzVg1ov+NJ63nSYajVBkKrLLhTRHO8Jgm1JkcG8q+mCOY", + "4mhGGSnjiwvQddfEvhup5jEHp8fIBkJ24dUhe3O2BSZWRSKVC9K1nEBYqNWyLhy3eS7QA/AvqiA6rGff", + "HbIJwZAndHxkmICHRl4kRhYNM0A0gRhXqiol6LpDZiB1LYSzPngpj0kCH0H/U6zIFV50UVH015WJSbDS", + "CrHs6peHzKQa2jXoAWYL8obZB35mhtRzkbw2Z0uQSaJVY4jAN/XPoe+NCRfIRjh3y7xS153JNjXD0ouW", + "4ujNmZ7fFDRBbuyB0NKbM7cbm10kOYoSCtQQYTZkUwgEcijGnFV2tUgom2ER9yKuLwEf1eqS8auExNMm", + "HnvoE9kdSjKVfgLH6ec6uT424WK2PAF9iA1S32rP3ZF9p53rzrb4Z/Ld2YqYN3DeGQsuMfzGrO43R14L", + "R1543ZxTL+RZO3JQlHcX8Wu6eKCAX0d7y2tunnhL9hAWPbThMH7AK8IFOj08RjiOBZFy89/b3qdnaqi0", + "lP/0/ahZ8UOEntixcGHRD629pUogj4UdvLWjRtjNq15o2L/ftipViBpvuqIgUXnl3f3tUev0JtdIKfSW", + "tPbtJlkbbEtlxKHeYkktPRCNElKIL8U59alonVXZhPEWV85Kccmy5+MjdyDvz75su85Z/W64B6Z4VGOI", + "D8gIq6nWfvnwx0TN74pddLDbK8zPXxdpDu5PCrpvU3SIzB+TuhjXlk1zQQN00niBviLKwJvcpZ5uewhM", + "/IwId6rNQBdm1sW0zKfI4LTAhMASs1r3PTavtFN9TXt/Js0XlucmEotd8m8iSgtlt1yrVQrusa2FfXf6", + "bQXJ7Z7DViyBBRYZrKhj53YCy+oGlgsWbX6LXPniFG3iGkslVrh5k7iwZBs0pULPui+57oD5hde1TGf1", + "WsrQJKHTmXUCxHQCsXrKL2QOo9y5h1EWBcMFVsSGKD7GvN9TvcjWCzwnwkNq9K/UrT8gaHW9quSY18rb", + "9d3b1z3CIh4XzpNmmdQ++cIKk6H/Si7v/Z+6R5jPSp140CQwfsb+m2ByVEB3/q+dnyx45//a+QknGWXk", + "f+0eJFgRqTbvjFgG93XT3bcC84iJT+svtLpoS6xpS+Gpn4dapUwHX3sTyqzAzjpjd5FI88Wo78sLeXWs", + "3lYS3r3Rvdak7hux5NiVJTBbyoVDr6pBEv97C31nFYK+d3tdpXsqYQQg2D3CKGUuHOoqtrDV1BG35U1s", + "Crjea4wRxVst7RHu/T+VScJM+kZGiWJdv9kl2tgl/OVaaZqwW3GnxgnTxwN53wtiC602PPoGt3UPDh1L", + "kR7cVsXDXQJuzbhU8OjxYS/Yq4QWFOdfGy09k+WBXHl9ONI9PurCQkLZfyg7ZFOb78lP6cZx74q37ff+", + "hZ6DdEynOc+lnzWdYhXNiLSIAgmpMuDHZhIor+dGo8BXTKWD+7w67l3n/0b3d2SNqG+oYd4m2GCdzO/e", + "aivz2/e1zG/Qli3qgq2N1HV18zYbkkAc3nJbMq7AUi8np4TGFdJF0DutqJTqAgINYn/I/rfWP35TBKcf", + "fnDp3flgsPMUfids/uEHl+HNThypEKYEtWVOD349ggiPKQTxQyXUEkyiPg6U5lIZ0nMlT/7tFKQyyKW9", + "huSo8JuG1EpD8pZrtYZk9+JuVaRq2aR715EcvYUW3NY7+HNqSX9y121Fg5P5ZEIjShgUnwLjn1yKVTaa", + "3Dev7S3BEpiNlfACHSuSSGs1suBaayT0svD/l4wk7DbWoOAIK0XSTKGpwBGZ5Imp2oLkLFcxv2KuJAVM", + "0FU3o+V8Qte7a2rkGgknyIZLtLfVdIuyjPet6tqOH2mGKs9shXGrXJaiTbN2+bDEe7c6ZYur9v61ysdM", + "YkZ9W166TGsIgRJrxs2T5iadt/iyRGfso/Pz1y51V6snwhXsU9xV6XOVnIfMr9LXRy/L8ofmBdeCVh9I", + "bFP9IaHZ1r2LCY4TygjkOhAZyrKt1tZ80GPx5SXgcOHQ+3ZxtzmWtib2w0nAD8YK7kXWLHz6yjpki6Pp", + "1RQtTouTN+HUPCp+ZRlQgPGEZL0tnCves2AAWzNuECLDILmnCY4AI1e/ZuAbLf6KwWv1mwJQFcGThAgD", + "y5nlyolbQ1YMjjLFi8LTVjK70M2PcqZoctE1oYaArSQRZguLTTdklc6szAcYCYD/ASMUJDMjrlXR1YOm", + "PJfwFsAZ+F0inFzhhRwyi6pgPofS64JEBsE2SfroZw6ANqaatMd4TSnX7+SQXdA4ISOLR3OBqERyxoUi", + "jMQo5XMiq/0SLBJKBEziEOuVkyjFCwCGNBi5Zn14Rgz4YgX1hut/YxZTKAqqey6mvD9kGO0MBiglmEmL", + "YSHxBC4c2waCQVQG9D3CaG/wwn5V2zcAL3fLv6FPkxBkziM8ThaIaCo2xbY3YQNTW6TXVH3X2zehQpr9", + "KuybtvpiZWOpdDVn4y7KWYnSAbb+nBWgGnq7VC4YzNN6AQkVxTVogYnGJMJ6PRmv9gOQsDyKchG6IPVW", + "e9Wi/x0FR296Z7BUYQyMBEwGEYlhzxlXMzjTHI7S5vcNVFUS1Z/jogkeEi4QRh5dlxYNEuXAGjcAQvWi", + "LH3KXCnzi83v3dnRx9cyAnf8DYjpY7mfgIj4ZFI5gOuvJnOAV+WeLZPwn/WcHrqa1z6LiymeMi4VjRwz", + "dAjVvtL8TSFspRCuXtkgNU+4uGwOOP6Ji8u2GpgLjHxcipg/w6/QEaGHByD4D++PAGu4UVY00dy7klan", + "r+KUgtBFlSxCglHC2VSfotIqf+9uA1+r2zCAlvoyFcbZXcCPaSVkZH80ZbO9sG5wMUS21YfmRbr3e3BG", + "/coVommWkJRAWe2eITa92SVU3XhhofQKwLab8Up9qnxcBaMLShN/0HXiENCV27ANkN6XtyvIVBM+XQ+I", + "WnTu0D8DiKhD9k6aUgUXxvV0gQoerAVaU34EXc1oNAN0VNBbdfsGPBVn2UUBDL+5j17BQfbx8aHzDVN0", + "RNOa5AkxoKfzNL3YXy4c/f7kBD4ywKimRPTFPnLFoov7Q+q3fLRTPYsES4V+tRiuG4UyDjt6obDWN4v5", + "bVoc1BK4f8hCmKiMXNkG6QRdePCoFw3YfY7fvuZT+dW4ispyK2YuiiOrOgJtEhZ3moI8aBJ2/GwPBqEq", + "AC1RWs0w7hikdWkwr/m0KPVSIWWcZW3J1w4TqHiepitoGG14kI9SxTxXf5EqJkLAx5a6m4gbbeDIlvnD", + "l5pQLcCnO9ibQH7BUCZTeyG4VJqpdrodwvK0s/+b/dc8TTvdjh2P/u5KZalXu+EGQv4a1Nt6g8uhN3qH", + "PGjbb+L5TUBrq0zfQ62t3SBWrW6WzN+aF/70XkNnu3tAMgQ5oWbM/ZpEUW+8VcMP4wX6LozsxX2MDCB6", + "UZRwSSqOnscD8GcNXjXZsdlg5Na4p4cX564iWptIljP76Zn78ivQwdfFjLgxIzfdew8eWR7BYwYrkEuz", + "mXBRR4VbF1Xy1RPSl9uSpam2oZBvtHlza2MrwtT6wjKLsB/EpkImzhVPsaIRVGeLZpxLj+wLCHdTR9Ea", + "kQvKBBOL0XZtJsGFJtULa46+sOrEvjWdIew/sn304XObfxD+wj0qv/jJsw4UHL/rVACoYCIRRmNByQRl", + "OJdES3V5SlC0iDRXNOX4CI5mKMKZygWBSqMEpZTRNE99bH69Y3MMOEIX2+lFF41zhRIspqCdmYcu6Cbi", + "aUpYTMBON2QzgudUq5YCJVgRFi16kkCF8jlBV1xcJhzHYGrIYgweH6hwKoimQCh0kBKFY6wwCDoX+sSP", + "TDLTRVG03Kj3jFyX1BAPmcjZ96bqim72wg30AhEoK0DlrChuG+GYsCgIt3/2dbOxL2+TPiOqPtEHihC6", + "FS99yJAh3/bqhvN1RBM9WmiIFmx+hdArm1XYahaII6N/zyNt5urm+ECOpmKJV53ir8PDVBDdV+Nleng3", + "Ehcozk133qkEMv+z+oYKhuIHXUGGqdnG2zqIiiqexTLfiOdt/eH+PL6FLe8r4YTdRsW+qV5cOemvgeXa", + "Vb0Vz30gI6a1Jfk2uYdjwS6y68HEJy48LvdYjK0VhLaCb/vcSQkM2hdn39h2nW3bwIfbsm1nm11y7XuM", + "nLIexIqGObg14zayams6+DfNSqnNzmOZD84iS8/FvcMtOtaY4UXCcfxnCBZe4T+KuBAGBgOANR4TRLRn", + "NfTTBMA2Vxai7LqszfcnJ5tNXEKolTxCqEfMIbzUHP1ZGi8bcN/MiRA0djCYhydHNmyXSiRy1kdvUqqQ", + "4uiSkKzMbIHswr6enwMEqQ27jvzR7RCmxCLjlKm1oyhfvZvBlD9wi4vyFYqStubAN3d4a3c4WPYfHzsD", + "LgO5G2YCqzVThdXaWsiUTbhIjVyGxzzXrWsepJdJ76dBLJjQhMiFVCQ10YmTPIHjBvVrbI1y+53Z5S7E", + "5uqTY9LmMiJSKiXlTA6ZzRnJiNB96891+16gVdAhoHDBX08Nk/w6gvj0YEzcGlZNqwbQTVD7uLPf2cJZ", + "thVjhRsCxezwPmNIP0FUHpKLdMwTGqGEskuJNhJ6adQTNJco0X9srgzrG8F3X7oC++1Pll7pYzbhwfqW", + "hmYLYv5TZXdZtuYck4+Orb0i/mFx/Ac2OszW1td4FwQnPaiZ7gB8UK5oQj8aVqcboVLRyKQe4WLt3p8U", + "TLU/ZCdECf0OhhS3JDHIBqBdbmWCR1vDfDDYjTIKKHC7BAYHDK/5cQo9Hp6+M+moJOVi0R0y/Q9o+Pzg", + "1Hh3J9haE7yB2uLu6HjrzZpA5zNYpn/jCEEzwZUoBsEN/+YSvDnWSOMZkg1HlGerVCWe/elDWK0E982u", + "8DjtCgD2VMxmowD4cqhcYRvCnCd5qv9h/jheh2+mcDR7D69+NdKuGc7abtwEH8WhtHOKiam/+yBOD7Ng", + "jzVmVS+cmwIIMZVowOAtcKD+jNT95c33/jp+he5Ou6KutvVXc7bu++azY3BIG/56PJZjbijNzUTx1dan", + "K0ybrU8/Jjy6lBaSxTcbar0NcNb1jyUutnURgpgAGaLIQhkZwCwiu0NWM0Aa5B+JMFJEpJThZAvmbBoB", + "hG9nxcJzTiFRO4I8lZ6kMWAnJQDjDTB4ejZgqHINeB5daav/+e/4zkjF0ZhEPCUO9XwzpLr9DVP1ExdV", + "CPOvhS+ee+sP0ICYgr19DWp7c4+fheJ+gq8hVDrOrUPZjWjjFS9/NKagLoK9GXZ2B3LY6aJhZycddvQO", + "HGIwoWKFnqCUslwR2UdHxr4FqbhPB0iSiLNYOvB1Z8HbHcimxFxDlg1Znk/hu/sUeyxVwVK+tZ2E2IN+", + "D+nvIWkHbfgHzp7JuAuHLkY8V8bcb8+VfSsmCswjm/fuq/XOyDfdvg0n/5s9vhUeBbus2aW39YazZ7mc", + "kWaT22tT0ChXYwD1dgWQ5Qz9nY9lFzFyZazhQqr+Et/TX5+aDu6j4IDu6ibFBuzcv1UaaFFpoFyrMGij", + "CbDUV7KjDoPcSK4zLhSgOdqce0NDoEkAggSUKXxzeDxkkWZFBmJQkJQDd7K46OYWPvjbGXp5+LaLjqAY", + "L/o5H2/20RuWLGwNe+ujGTIjiRnmFWGGxoZqSRy6ns3YgXruMlhcd/BA1e3NyQh4VtxeuSDxbmdGcAwS", + "yR+d19x0FkAffvtaHyAAADZfFtveWSl8dN4SJRa9g4kiYrnZE5snxQrsDHtJOyg6K7gZAEzdoXQIbGWf", + "RjYwEBm7O50AYsanb8Uf7r6I8/14yUyciCm7N87Vo63fCgexYI4hFuhf10X5hKYsYcvLVioY0GVT5PdX", + "ZHJfybsqGPP/rqcLZvpoHU1ZZZ80ERdlV9Z6el1y8MzAIltHVYQzHFG16CKcJPaOsjdBEZHSK8TfsSD4", + "MuZXrD9kb4uCLzahFx2evus6Ry2Kqbw0LVhfbB+9mRMh83ExOAQHzXiNYc1JPGSKowgnUZ5ocYNMJiSC", + "XFyo4yIbfLnFUDp3eHbKToJFZ7yo9vzR1boL0wTsXkkWdYrbMlu9JUiUYJo2g5BbQQ0CDiHUYKwb5QxR", + "NklsSFUkuJTINtUjCZ3ScWIDhGQfnc8IkjglQ5YlmDEiUC5NVLweei8TRMrcJHjrBgCs11BUF5UAg5ng", + "yoYmJJwLaaIJNIW/P0FSkWwFmb01LZ/AnO9ItjWN254eyEhdG0OzKcS+gvSGGEoxC67pKE9cAOO9hqKb", + "AT20lPhYDv65oNMpEfpUYMNkTTieOdZuOc2hr2QsN9a9PCvealf3smjVy0r0MvZWAsSNSsztuHOzqL9A", + "55e0EUPQPrpZFvEv+qOWfVezVcODsI8+c5ahEp7/jtUyz7wkwbYGrJLCH5s5yRt55ahWEm3Xw2q1zqy9", + "y0zX1vhZDwab9ZjRsnAlfbZJ4f36CGFwvygP911s7XHTVgXtqqKbNqT8r0fV/yoo8G7g9B8Y5eQWcPpf", + "Vd494J0/HP5J8KA+VB59xffsiu7+6RHx7yp93sDiAxxbU/q84Xo2eHWlovTevtNOTbIt/pkkeBvveAP5", + "3S37N62/hcrgLdY6F7QmeJJmauEC2qyvsgw6k/Qj6Tc4gou41btzBd8ipPPLkYej08aAzj9njfwHiRm1", + "JQSpRMdHgeLzjwxj0D9zlYtlS986PSyiGZ2TZqN79QTbJcoE6WU8A+dKbBbMroe7yxQW/elHZJu3mKv2", + "X1CDEqD6SYxiKkikkoWpB6o5gunjO4kE15oAPOdi0RwlYo7IT4KnB3Y2a+5De6asMayMM0wXvRgr3Js7", + "brPChPYZ0Z0unlIzPEQZevUj2iDXSphKF2iiNR9EJ8WSkuuIkFgCTW76A94eNFg26Ucymo7bjHJFzZI3", + "tiYMinKpeOr2/vgIbUANtClhei+0qD8BSTYTfE5jElfG2JnzxKzqdsOC3tTuqoWKooCdUy7M4B5Ehmlz", + "IU0/0qzKFoqQmDFlGAa3tipI9UyZJH7dH6bMBeDYPXKj+HaFWc1vwyk7mhKhHqddRMW5gXje/HbNPeZr", + "zk+Gcnda5bZz4Tmrjdft8qNapi3dReGHInfufs3W77+elB4qH2U2jzWdzwuFtMls/nWR4OD+7of7Npe/", + "f8QpoK+IU749Uzk0oFsMEcxriOmOyZwkPEuhLjq82+l2cpF09jszpbL9rS2I/Z5xqfb3Xjzb7Xz68On/", + "DwAA//+uIZKx6fkBAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/paths/paths.go b/lib/paths/paths.go index fc3f221eb..b294f2417 100644 --- a/lib/paths/paths.go +++ b/lib/paths/paths.go @@ -177,6 +177,28 @@ func (p *Paths) ImageRepositoryTagSymlink(repository, tag string) string { return filepath.Join(p.ImageRepositoriesDir(), repository, tag) } +// ImageLayersDir returns the root directory of the per-layer artifact store. +// Layer artifacts are content-addressed by the compressed layer blob digest. +func (p *Paths) ImageLayersDir() string { + return filepath.Join(p.dataDir, "images", "layers") +} + +// ImageLayerDir returns the artifact directory for one layer digest. +func (p *Paths) ImageLayerDir(layerHex string) string { + return filepath.Join(p.ImageLayersDir(), layerHex) +} + +// ImageLayerArtifactForFormat returns the path to a materialized layer artifact. +func (p *Paths) ImageLayerArtifactForFormat(layerHex, format string) string { + return filepath.Join(p.ImageLayerDir(layerHex), "layer."+format) +} + +// ImageLayerRecordForFormat returns the path to the artifact record for one +// materialized layer and format. +func (p *Paths) ImageLayerRecordForFormat(layerHex, format string) string { + return filepath.Join(p.ImageLayerDir(layerHex), "artifact."+format+".json") +} + // ImageDigestDir returns the directory for a specific image digest. func (p *Paths) ImageDigestDir(repository, digestHex string) string { return filepath.Join(p.dataDir, "images", repository, digestHex) diff --git a/lib/resources/disk.go b/lib/resources/disk.go index c8847ce7b..9c15e4588 100644 --- a/lib/resources/disk.go +++ b/lib/resources/disk.go @@ -2,6 +2,7 @@ package resources import ( "context" + "fmt" "strings" "github.com/c2h5oh/datasize" @@ -75,31 +76,35 @@ func (d *DiskResource) GetBreakdown(ctx context.Context) (*DiskBreakdown, error) // Get image sizes if d.imageLister != nil { imageBytes, err := d.imageLister.TotalImageBytes(ctx) - if err == nil { - breakdown.Images = imageBytes + if err != nil { + return nil, fmt.Errorf("get image disk usage: %w", err) } + breakdown.Images = imageBytes ociCacheBytes, err := d.imageLister.TotalOCICacheBytes(ctx) - if err == nil { - breakdown.OCICache = ociCacheBytes + if err != nil { + return nil, fmt.Errorf("get OCI cache disk usage: %w", err) } + breakdown.OCICache = ociCacheBytes } // Get volume sizes if d.volumeLister != nil { volumeBytes, err := d.volumeLister.TotalVolumeBytes(ctx) - if err == nil { - breakdown.Volumes = volumeBytes + if err != nil { + return nil, fmt.Errorf("get volume disk usage: %w", err) } + breakdown.Volumes = volumeBytes } // Get overlay sizes from instances if d.instanceLister != nil { instances, err := d.instanceLister.ListInstanceAllocations(ctx) - if err == nil { - for _, inst := range instances { - if isActiveState(inst.State) { - breakdown.Overlays += inst.OverlayBytes + inst.VolumeOverlayBytes - } + if err != nil { + return nil, fmt.Errorf("get instance disk usage: %w", err) + } + for _, inst := range instances { + if isActiveState(inst.State) { + breakdown.Overlays += inst.OverlayBytes + inst.VolumeOverlayBytes } } } diff --git a/lib/resources/resource.go b/lib/resources/resource.go index d674c5d01..f8f624e93 100644 --- a/lib/resources/resource.go +++ b/lib/resources/resource.go @@ -90,7 +90,7 @@ type AllocationBreakdown struct { // DiskBreakdown shows disk usage by category. type DiskBreakdown struct { Images int64 `json:"images_bytes"` // Exported rootfs disk files - OCICache int64 `json:"oci_cache_bytes"` // OCI layer cache (shared blobs) + OCICache int64 `json:"oci_cache_bytes"` // OCI blobs and materialized shared layers Volumes int64 `json:"volumes_bytes"` Overlays int64 `json:"overlays_bytes"` // Rootfs overlays + volume overlays } @@ -132,7 +132,7 @@ type InstanceAllocation struct { type ImageLister interface { // TotalImageBytes returns the total size of all images on disk. TotalImageBytes(ctx context.Context) (int64, error) - // TotalOCICacheBytes returns the total size of the OCI layer cache. + // TotalOCICacheBytes returns the total size of the OCI and materialized layer caches. TotalOCICacheBytes(ctx context.Context) (int64, error) } diff --git a/lib/resources/resource_test.go b/lib/resources/resource_test.go index 0ba2504af..227392643 100644 --- a/lib/resources/resource_test.go +++ b/lib/resources/resource_test.go @@ -2,6 +2,7 @@ package resources import ( "context" + "errors" "runtime" "testing" @@ -14,10 +15,11 @@ import ( // mockInstanceLister implements InstanceLister for testing type mockInstanceLister struct { allocations []InstanceAllocation + err error } func (m *mockInstanceLister) ListInstanceAllocations(ctx context.Context) ([]InstanceAllocation, error) { - return m.allocations, nil + return m.allocations, m.err } // mockImageLister implements ImageLister for testing @@ -37,10 +39,11 @@ func (m *mockImageLister) TotalOCICacheBytes(ctx context.Context) (int64, error) // mockVolumeLister implements VolumeLister for testing type mockVolumeLister struct { totalBytes int64 + err error } func (m *mockVolumeLister) TotalVolumeBytes(ctx context.Context) (int64, error) { - return m.totalBytes, nil + return m.totalBytes, m.err } func TestNewManager(t *testing.T) { @@ -487,6 +490,23 @@ func TestDiskBreakdown_IncludesOCICacheAndVolumeOverlays(t *testing.T) { assert.Equal(t, int64(25*1024*1024*1024), status.DiskDetail.Overlays) } +func TestDiskBreakdownPropagatesVolumeAndInstanceErrors(t *testing.T) { + cfg := &config.Config{DataDir: t.TempDir(), Capacity: config.CapacityConfig{Disk: "100GB"}} + p := paths.New(cfg.DataDir) + + volumeErr := errors.New("volume lookup failed") + disk, err := NewDiskResource(cfg, p, nil, nil, &mockVolumeLister{err: volumeErr}) + require.NoError(t, err) + _, err = disk.GetBreakdown(context.Background()) + require.ErrorIs(t, err, volumeErr) + + instanceErr := errors.New("instance lookup failed") + disk, err = NewDiskResource(cfg, p, &mockInstanceLister{err: instanceErr}, nil, nil) + require.NoError(t, err) + _, err = disk.GetBreakdown(context.Background()) + require.ErrorIs(t, err, instanceErr) +} + func TestValidateAllocation_Disk(t *testing.T) { cfg := &config.Config{ DataDir: t.TempDir(), diff --git a/openapi.yaml b/openapi.yaml index 3621c3e8c..d221016c9 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -2283,7 +2283,7 @@ components: oci_cache_bytes: type: integer format: int64 - description: Disk used by OCI layer cache (shared blobs) + description: Disk used by OCI layer cache (shared blobs and materialized layers) example: 53687091200 volumes_bytes: type: integer