From 557bc5221f72c1eb608c5aebe842a06c458a0861 Mon Sep 17 00:00:00 2001 From: Stuart Cameron Date: Sat, 10 Oct 2026 16:27:01 +1100 Subject: [PATCH 1/2] feat(deps): ship the v2 CPU tier as a delta, and bundle deps in the AppImage The AppImage catalog's test launches an AppImage with no network, and VapourBox's first act was to download its deps, so the listing failed on an "Installation Failed" dialog (AppImage/appimage.github.io#9839). Deps 1.13.0 (packaging only; same sources as 1.12.0): - Each x64 platform publishes its v3 bundle plus a small -v2-delta.zip of the files in _tierFiles, extracted over it, instead of a second full bundle. Scripts/make-deps-delta.py cuts it and the build-deps-* workflows gate the composed v2 tree before uploading. - deps-version.json gains "tierFormat": "delta". App: - DependencyManager takes each zip from bundled-deps/ beside the executable, then the cache, then the network; applies the delta over the bundle on a v2 CPU and refuses one cut against a different bundle. - A zip's own version.json is never extracted; the app's copy stays the commit marker. Linux package: - The AppImage carries the deps zips (--bundle-deps); the tarball does not. - Renamed to VapourBox-X-x86_64.AppImage / -aarch64.AppImage, with a copy of the .zsync under the old name so 1.2.0 can still update. - AppStream metadata (app.vapourbox.VapourBox.appdata.xml). CI installs a v2 tier the way the app does, through .github/scripts/install-deps-bundle.sh. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/fetch-deps-bundle.sh | 18 +- .github/scripts/install-deps-bundle.sh | 53 +++ .github/workflows/build-deps-linux.yml | 73 +++- .github/workflows/build-deps-macos.yml | 62 +++- .github/workflows/build-deps-windows.yml | 72 +++- .github/workflows/build-linux.yml | 65 ++-- .github/workflows/ci-test.yml | 23 +- .github/workflows/nightly.yml | 21 +- .github/workflows/probe-cpu-compat.yml | 20 +- CLAUDE.md | 139 ++++++-- README.md | 16 +- Scripts/deps-expected-plugins.json | 13 + Scripts/make-deps-delta.py | 182 ++++++++++ Scripts/package-linux.sh | 102 +++++- Scripts/release.sh | 4 +- app/assets/deps-version.json | 7 +- app/lib/services/dependency_manager.dart | 318 ++++++++++++++---- app/lib/views/dependency_download_dialog.dart | 6 +- app/test/deps_delta_install_test.dart | 252 ++++++++++++++ app/test/packaging_test.dart | 80 ++++- app/test/support/worker_harness.dart | 66 ++-- docs/BUILDING.md | 21 +- docs/ENGINEERING_NOTES.md | 40 +++ .../linux/app.vapourbox.VapourBox.appdata.xml | 84 +++++ 24 files changed, 1476 insertions(+), 261 deletions(-) create mode 100755 .github/scripts/install-deps-bundle.sh create mode 100755 Scripts/make-deps-delta.py create mode 100644 app/test/deps_delta_install_test.dart create mode 100644 packaging/linux/app.vapourbox.VapourBox.appdata.xml diff --git a/.github/scripts/fetch-deps-bundle.sh b/.github/scripts/fetch-deps-bundle.sh index 37a3678..7ba371a 100755 --- a/.github/scripts/fetch-deps-bundle.sh +++ b/.github/scripts/fetch-deps-bundle.sh @@ -1,11 +1,16 @@ #!/usr/bin/env bash # -# Fetch one platform's dependency bundle for a CI run and print the path of the +# Fetch ONE dependency release asset for a CI run and print the path of the # downloaded .zip on stdout. Everything else goes to stderr, so the caller can # do: # # ZIP=$(bash .github/scripts/fetch-deps-bundle.sh macos-arm64 deps-v1.9.0 1.9.0) # +# The first argument is an asset id as it appears in the file name: a platform +# (`linux-x64`), or a v2 delta (`linux-x64-v2-delta`). Workflows should not +# call this directly — install-deps-bundle.sh knows which assets make up a +# platform's bundle (a v2 tier is two of them) and extracts them in order. +# # Two sources, chosen by whether $DEPS_RUN_ID is set: # # (default) the published release named by app/assets/deps-version.json. This @@ -45,7 +50,10 @@ TAG="${2:?missing release tag}" VER="${3:?missing version}" if [ -n "${DEPS_RUN_ID:-}" ]; then - rm -rf .deps-artifact + # One directory per asset: installing a v2 tier fetches two, and the second + # must not wipe the first. + ARTIFACT_DIR=".deps-artifact/$PLATFORM" + rm -rf "$ARTIFACT_DIR" ZIP="" FROM_RUN="" @@ -55,13 +63,13 @@ if [ -n "${DEPS_RUN_ID:-}" ]; then for RUN in $(echo "$DEPS_RUN_ID" | tr ',' ' '); do echo "looking for a $PLATFORM artifact in run $RUN" >&2 if gh run download "$RUN" \ - --pattern "VapourBox-deps-*-${PLATFORM}" --dir .deps-artifact >&2 2>/dev/null; then + --pattern "VapourBox-deps-*-${PLATFORM}" --dir "$ARTIFACT_DIR" >&2 2>/dev/null; then # gh nests each artifact in a directory named after it, so glob rather # than assuming a layout. - ZIP=$(find .deps-artifact -name "VapourBox-deps-*-${PLATFORM}.zip" | head -1) + ZIP=$(find "$ARTIFACT_DIR" -name "VapourBox-deps-*-${PLATFORM}.zip" | head -1) if [ -n "$ZIP" ]; then FROM_RUN="$RUN"; break; fi fi - rm -rf .deps-artifact + rm -rf "$ARTIFACT_DIR" done if [ -z "$ZIP" ]; then diff --git a/.github/scripts/install-deps-bundle.sh b/.github/scripts/install-deps-bundle.sh new file mode 100755 index 0000000..1f01d9c --- /dev/null +++ b/.github/scripts/install-deps-bundle.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# +# Fetch a platform's dependency bundle and extract it into a directory: +# +# bash .github/scripts/install-deps-bundle.sh linux-x64-v2 deps-v1.13.0 1.13.0 deps/linux-x64 +# +# The first argument names the bundle the way the test matrix does: a platform +# (`linux-x64`, `macos-arm64`), optionally suffixed `-v2` for the older-CPU tier +# of an x64 platform (issue #92). +# +# A v2 tier is not one download. From deps 1.13.0 (`"tierFormat": "delta"` in +# app/assets/deps-version.json) it is the platform's full bundle plus a small +# delta extracted over it — exactly what DependencyManager does on a user's +# machine, so CI installs it the same way rather than from some full v2 zip no +# user ever receives. This script is the one place in CI that knows that. +# +# Sources and $DEPS_RUN_ID are as documented in fetch-deps-bundle.sh. +set -euo pipefail + +BUNDLE="${1:?usage: install-deps-bundle.sh [-v2] }" +TAG="${2:?missing release tag}" +VER="${3:?missing version}" +DEST="${4:?missing destination directory}" +HERE="$(cd "$(dirname "$0")" && pwd)" +POINTER="$HERE/../../app/assets/deps-version.json" + +ASSETS=("$BUNDLE") +if [[ "$BUNDLE" == *-v2 ]] && grep -q '"tierFormat": *"delta"' "$POINTER"; then + ASSETS=("${BUNDLE%-v2}" "$BUNDLE-delta") +fi + +mkdir -p "$DEST" +for ASSET in "${ASSETS[@]}"; do + ZIP=$(bash "$HERE/fetch-deps-bundle.sh" "$ASSET" "$TAG" "$VER") + [ -f "$ZIP" ] || { echo "::error::no zip for $ASSET" >&2; exit 1; } + echo "extracting $ZIP into $DEST" >&2 + if [ "${RUNNER_OS:-}" = Windows ]; then + # The Windows bundle uses backslash separators; 7-Zip copes, info-zip + # `unzip` warns and exits non-zero. + 7z x "$ZIP" -o"$DEST" -y >/dev/null + else + unzip -q -o "$ZIP" -d "$DEST" + fi + rm -f "$ZIP" +done +rm -rf .deps-artifact + +# The delta carries the version.json that marks the tree as the v2 tier; if it +# did not land last, the tests would silently run the v3 bundle. +if [[ "$BUNDLE" == *-v2 ]] && ! grep -q '"tier": *"v2"' "$DEST/version.json"; then + echo "::error::$DEST/version.json does not say tier v2 after installing $BUNDLE" >&2 + exit 1 +fi diff --git a/.github/workflows/build-deps-linux.yml b/.github/workflows/build-deps-linux.yml index 7047b60..706d332 100644 --- a/.github/workflows/build-deps-linux.yml +++ b/.github/workflows/build-deps-linux.yml @@ -27,7 +27,9 @@ permissions: jobs: # x64 ships in two CPU tiers (issue #92): v3 for x86-64-v3 CPUs, v2 for - # anything older. Same script, one --tier switch. + # anything older. Same script, one --tier switch. Both are built in full, but + # only v3 is published as a bundle: delta-x64 below reduces the v2 build to + # the few files that differ (Scripts/make-deps-delta.py). build-x64: if: ${{ inputs.arch == 'x64' || inputs.arch == 'both' }} runs-on: ubuntu-24.04 @@ -62,18 +64,59 @@ jobs: - name: Package dependencies run: ./Scripts/package-deps-linux.sh --version "${{ inputs.version }}" --arch x64 + # The v2 build is an intermediate ("-v2-build"): nothing downstream may + # mistake it for a publishable bundle. - uses: actions/upload-artifact@v5 with: - name: ${{ env.ASSET }} + name: ${{ env.ASSET }}${{ matrix.tier == 'v2' && '-build' || '' }} + retention-days: ${{ matrix.tier == 'v2' && 3 || 90 }} path: | dist/${{ env.ASSET }}.zip dist/${{ env.ASSET }}.zip.sha256.json - # The v2 bundle's promise is that it runs on CPUs without AVX. Every hosted + # Cut the v2 delta against the v3 bundle, and compose the tree a v2 user + # ends up with (v3 bundle + delta) for the gate to test. + delta-x64: + needs: build-x64 + runs-on: ubuntu-24.04 + env: + STEM: VapourBox-deps-${{ inputs.version }}-linux-x64 + steps: + - uses: actions/checkout@v5 + - uses: actions/download-artifact@v5 + with: + name: ${{ env.STEM }} + path: base + - uses: actions/download-artifact@v5 + with: + name: ${{ env.STEM }}-v2-build + path: v2-build + - name: Make the v2 delta + run: | + python3 Scripts/make-deps-delta.py --platform linux-x64 \ + --version "${{ inputs.version }}" \ + --base "base/$STEM.zip" --v2 "v2-build/$STEM-v2.zip" \ + --out dist --composed composed + - uses: actions/upload-artifact@v5 + with: + name: ${{ env.STEM }}-v2-delta + path: | + dist/${{ env.STEM }}-v2-delta.zip + dist/${{ env.STEM }}-v2-delta.zip.sha256.json + # What the gate probes. Named as the gate expects a v2 bundle to be; never + # uploaded to a release. + - uses: actions/upload-artifact@v5 + with: + name: ${{ env.STEM }}-v2 + retention-days: 3 + path: composed/${{ env.STEM }}-v2.zip + + # The v2 tier's promise is that it runs on CPUs without AVX. Every hosted # runner has AVX2, so prove it by running every plugin under Intel SDE - # emulating Westmere (the Mac Pro 5,1 from issue #92). + # emulating Westmere (the Mac Pro 5,1 from issue #92) — against the v3 bundle + # with the delta applied, which is what such a machine actually installs. gate-x64-v2: - needs: build-x64 + needs: delta-x64 uses: ./.github/workflows/probe-cpu-compat.yml with: artifact_prefix: VapourBox-deps-${{ inputs.version }}- @@ -81,26 +124,32 @@ jobs: gate: true matrix_json: '[{"platform":"linux-x64","runner":"ubuntu-24.04","sde":"sde64","python":"python/bin/python3","chip":"wsm"}]' - # Published only once the v2 gate has passed, so a bundle that would crash on - # an older CPU never reaches a release. + # Published only once the v2 gate has passed, so a delta that would crash on + # an older CPU never reaches a release. Exactly four files: the v3 bundle, + # the v2 delta, and a sidecar each. upload-x64: - needs: [build-x64, gate-x64-v2] + needs: [build-x64, delta-x64, gate-x64-v2] if: inputs.release_tag != '' runs-on: ubuntu-24.04 + env: + STEM: VapourBox-deps-${{ inputs.version }}-linux-x64 steps: - uses: actions/download-artifact@v5 with: - pattern: VapourBox-deps-${{ inputs.version }}-linux-x64* + name: ${{ env.STEM }} + path: dist + - uses: actions/download-artifact@v5 + with: + name: ${{ env.STEM }}-v2-delta path: dist - merge-multiple: true - name: Upload to release env: GH_TOKEN: ${{ github.token }} run: | gh release upload "${{ inputs.release_tag }}" \ --repo "${{ github.repository }}" \ - dist/VapourBox-deps-${{ inputs.version }}-linux-x64*.zip \ - dist/VapourBox-deps-${{ inputs.version }}-linux-x64*.zip.sha256.json --clobber + "dist/$STEM.zip" "dist/$STEM.zip.sha256.json" \ + "dist/$STEM-v2-delta.zip" "dist/$STEM-v2-delta.zip.sha256.json" --clobber build-arm64: if: ${{ inputs.arch == 'arm64' || inputs.arch == 'both' }} diff --git a/.github/workflows/build-deps-macos.yml b/.github/workflows/build-deps-macos.yml index 5b900ea..ddd101f 100644 --- a/.github/workflows/build-deps-macos.yml +++ b/.github/workflows/build-deps-macos.yml @@ -61,10 +61,16 @@ jobs: "dist/VapourBox-deps-${{ inputs.version }}-macos-arm64.zip.sha256.json" --clobber # x64 ships in two CPU tiers (issue #92): v3 for x86-64-v3 CPUs, v2 for - # anything older. Same script, one --tier switch. The v2 bundle is gated by - # package-deps-macos.sh itself (Scripts/check-load-time-simd.py): macOS has - # no Intel SDE, so it proves statically that no plugin runs AVX while it - # loads — and fails before anything below can upload it. + # anything older. Same script, one --tier switch. Both are built in full, but + # only v3 is published as a bundle: delta-x64 below reduces the v2 build to + # the few files that differ (Scripts/make-deps-delta.py). + # + # The v2 build is gated by package-deps-macos.sh itself + # (Scripts/check-load-time-simd.py): macOS has no Intel SDE, so it proves + # statically that no plugin runs AVX while it loads — and fails this job, so + # nothing below can publish a delta cut from it. The check covers every + # plugin the v2 build holds; the ones outside the delta are the same sources + # and binaries the v3 bundle was built from. build-x64: if: ${{ inputs.arch == 'x64' || inputs.arch == 'both' }} runs-on: macos-15-intel # only hosted Intel image (macos-13 retired; last one until ~Aug 2027) @@ -87,18 +93,60 @@ jobs: - name: Package dependencies run: ./Scripts/package-deps-macos.sh --version "${{ inputs.version }}" --arch x64 + # The v2 build is an intermediate ("-v2-build"): nothing downstream may + # mistake it for a publishable bundle. - uses: actions/upload-artifact@v5 with: - name: ${{ env.ASSET }} + name: ${{ env.ASSET }}${{ matrix.tier == 'v2' && '-build' || '' }} + retention-days: ${{ matrix.tier == 'v2' && 3 || 90 }} path: | dist/${{ env.ASSET }}.zip dist/${{ env.ASSET }}.zip.sha256.json + # Cut the v2 delta against the v3 bundle, then publish both. Runs on macOS so + # the composed tree can be put through the same load-time check the v2 build + # passed, this time on exactly what a v2 user ends up with. + delta-x64: + needs: build-x64 + runs-on: macos-15 + env: + STEM: VapourBox-deps-${{ inputs.version }}-macos-x64 + steps: + - uses: actions/checkout@v5 + - uses: actions/download-artifact@v5 + with: + name: ${{ env.STEM }} + path: dist + - uses: actions/download-artifact@v5 + with: + name: ${{ env.STEM }}-v2-build + path: v2-build + - name: Make the v2 delta + run: | + python3 Scripts/make-deps-delta.py --platform macos-x64 \ + --version "${{ inputs.version }}" \ + --base "dist/$STEM.zip" --v2 "v2-build/$STEM-v2.zip" \ + --out dist --composed composed + + - name: Gate the composed v2 tree (no AVX at load time) + run: | + mkdir composed-tree + unzip -q "composed/$STEM-v2.zip" -d composed-tree + python3 Scripts/check-load-time-simd.py composed-tree/vapoursynth/plugins/*.dylib + + - uses: actions/upload-artifact@v5 + with: + name: ${{ env.STEM }}-v2-delta + path: | + dist/${{ env.STEM }}-v2-delta.zip + dist/${{ env.STEM }}-v2-delta.zip.sha256.json + - name: Upload to release if: inputs.release_tag != '' env: GH_TOKEN: ${{ github.token }} run: | gh release upload "${{ inputs.release_tag }}" \ - "dist/${{ env.ASSET }}.zip" \ - "dist/${{ env.ASSET }}.zip.sha256.json" --clobber + --repo "${{ github.repository }}" \ + "dist/$STEM.zip" "dist/$STEM.zip.sha256.json" \ + "dist/$STEM-v2-delta.zip" "dist/$STEM-v2-delta.zip.sha256.json" --clobber diff --git a/.github/workflows/build-deps-windows.yml b/.github/workflows/build-deps-windows.yml index 95f4bf4..14e3b1e 100644 --- a/.github/workflows/build-deps-windows.yml +++ b/.github/workflows/build-deps-windows.yml @@ -24,7 +24,9 @@ permissions: jobs: # x64 ships in two CPU tiers (issue #92): v3 for x86-64-v3 CPUs, v2 for - # anything older. Same script, one -Tier switch. + # anything older. Same script, one -Tier switch. Both are built in full, but + # only v3 is published as a bundle: delta-x64 below reduces the v2 build to + # the few files that differ (Scripts/make-deps-delta.py). build-x64: runs-on: windows-latest strategy: @@ -44,19 +46,61 @@ jobs: shell: pwsh run: ./Scripts/package-deps-windows.ps1 -Version "${{ inputs.version }}" + # The v2 build is an intermediate ("-v2-build"): nothing downstream may + # mistake it for a publishable bundle. - uses: actions/upload-artifact@v5 with: - name: ${{ env.ASSET }} + name: ${{ env.ASSET }}${{ matrix.tier == 'v2' && '-build' || '' }} + retention-days: ${{ matrix.tier == 'v2' && 3 || 90 }} path: | dist/${{ env.ASSET }}.zip dist/${{ env.ASSET }}.zip.sha256.json - # The v2 bundle must run below x86-64-v3, and every hosted runner has AVX2. + # Cut the v2 delta against the v3 bundle, and compose the tree a v2 user + # ends up with (v3 bundle + delta) for the gate to test. Pure zip work, so it + # runs on Linux like the other platforms' delta jobs. + delta-x64: + needs: build-x64 + runs-on: ubuntu-24.04 + env: + STEM: VapourBox-deps-${{ inputs.version }}-windows-x64 + steps: + - uses: actions/checkout@v5 + - uses: actions/download-artifact@v5 + with: + name: ${{ env.STEM }} + path: base + - uses: actions/download-artifact@v5 + with: + name: ${{ env.STEM }}-v2-build + path: v2-build + - name: Make the v2 delta + run: | + python3 Scripts/make-deps-delta.py --platform windows-x64 \ + --version "${{ inputs.version }}" \ + --base "base/$STEM.zip" --v2 "v2-build/$STEM-v2.zip" \ + --out dist --composed composed + - uses: actions/upload-artifact@v5 + with: + name: ${{ env.STEM }}-v2-delta + path: | + dist/${{ env.STEM }}-v2-delta.zip + dist/${{ env.STEM }}-v2-delta.zip.sha256.json + # What the gate probes. Named as the gate expects a v2 bundle to be; never + # uploaded to a release. + - uses: actions/upload-artifact@v5 + with: + name: ${{ env.STEM }}-v2 + retention-days: 3 + path: composed/${{ env.STEM }}-v2.zip + + # The v2 tier must run below x86-64-v3, and every hosted runner has AVX2. # Windows is probed at Sandy Bridge (AVX without AVX2): at pre-AVX chips SDE # cannot emulate what Microsoft's runtime reads from the kernel, so the core - # control fails and nothing could be proven (see probe-cpu-compat.yml). + # control fails and nothing could be proven (see probe-cpu-compat.yml). The + # gate runs against the v3 bundle with the delta applied. gate-x64-v2: - needs: build-x64 + needs: delta-x64 uses: ./.github/workflows/probe-cpu-compat.yml with: artifact_prefix: VapourBox-deps-${{ inputs.version }}- @@ -64,22 +108,28 @@ jobs: gate: true matrix_json: '[{"platform":"windows-x64","runner":"windows-latest","sde":"sde.exe","python":"vapoursynth/python.exe","chip":"snb"}]' - # Published only once the v2 gate has passed. + # Published only once the v2 gate has passed. Exactly four files: the v3 + # bundle, the v2 delta, and a sidecar each. upload-x64: - needs: [build-x64, gate-x64-v2] + needs: [build-x64, delta-x64, gate-x64-v2] if: inputs.release_tag != '' runs-on: ubuntu-24.04 + env: + STEM: VapourBox-deps-${{ inputs.version }}-windows-x64 steps: - uses: actions/download-artifact@v5 with: - pattern: VapourBox-deps-${{ inputs.version }}-windows-x64* + name: ${{ env.STEM }} + path: dist + - uses: actions/download-artifact@v5 + with: + name: ${{ env.STEM }}-v2-delta path: dist - merge-multiple: true - name: Upload to release env: GH_TOKEN: ${{ github.token }} run: | gh release upload "${{ inputs.release_tag }}" \ --repo "${{ github.repository }}" \ - dist/VapourBox-deps-${{ inputs.version }}-windows-x64*.zip \ - dist/VapourBox-deps-${{ inputs.version }}-windows-x64*.zip.sha256.json --clobber + "dist/$STEM.zip" "dist/$STEM.zip.sha256.json" \ + "dist/$STEM-v2-delta.zip" "dist/$STEM-v2-delta.zip.sha256.json" --clobber diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index 74925cd..356098f 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -37,7 +37,7 @@ jobs: libgtk-3-dev liblzma-dev libstdc++-12-dev # zsyncmake writes the .zsync the AppImage's update information names; # desktop-file-validate lets appimagetool check the desktop entry. - sudo apt-get install -y zsync desktop-file-utils + sudo apt-get install -y zsync desktop-file-utils appstream - name: Setup Flutter uses: subosito/flutter-action@v2 @@ -63,21 +63,27 @@ jobs: restore-keys: | ${{ runner.os }}-x64-cargo- - - name: Download Linux dependencies + # The AppImage carries the deps zips so its first launch needs no + # network. They go in as published, sidecars included; package-linux.sh + # checks each against its sidecar and that they are the version + # app/assets/deps-version.json names. + - name: Download the deps to bundle run: | DEPS_VERSION="${{ inputs.deps_tag }}" DEPS_VERSION="${DEPS_VERSION#deps-v}" + BASE="https://github.com/${{ github.repository }}/releases/download/${{ inputs.deps_tag }}" + STEM="VapourBox-deps-${DEPS_VERSION}-linux-x64" - mkdir -p deps/linux-x64 + mkdir -p bundled-deps + for F in "$STEM.zip" "$STEM.zip.sha256.json"; do + curl -fL --retry 3 -o "bundled-deps/$F" "$BASE/$F" + done - DEPS_URL="https://github.com/${{ github.repository }}/releases/download/${{ inputs.deps_tag }}/VapourBox-deps-${DEPS_VERSION}-linux-x64.zip" - echo "Downloading deps from: $DEPS_URL" - curl -L -o deps.zip "$DEPS_URL" - unzip -o deps.zip -d deps/linux-x64 - rm -f deps.zip - - echo "Dependencies extracted:" - ls -la deps/ + # x64 only: the few files older CPUs need instead (see "CPU tiers"). + for F in "$STEM-v2-delta.zip" "$STEM-v2-delta.zip.sha256.json"; do + curl -fL --retry 3 -o "bundled-deps/$F" "$BASE/$F" + done + ls -la bundled-deps - name: Update version numbers run: | @@ -110,7 +116,8 @@ jobs: - name: Package release run: | - ./Scripts/package-linux.sh --version "${{ inputs.version }}" --arch x64 --skip-build + ./Scripts/package-linux.sh --version "${{ inputs.version }}" --arch x64 --skip-build \ + --bundle-deps bundled-deps - name: Upload artifacts uses: actions/upload-artifact@v5 @@ -118,7 +125,8 @@ jobs: name: VapourBox-${{ inputs.version }}-linux-x64 if-no-files-found: error path: | - dist/VapourBox-${{ inputs.version }}-linux-x64.AppImage + dist/VapourBox-${{ inputs.version }}-x86_64.AppImage + dist/VapourBox-${{ inputs.version }}-x86_64.AppImage.zsync dist/VapourBox-${{ inputs.version }}-linux-x64.AppImage.zsync dist/VapourBox-${{ inputs.version }}-linux-x64.tar.gz @@ -137,7 +145,7 @@ jobs: libgtk-3-dev liblzma-dev libstdc++-12-dev # zsyncmake writes the .zsync the AppImage's update information names; # desktop-file-validate lets appimagetool check the desktop entry. - sudo apt-get install -y zsync desktop-file-utils + sudo apt-get install -y zsync desktop-file-utils appstream - name: Setup Flutter run: | @@ -168,21 +176,22 @@ jobs: restore-keys: | ${{ runner.os }}-arm64-cargo- - - name: Download Linux dependencies + # The AppImage carries the deps zips so its first launch needs no + # network. They go in as published, sidecars included; package-linux.sh + # checks each against its sidecar and that they are the version + # app/assets/deps-version.json names. + - name: Download the deps to bundle run: | DEPS_VERSION="${{ inputs.deps_tag }}" DEPS_VERSION="${DEPS_VERSION#deps-v}" + BASE="https://github.com/${{ github.repository }}/releases/download/${{ inputs.deps_tag }}" + STEM="VapourBox-deps-${DEPS_VERSION}-linux-arm64" - mkdir -p deps/linux-arm64 - - DEPS_URL="https://github.com/${{ github.repository }}/releases/download/${{ inputs.deps_tag }}/VapourBox-deps-${DEPS_VERSION}-linux-arm64.zip" - echo "Downloading deps from: $DEPS_URL" - curl -L -o deps.zip "$DEPS_URL" - unzip -o deps.zip -d deps/linux-arm64 - rm -f deps.zip - - echo "Dependencies extracted:" - ls -la deps/ + mkdir -p bundled-deps + for F in "$STEM.zip" "$STEM.zip.sha256.json"; do + curl -fL --retry 3 -o "bundled-deps/$F" "$BASE/$F" + done + ls -la bundled-deps - name: Update version numbers run: | @@ -215,7 +224,8 @@ jobs: - name: Package release run: | - ./Scripts/package-linux.sh --version "${{ inputs.version }}" --arch arm64 --skip-build + ./Scripts/package-linux.sh --version "${{ inputs.version }}" --arch arm64 --skip-build \ + --bundle-deps bundled-deps - name: Upload artifacts uses: actions/upload-artifact@v5 @@ -223,6 +233,7 @@ jobs: name: VapourBox-${{ inputs.version }}-linux-arm64 if-no-files-found: error path: | - dist/VapourBox-${{ inputs.version }}-linux-arm64.AppImage + dist/VapourBox-${{ inputs.version }}-aarch64.AppImage + dist/VapourBox-${{ inputs.version }}-aarch64.AppImage.zsync dist/VapourBox-${{ inputs.version }}-linux-arm64.AppImage.zsync dist/VapourBox-${{ inputs.version }}-linux-arm64.tar.gz diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index b7647d4..8542a5e 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -75,11 +75,8 @@ jobs: DEPS_RUN_ID: ${{ inputs.deps_run_id }} run: | PLATFORM="macos-${{ matrix.arch }}" - mkdir -p "deps/$PLATFORM" - ZIP=$(bash .github/scripts/fetch-deps-bundle.sh "$PLATFORM" \ - "${{ steps.deps.outputs.tag }}" "${{ steps.deps.outputs.ver }}") - unzip -q -o "$ZIP" -d "deps/$PLATFORM" - rm -rf "$ZIP" .deps-artifact + bash .github/scripts/install-deps-bundle.sh "$PLATFORM" \ + "${{ steps.deps.outputs.tag }}" "${{ steps.deps.outputs.ver }}" "deps/$PLATFORM" xattr -cr "deps/$PLATFORM" 2>/dev/null || true # The whisper small model (~466 MB) is architecture-independent; cache it. @@ -149,13 +146,8 @@ jobs: GH_TOKEN: ${{ github.token }} DEPS_RUN_ID: ${{ inputs.deps_run_id }} run: | - mkdir -p deps/windows-x64 - ZIP=$(bash .github/scripts/fetch-deps-bundle.sh windows-x64 \ - "${{ steps.deps.outputs.tag }}" "${{ steps.deps.outputs.ver }}") - # The Windows deps zip uses backslash separators; 7-Zip handles that - # (info-zip `unzip` warns and exits non-zero, tripping `set -e`). - 7z x "$ZIP" -o"deps/windows-x64" -y >/dev/null - rm -rf "$ZIP" .deps-artifact + bash .github/scripts/install-deps-bundle.sh windows-x64 \ + "${{ steps.deps.outputs.tag }}" "${{ steps.deps.outputs.ver }}" deps/windows-x64 - name: Cache whisper model uses: actions/cache@v5 @@ -231,11 +223,8 @@ jobs: GH_TOKEN: ${{ github.token }} DEPS_RUN_ID: ${{ inputs.deps_run_id }} run: | - mkdir -p deps/linux-x64 - ZIP=$(bash .github/scripts/fetch-deps-bundle.sh linux-x64 \ - "${{ steps.deps.outputs.tag }}" "${{ steps.deps.outputs.ver }}") - unzip -q -o "$ZIP" -d deps/linux-x64 - rm -rf "$ZIP" .deps-artifact + bash .github/scripts/install-deps-bundle.sh linux-x64 \ + "${{ steps.deps.outputs.tag }}" "${{ steps.deps.outputs.ver }}" deps/linux-x64 - name: Cache whisper model uses: actions/cache@v5 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 7400e4b..56f40ab 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -87,11 +87,8 @@ jobs: run: | # The tier is in the asset name only; either installs to deps/. PLATFORM="macos-${{ matrix.arch }}" - mkdir -p "deps/$PLATFORM" - ZIP=$(bash .github/scripts/fetch-deps-bundle.sh "${{ matrix.asset }}" \ - "${{ steps.deps.outputs.tag }}" "${{ steps.deps.outputs.ver }}") - unzip -q -o "$ZIP" -d "deps/$PLATFORM" - rm -rf "$ZIP" .deps-artifact + bash .github/scripts/install-deps-bundle.sh "${{ matrix.asset }}" \ + "${{ steps.deps.outputs.tag }}" "${{ steps.deps.outputs.ver }}" "deps/$PLATFORM" xattr -cr "deps/$PLATFORM" 2>/dev/null || true - name: Cache whisper model @@ -168,11 +165,8 @@ jobs: GH_TOKEN: ${{ github.token }} DEPS_RUN_ID: ${{ inputs.deps_run_id }} run: | - mkdir -p deps/windows-x64 - ZIP=$(bash .github/scripts/fetch-deps-bundle.sh "${{ matrix.asset }}" \ - "${{ steps.deps.outputs.tag }}" "${{ steps.deps.outputs.ver }}") - 7z x "$ZIP" -o"deps/windows-x64" -y >/dev/null - rm -rf "$ZIP" .deps-artifact + bash .github/scripts/install-deps-bundle.sh "${{ matrix.asset }}" \ + "${{ steps.deps.outputs.tag }}" "${{ steps.deps.outputs.ver }}" deps/windows-x64 - name: Cache whisper model uses: actions/cache@v5 @@ -252,11 +246,8 @@ jobs: GH_TOKEN: ${{ github.token }} DEPS_RUN_ID: ${{ inputs.deps_run_id }} run: | - mkdir -p deps/linux-x64 - ZIP=$(bash .github/scripts/fetch-deps-bundle.sh "${{ matrix.asset }}" \ - "${{ steps.deps.outputs.tag }}" "${{ steps.deps.outputs.ver }}") - unzip -q -o "$ZIP" -d deps/linux-x64 - rm -rf "$ZIP" .deps-artifact + bash .github/scripts/install-deps-bundle.sh "${{ matrix.asset }}" \ + "${{ steps.deps.outputs.tag }}" "${{ steps.deps.outputs.ver }}" deps/linux-x64 - name: Cache whisper model uses: actions/cache@v5 diff --git a/.github/workflows/probe-cpu-compat.yml b/.github/workflows/probe-cpu-compat.yml index 57a4354..9d68ed6 100644 --- a/.github/workflows/probe-cpu-compat.yml +++ b/.github/workflows/probe-cpu-compat.yml @@ -101,19 +101,21 @@ jobs: run: | mkdir -p "$D" if [ -d .deps-artifact ]; then + # Gate mode: one full zip built earlier in this run. For a v2 tier + # that is the v3 bundle with the delta already applied (see the + # delta-x64 job of the calling build-deps-* workflow). ZIP=$(find .deps-artifact -name "*-$ASSET_ID.zip" | head -1) + [ -f "$ZIP" ] || { echo "no bundle zip for $ASSET_ID"; exit 1; } + if [ "$RUNNER_OS" = Windows ]; then + 7z x "$ZIP" -o"$D" -y >/dev/null + else + unzip -q -o "$ZIP" -d "$D" + fi + rm -rf "$ZIP" .deps-artifact else TAG=$(sed -n 's/.*"releaseTag": *"\([^"]*\)".*/\1/p' app/assets/deps-version.json) - ZIP=$(bash .github/scripts/fetch-deps-bundle.sh "$ASSET_ID" "$TAG" "${TAG#deps-v}") + bash .github/scripts/install-deps-bundle.sh "$ASSET_ID" "$TAG" "${TAG#deps-v}" "$D" fi - [ -f "$ZIP" ] || { echo "no bundle zip for $ASSET_ID"; exit 1; } - if [ "$RUNNER_OS" = Windows ]; then - # The Windows zip uses backslash separators; 7-Zip copes, unzip does not. - 7z x "$ZIP" -o"$D" -y >/dev/null - else - unzip -q -o "$ZIP" -d "$D" - fi - rm -rf "$ZIP" .deps-artifact - name: Fetch the positive control (upstream zsmooth, AVX2-only) run: | diff --git a/CLAUDE.md b/CLAUDE.md index bfa3f73..0087147 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -371,7 +371,10 @@ Adding a filter touches many files. Missing any step causes silent failures (fil present before zipping and fail the build if any are missing, so a dead download URL becomes a red build instead of a silently-incomplete bundle. x64 platforms have a second key per CPU tier (`macos-x64-v2`, …) that must - list the same plugins — `attribution_test.dart` asserts it. + list the same plugins — `attribution_test.dart` asserts it. **If the plugin + is built differently for the v2 tier, also add its path to `_tierFiles`** in + the same file: that list is the entire content of the published v2 delta, so + a tier-specific plugin missing from it ships as its v3 build to older CPUs. - **Check it for load-time SIMD on x64** (see "CPU tiers" below): a prebuilt x86 binary built with `-mavx2`-style flags can SIGILL inside `dlopen` on an older CPU, and autoload makes that every job. Run @@ -671,20 +674,41 @@ disagree. Pin the **series** against a rolling host tag (e.g. BtbN's `n9.0-latest`), never an exact old build — a fixed old tag gets garbage-collected off a rolling `latest` alias and 404s the deps build outright. -**CPU tiers (issue #92): every x86 deps bundle ships twice.** `v3` is built -for x86-64-v3 CPUs (Haswell, 2013, and later); `v2` runs on anything older. -The rules: +**CPU tiers (issue #92): every x86 platform's deps come in two tiers.** `v3` +is built for x86-64-v3 CPUs (Haswell, 2013, and later); `v2` runs on anything +older. From deps 1.13.0 only v3 is a full bundle: v2 is a small **delta** +extracted over it. The rules: - **One decision, in the worker.** `cpu::cpu_tier()` (`worker/src/cpu.rs`) requires the *whole* x86-64-v3 set, not just AVX2. `--probe-cpu` reports it, the app downloads by it, and `ctmf_opt` derives from it. Never re-derive the tier elsewhere (sysctl is wrong under Rosetta); on x86 an unclear answer means `v2`, which only costs speed. `VAPOURBOX_DEPS_TIER=v2|v3` overrides it. -- **The tier lives only in the asset name and version.json.** `v3` keeps the - plain names (`VapourBox-deps-X-macos-x64.zip`), `v2` is suffixed - (`…-macos-x64-v2.zip`); both install to `deps/`. The app re-checks - the installed tier at startup (`DependencyStatus.wrongTier`) and replaces a - mismatch even when it is newer than expected. +- **The tier lives only in which assets are installed and in version.json.** + The v3 bundle keeps the plain name (`VapourBox-deps-X-macos-x64.zip`); the v2 + tier is that same bundle plus `…-macos-x64-v2-delta.zip`, extracted over it. + Either way the result installs to `deps/`. + `DependencyManager.installAssetIds` is the one statement of which zips make + up an install, and `"tierFormat": "delta"` in `deps-version.json` is what + selects this (absent = the 1.11.0–1.12.0 scheme of a second full + `…-x64-v2.zip`). The app re-checks the installed tier at startup + (`DependencyStatus.wrongTier`) and reinstalls on a mismatch — from the + bundle, never by patching the installed tree — even when it is newer than + expected. +- **The delta is the files in `_tierFiles`, nothing else, and it is tied to + its bundle.** `Scripts/make-deps-delta.py` takes exactly the paths listed + per platform under `_tierFiles` in `Scripts/deps-expected-plugins.json` from + a full v2 build. It is deliberately not a diff of the two builds (two builds + of the same sources are not byte-identical, so a diff names nearly every + binary); instead it fails if the builds hold different *sets* of paths + (a pure overwrite could not reproduce v2) or if a tier file is + byte-identical in both (the tier switch did nothing). The delta's + `version.json` records `baseSha256`, and both the app and the test harness + refuse to apply it over any other bundle. +- **No zip's `version.json` is ever extracted by the app.** + `_writeInstalledVersion` writes that file last, as the commit marker. The v3 + bundle's own copy says `tier: v3`; letting it through would make a v2 + install interrupted before its delta look like a finished v3 one. - **One `--tier` block per `download-deps-*` script** maps the tier to build variables; nothing else in the script branches on it. The two tiers differ only where running the bundle on an old CPU proved they must: zsmooth @@ -692,11 +716,21 @@ The rules: runtime dispatch) everywhere, and on macOS MVTools (v2 builds v24 from source with `patches/mvtools-v24-no-avx2.patch`, because Stefan-Olt's build SIGILLs in its AVX2 static initializers inside `dlopen`). -- **A v2 bundle is gated before it can be published**: under Intel SDE by - `probe-cpu-compat.yml` (Linux at Westmere; Windows at Sandy Bridge, since SDE - cannot emulate pre-AVX Windows — Microsoft's runtime reads the host's CPU - features from the kernel), and on macOS statically by - `Scripts/check-load-time-simd.py` inside `package-deps-macos.sh`. +- **A v2 delta is gated before it can be published**, against what a v2 user + ends up with — the v3 bundle with the delta applied, composed by the + `delta-x64` job of each `build-deps-*` workflow — not against the v2 build it + was cut from: under Intel SDE by `probe-cpu-compat.yml` (Linux at Westmere; + Windows at Sandy Bridge, since SDE cannot emulate pre-AVX Windows — + Microsoft's runtime reads the host's CPU features from the kernel), and on + macOS statically by `Scripts/check-load-time-simd.py` (once on the v2 build + inside `package-deps-macos.sh`, again on the composed tree). The full v2 + build is a CI intermediate (artifact `…-x64-v2-build`) and must never reach a + release; `deps_delta_install_test.dart` asserts no workflow can upload it. +- **CI installs a v2 tier the way the app does.** + `.github/scripts/install-deps-bundle.sh` is the one place the workflows learn + that `linux-x64-v2` means "bundle, then delta"; `WorkerHarness._downloadDeps` + is the same for local test runs. Don't unzip a deps asset in a workflow + directly. **CTMF's `opt` (SIMD level) must be chosen by the worker from the CPU, never left at the plugin's own auto-detect (`opt=0`).** Its AVX-512 kernel for 8-bit @@ -1312,7 +1346,9 @@ gh workflow run ci-test.yml --ref \ It's a comma-separated list because macOS/Windows/Linux are three separate `build-deps-*` workflow runs; each CI job tries every ID and takes the first -holding its platform's artifact. The bundle version is deliberately not +holding its platform's artifact (`.github/scripts/install-deps-bundle.sh`, +which for a `-v2` row fetches the bundle and the `-v2-delta` artifact and +extracts one over the other). The bundle version is deliberately not cross-checked against `deps-version.json` — it logs a `::warning::` instead, so a green run against an unreleased bundle can't be mistaken for one against the released bundle. @@ -1337,6 +1373,13 @@ there's nothing to protect. `version.json` is written **last**, as the commit marker. Direction is checked with `compareVersions`, not `!=`/string compare — a deliberately newer install must never be treated as outdated and overwritten. +An install can be more than one zip (the v2 tier is bundle + delta, see "CPU +tiers"): each is extracted into the same target in order (`extractBundle`, +`overlay: true` for all but the first), the macOS quarantine strip and re-sign +run once after the last, and the cache keeps every zip of the current install +(`_cachedDownloadPath(keep:)`) so fetching the second doesn't prune the first. +`extractBundle` reads a whole zip into memory, ~240 MB for the Linux bundle. + On Windows, both renames go through a retry loop (`retryTransientFsOperation`) because a directory rename fails with **errno 5** (not a permissions error) if anything — a scanner, the search indexer — briefly holds a handle inside it @@ -1526,18 +1569,48 @@ placement and a version skew would change chroma per-OS. environment is inherited by the worker and by the ffmpeg/vspipe it spawns from the downloaded deps bundle, so anything set there leaks into them. - **The mount is read-only** (`/tmp/.mount_*`). Nothing may be written next - to the executable on Linux; deps and add-ons already live under - `$XDG_DATA_HOME`, which is also why deps stay a first-launch download - instead of being packed in (they are versioned separately and come in two - CPU tiers). + to the executable on Linux; deps and add-ons install under + `$XDG_DATA_HOME`. + - **The AppImage carries the deps zips; the tarball does not.** + `package-linux.sh --bundle-deps DIR` copies the published bundle (and on + x64 the v2 delta), each with its `.sha256.json`, into + `usr/lib/vapourbox/bundled-deps/`, and `DependencyManager` installs from + there instead of downloading (see "Dependency Versioning"). They go in as + the published zips, not unpacked: the mount is read-only and the install + path, tier choice and verification stay exactly the download's. So a deps + release must exist **before** the app build (`build-linux.yml` fetches it + by `deps_tag`), the script refuses zips of any version but the one + `deps-version.json` names, and a deps-only fix still reaches AppImage + users only through a new AppImage or the normal download fallback. + - **Named `VapourBox-X-x86_64.AppImage` / `-aarch64.AppImage`** — the + AppImage convention, which the catalog (appimage.github.io) checks: the + machine's architecture name and no "linux". The tarball keeps + `linux-`. - **The `.AppImage.zsync` must be uploaded beside every AppImage.** The embedded update information names it - (`gh-releases-zsync|…|latest|VapourBox-*-linux-.AppImage.zsync`), so - one missing is an updater that fails for every user. The script fails if - `zsyncmake` is absent rather than build without it, and - `packaging_test.dart` asserts the release upload picks both files up. + (`gh-releases-zsync|…|latest|VapourBox-*-.AppImage.zsync`), + so one missing is an updater that fails for every user. The script fails + if `zsyncmake` is absent rather than build without it, and + `packaging_test.dart` asserts the release upload picks the files up. `latest` is GitHub's Latest release — one more reason a deps or whisper release must never be marked Latest. + - **A second `.zsync`, under the old name, is a bridge for 1.2.0.** 1.2.0 was + the only release named `…-linux-.AppImage` and embeds + `VapourBox-*-linux-.AppImage.zsync`. Each release therefore also + publishes a copy of its `.zsync` as `VapourBox-X-linux-.AppImage.zsync` + (`LEGACY_ZSYNC`); its relative `URL:` header names the new file. Remove it + only once updating from 1.2.0 no longer matters. **Not yet proven + end-to-end**: it needs one real AppImageUpdate run from a 1.2.0 AppImage. + - **AppStream metadata** is `packaging/linux/app.vapourbox.VapourBox.appdata.xml`, + installed to `usr/share/metainfo/` with the version, date and a + tag-pinned screenshot URL stamped in at package time. It is what the + catalog and software centres list. Named `.appdata.xml` because the + catalog's lint only looks for that name, and limited to tags a 2021 + `appstreamcli` knows because that is what the catalog runs. The script + validates it with `--no-net` and passes `--no-appstream` to appimagetool, + whose own check would fetch the screenshot from a tag that does not exist + until the release is published. **`docs/images/screenshot.png` is the + listing's screenshot** — keep it current. - The desktop file and icon are named for `APPLICATION_ID` (`app.vapourbox.VapourBox`, `app/linux/CMakeLists.txt`), which is what the window reports as its app id; rename one and the running window stops @@ -1554,7 +1627,19 @@ Plugin lists for all platforms: see `deps/` directories or download scripts. ## Dependency Versioning and Auto-Download -Dependencies are versioned separately from the app via `app/assets/deps-version.json` and distributed as separate GitHub releases (tag: `deps-vX.Y.Z`). The app auto-downloads deps on launch if missing, outdated, or built for a different CPU tier (x64 only — see "CPU tiers"; the asset is `DependencyManager.assetIdFor(platformId, tier)`). +Dependencies are versioned separately from the app via `app/assets/deps-version.json` and distributed as separate GitHub releases (tag: `deps-vX.Y.Z`). The app auto-downloads deps on launch if missing, outdated, or built for a different CPU tier (x64 only — see "CPU tiers"; the assets are `DependencyManager.installAssetIds(platformId, tier, …)`). + +**"Download" is the last of three sources.** For each zip an install needs, +`DependencyManager._obtainZip` takes, in order: a copy the package shipped in +`bundled-deps/` beside the executable (`findBundledZip`; today only the Linux +AppImage ships one), the cached download from an earlier attempt, then the +network. The bundled directory is one fixed path inside the app's own tree and +must never become a search: a zip found there is extracted and run, so it gets +the trust of the executable next to it and no more places than that. A bundled +zip **must** match the `.sha256.json` shipped beside it (not best-effort, as a +download's sidecar is — offline there is nothing else to say it is whole); one +that doesn't is ignored and the download happens as usual. A bundled zip is +never deleted or copied; only zips the app fetched are cleaned up. `deps-version.json` is a **slim pointer** — `{version, releaseTag, githubRepo}`. Integrity metadata is **not** stored here: each `package-deps-*` script writes a **sidecar** `.sha256.json` uploaded next to the zip, which the app fetches and verifies at download time (best-effort if the sidecar is missing). **Net effect: a new deps release only needs a `version`/`releaseTag` bump.** @@ -1654,6 +1739,7 @@ Notes: | `Scripts/package-deps-macos.sh` | Package macOS deps | | `Scripts/package-deps-windows.ps1` | Package Windows deps | | `Scripts/package-deps-linux.sh` | Package Linux deps | +| `Scripts/make-deps-delta.py` | Cut an x64 platform's v2 delta from a full v2 build and the v3 bundle (run by the `delta-x64` job of each `build-deps-*` workflow) | | `Scripts/download-deps-linux.sh` | Build Linux deps from source | | `Scripts/run-debug-linux.sh` | Dev build + run for Linux | @@ -1661,8 +1747,8 @@ Notes: 1. **Confirm version** — ask user, update `pubspec.yaml` 2. **Check deps** — run `check-deps-changed.sh`; if changed, bump `version`/`releaseTag` in `deps-version.json`. -3. **Build & package deps in CI** — dispatch `build-deps-{macos,windows,linux}.yml` with `release_tag`. They build both x64 tiers, gate the v2 bundles, and only then upload each zip **and** its `.sha256.json` sidecar. -4. **Check the deps release** — every platform, both tiers of each x64: `macos-arm64`, `macos-x64`, `macos-x64-v2`, `windows-x64`, `windows-x64-v2`, `linux-x64`, `linux-x64-v2`, `linux-arm64`. +3. **Build & package deps in CI** — dispatch `build-deps-{macos,windows,linux}.yml` with `release_tag`. They build both x64 tiers, cut the v2 delta, gate the composed v2 tree, and only then upload each zip **and** its `.sha256.json` sidecar. +4. **Check the deps release** — eight zips, each with a sidecar: the five bundles `macos-arm64`, `macos-x64`, `windows-x64`, `linux-x64`, `linux-arm64`, and the three deltas `macos-x64-v2-delta`, `windows-x64-v2-delta`, `linux-x64-v2-delta`. There must be **no** full `…-x64-v2.zip`. 5. **Test** — fresh install + upgrade test 6. **Create GitHub releases** — deps release first (if changed, tag `deps-vX.Y.Z`), then app release (tag `vX.Y.Z`) @@ -1765,3 +1851,4 @@ Full write-ups (root causes, measurements) for each entry are in | 1.10.0 | 2026-08-31 | **Issue #82**: zsmooth now ships one build per x86 CPU baseline (haswell, x86_64_v2), fixing an illegal-instruction crash on pre-2013 CPUs. **FFmpeg pinned to 9.0 on all four platforms** (they had silently diverged: Windows on an unpinned post-9.0 master, macOS on floating 9.0.1, Linux stuck at 7.1 after BtbN garbage-collected the pinned tag) | | 1.11.0 | 2026-09 | **Issue #92**: every x86 bundle ships in two **CPU tiers** — `v3` (x86-64-v3; the plain asset names) and `v2` (`…-x64-v2`, anything older), chosen by the app from `vapourbox-worker --probe-cpu`. Each bundle carries one autoloaded zsmooth (replacing 1.10.0's two-builds-loaded-by-path). macOS v2 builds MVTools v24 from source without its AVX2 files, whose static initializers SIGILLed inside `dlopen` on pre-AVX Macs. v2 bundles are gated (SDE on Linux/Windows, static check on macOS) before publishing | | 1.12.0 | 2026-10-04 | **Issue #101**: akarin 1.4.1 → **1.5.0** on every platform. The 1.4.1 Linux wheel put JIT code on the `brk` heap, where SELinux denies `PROT_EXEC`, so every job segfaulted on Fedora/RHEL from deps 1.8.0 on. `download-deps-linux.sh` now fails if `libakarin.so` lacks the mmap allocator | +| 1.13.0 | 2026-10 | **Packaging only — every binary is built from the same sources as 1.12.0.** The v2 CPU tier is no longer a second full bundle: each x64 platform publishes its v3 bundle plus `…-x64-v2-delta.zip`, the files in `_tierFiles` (zsmooth; on macOS also MVTools), extracted over it (`"tierFormat": "delta"`). Every x64 machine downloads the same bundle; only an older CPU fetches the few MB more. This is also what lets the Linux AppImage carry its deps for both tiers without doubling in size | diff --git a/README.md b/README.md index 7b4e941..64e5c10 100644 --- a/README.md +++ b/README.md @@ -39,8 +39,8 @@ VapourBox runs [VapourSynth](https://www.vapoursynth.com/), QTGMC and FFmpeg — | macOS (Apple Silicon) | macOS 15 Sequoia | `VapourBox-x.x.x-macos-arm64.dmg` | | macOS (Intel) | macOS 12 Monterey | `VapourBox-x.x.x-macos-x64.dmg` | | Windows (x64) | Windows 10/11 | `VapourBox-x.x.x-windows-x64-setup.exe` | -| Linux (x64) | glibc 2.39 (Ubuntu 24.04, Debian 13) | `VapourBox-x.x.x-linux-x64.AppImage` | -| Linux (arm64) | glibc 2.39 (Ubuntu 24.04, Debian 13) | `VapourBox-x.x.x-linux-arm64.AppImage` | +| Linux (x64) | glibc 2.39 (Ubuntu 24.04, Debian 13) | `VapourBox-x.x.x-x86_64.AppImage` | +| Linux (arm64) | glibc 2.39 (Ubuntu 24.04, Debian 13) | `VapourBox-x.x.x-aarch64.AppImage` | All processing is local. There is no account, no telemetry, and nothing is uploaded. @@ -72,20 +72,20 @@ It installs for the current user only (under `%LOCALAPPDATA%\Programs\VapourBox`
Installing on Linux -1. Download the `.AppImage` for your architecture. -2. Make it executable: `chmod +x VapourBox-x.x.x-linux-x64.AppImage` (or tick *Allow executing file as program* in the file manager's Properties). -3. Run it: `./VapourBox-x.x.x-linux-x64.AppImage` -4. First launch downloads its processing dependencies (~185 MB). +1. Download the `.AppImage` for your architecture (`x86_64` for Intel/AMD, `aarch64` for ARM). +2. Make it executable: `chmod +x VapourBox-x.x.x-x86_64.AppImage` (or tick *Allow executing file as program* in the file manager's Properties). +3. Run it: `./VapourBox-x.x.x-x86_64.AppImage` +4. First launch unpacks the processing dependencies it carries. No download is needed, so it works offline. The AppImage is a single file that needs no installation and can live anywhere. To get a menu entry and icon, open it with an AppImage manager such as [Gear Lever](https://github.com/mijorus/gearlever) or [AppImageLauncher](https://github.com/TheAssassin/AppImageLauncher); those, and [AppImageUpdate](https://github.com/AppImageCommunity/AppImageUpdate), can also update it in place, downloading only what changed. -Each release also carries a plain `.tar.gz` of the same build, for systems where an AppImage can't be mounted: extract it and run `./vapourbox`. +Each release also carries a plain `.tar.gz` of the same build, for systems where an AppImage can't be mounted: extract it and run `./vapourbox`. It is much smaller because it leaves the processing dependencies out, and downloads them on first launch (~240 MB) instead. GPU-accelerated deinterlacing (NNEDI3CL) needs your GPU's OpenCL driver installed. Without it, VapourBox falls back to the CPU automatically.
-**Older Intel and AMD processors** (before about 2013 — no AVX2) are supported on all three platforms. On first launch VapourBox checks the processor and downloads the matching set of processing components; a machine moved to a different CPU gets the right set on its next launch. The older-CPU set runs some filters more slowly, never differently. Every build of that set is checked for pre-AVX processors (e.g. a 2010 Mac Pro) on macOS and Linux; on Windows it is checked for processors with AVX but no AVX2, and older ones are expected to work but can't be tested automatically. +**Older Intel and AMD processors** (before about 2013 — no AVX2) are supported on all three platforms. On first launch VapourBox checks the processor and installs the matching set of processing components; a machine moved to a different CPU gets the right set on its next launch. The older-CPU set runs some filters more slowly, never differently. Every build of that set is checked for pre-AVX processors (e.g. a 2010 Mac Pro) on macOS and Linux; on Windows it is checked for processors with AVX but no AVX2, and older ones are expected to work but can't be tested automatically. ## Output formats diff --git a/Scripts/deps-expected-plugins.json b/Scripts/deps-expected-plugins.json index a0f1784..d3a2740 100644 --- a/Scripts/deps-expected-plugins.json +++ b/Scripts/deps-expected-plugins.json @@ -1,5 +1,18 @@ { "_comment": "Required VapourSynth plugin filenames per platform — the contract for a COMPLETE deps bundle. The package-deps-* scripts assert every file listed here exists in the staged bundle's plugin directory before zipping and FAIL the build if any are missing, so a silently-failed download (e.g. a dead upstream URL) becomes a red build instead of an incomplete bundle shipping. Plugin directory: windows-x64 = vapoursynth/vs-plugins, macos/linux = vapoursynth/plugins. Lists exclude data files (nnedi3 weights) and runtime libs (fftw); they cover the VapourSynth plugin binaries only. Update this when adding or removing a plugin. The x64 platforms ship in two CPU tiers (issue #92): the plain key is the v3 bundle, the -v2 key the bundle for CPUs below x86-64-v3. Both tiers carry the same set of plugins, built differently, so their lists must stay identical (asserted by app/test/attribution_test.dart).", + "_tierFilesComment": "The files that differ between an x64 platform's two CPU tiers, as paths from the bundle root. From deps 1.13.0 the v2 tier is not published as a second full bundle: Scripts/make-deps-delta.py takes exactly these files from the v2 build and publishes them as VapourBox-deps---v2-delta.zip, which the app extracts over the v3 bundle. A file that must differ on older CPUs but is missing here ships as its v3 build and crashes there, so add it in the same change as the download-deps-* tier block that builds it differently. app/test/attribution_test.dart asserts each entry is a plugin the platform lists.", + "_tierFiles": { + "windows-x64": [ + "vapoursynth/vs-plugins/zsmooth.dll" + ], + "macos-x64": [ + "vapoursynth/plugins/libmvtools.dylib", + "vapoursynth/plugins/libzsmooth.dylib" + ], + "linux-x64": [ + "vapoursynth/plugins/libzsmooth.so" + ] + }, "windows-x64": [ "AddGrain.dll", "bifrost.dll", diff --git a/Scripts/make-deps-delta.py b/Scripts/make-deps-delta.py new file mode 100755 index 0000000..3ac4c74 --- /dev/null +++ b/Scripts/make-deps-delta.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +"""Turn a full v2-tier deps bundle into a delta over the v3 bundle. + +From deps 1.13.0 an x64 platform publishes ONE full bundle (the v3 tier, under +the plain asset name) plus a small delta for older CPUs: + + VapourBox-deps--.zip full bundle, v3 tier + VapourBox-deps---v2-delta.zip the files v2 replaces + +The app (and every CI consumer) installs the v2 tier by extracting the delta +over the full bundle. The delta holds exactly the files listed for the platform +under `_tierFiles` in Scripts/deps-expected-plugins.json, taken from a real v2 +build, plus a version.json stamped `tier: v2` — so "unzip the bundle, unzip the +delta over it" yields a correct tree with nothing else to do. + +It is NOT made by diffing the two builds: two builds of the same sources are +not byte-identical, so a diff names nearly every binary. What IS checked is +that the two builds hold the same set of paths, which is what makes a pure +overwrite sufficient (a file only one tier has would need a delete step). + +The delta records the sha256 of the bundle it was cut against (`baseSha256`, +in its version.json and its sidecar). The app refuses to apply it over any +other bundle, so a rebuilt v3 can never be paired with a stale delta. + +Usage: + make-deps-delta.py --platform linux-x64 --version 1.13.0 \\ + --base dist/VapourBox-deps-1.13.0-linux-x64.zip \\ + --v2 build/VapourBox-deps-1.13.0-linux-x64-v2.zip \\ + --out dist [--composed dist-composed] + +--composed also writes the full v2 bundle obtained by applying the delta to the +base (VapourBox-deps---v2.zip). That file is a CI +intermediate for the CPU gate, which must test what a user ends up with rather +than the v2 build the delta was cut from. It is never published. +""" +import argparse +import datetime +import hashlib +import json +import os +import sys +import zipfile + +MANIFEST = os.path.join(os.path.dirname(os.path.abspath(__file__)), + "deps-expected-plugins.json") +VERSION_FILE = "version.json" + + +def norm(name): + # Compress-Archive writes backslash separators on Windows; everything here + # compares and writes forward slashes. + return name.replace("\\", "/") + + +def sha256_of(path): + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def file_entries(zf): + """{normalized path: ZipInfo} for every non-directory entry.""" + return {norm(i.filename): i for i in zf.infolist() if not i.is_dir()} + + +def copy_entry(dst, src, info, name): + """Copy one entry, keeping its mode bits (and so its symlink-ness).""" + out = zipfile.ZipInfo(name, date_time=info.date_time) + out.external_attr = info.external_attr + out.create_system = info.create_system + out.compress_type = zipfile.ZIP_DEFLATED + dst.writestr(out, src.read(info)) + + +def write_sidecar(zip_path, version, extra=None): + sidecar = { + "filename": os.path.basename(zip_path), + "sha256": sha256_of(zip_path), + "size": os.path.getsize(zip_path), + "version": version, + } + sidecar.update(extra or {}) + with open(zip_path + ".sha256.json", "w") as f: + json.dump(sidecar, f, indent=2) + f.write("\n") + return sidecar + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--platform", required=True, help="e.g. linux-x64") + ap.add_argument("--version", required=True) + ap.add_argument("--base", required=True, help="the published v3 bundle zip") + ap.add_argument("--v2", required=True, help="a full v2-tier build zip") + ap.add_argument("--out", required=True, help="directory for the delta zip") + ap.add_argument("--composed", help="also write base+delta as a full zip here") + args = ap.parse_args() + + tier_files = json.load(open(MANIFEST, encoding="utf-8")) \ + .get("_tierFiles", {}).get(args.platform) + if not tier_files: + sys.exit(f"no _tierFiles entry for {args.platform} in {MANIFEST}") + + base_sha = sha256_of(args.base) + version_json = json.dumps({ + "version": args.version, + "tier": "v2", + "baseSha256": base_sha, + "installedAt": datetime.datetime.now(datetime.timezone.utc) + .strftime("%Y-%m-%dT%H:%M:%SZ"), + }, indent=2) + "\n" + + os.makedirs(args.out, exist_ok=True) + stem = f"VapourBox-deps-{args.version}-{args.platform}" + delta_path = os.path.join(args.out, f"{stem}-v2-delta.zip") + + with zipfile.ZipFile(args.base) as base, zipfile.ZipFile(args.v2) as v2: + base_files, v2_files = file_entries(base), file_entries(v2) + + for label, zf, files in (("base", args.base, base_files), ("v2", args.v2, v2_files)): + if VERSION_FILE not in files: + sys.exit(f"{zf} has no {VERSION_FILE}; not a deps bundle") + tier = json.loads(zipfile.ZipFile(zf).read(files[VERSION_FILE])).get("tier", "v3") + want = "v3" if label == "base" else "v2" + if tier != want: + sys.exit(f"{zf} is a {tier} bundle; --{label} must be the {want} one") + + # A pure overwrite only works if neither tier has a file the other lacks. + only_base = sorted(set(base_files) - set(v2_files)) + only_v2 = sorted(set(v2_files) - set(base_files)) + if only_base or only_v2: + for p in only_base: + print(f" only in the v3 bundle: {p}", file=sys.stderr) + for p in only_v2: + print(f" only in the v2 build: {p}", file=sys.stderr) + sys.exit("the two tiers do not hold the same files, so extracting a " + "delta over the v3 bundle cannot reproduce the v2 one") + + missing = [p for p in tier_files if p not in v2_files] + if missing: + sys.exit("_tierFiles names files the v2 build does not contain: " + + ", ".join(missing)) + + # A tier file identical in both builds was not built differently at + # all, which means the tier switch in download-deps-* did nothing. + same = [p for p in tier_files + if base.read(base_files[p]) == v2.read(v2_files[p])] + if same: + sys.exit("these tier files are byte-identical in both tiers, so the " + "v2 build did not actually build them for older CPUs: " + + ", ".join(same)) + + with zipfile.ZipFile(delta_path, "w", zipfile.ZIP_DEFLATED) as delta: + for p in tier_files: + copy_entry(delta, v2, v2_files[p], p) + delta.writestr(VERSION_FILE, version_json) + + if args.composed: + os.makedirs(args.composed, exist_ok=True) + composed_path = os.path.join(args.composed, f"{stem}-v2.zip") + with zipfile.ZipFile(delta_path) as delta, \ + zipfile.ZipFile(composed_path, "w", zipfile.ZIP_DEFLATED) as out: + overlay = file_entries(delta) + for p, info in base_files.items(): + if p not in overlay: + copy_entry(out, base, info, p) + for p, info in overlay.items(): + copy_entry(out, delta, info, p) + write_sidecar(composed_path, args.version) + print(f"composed: {composed_path}") + + sidecar = write_sidecar(delta_path, args.version, + {"tier": "v2", "baseSha256": base_sha}) + print(f"delta: {delta_path} ({sidecar['size']} bytes, " + f"{len(tier_files)} file(s))") + print(f"base: {os.path.basename(args.base)} sha256 {base_sha}") + + +if __name__ == "__main__": + main() diff --git a/Scripts/package-linux.sh b/Scripts/package-linux.sh index 215a3f7..ba711e4 100755 --- a/Scripts/package-linux.sh +++ b/Scripts/package-linux.sh @@ -10,13 +10,19 @@ # architecture — appimagetool is fetched automatically, or set $APPIMAGETOOL # # Usage: ./Scripts/package-linux.sh --version X.Y.Z [--skip-build] [--arch x64|arm64] -# [--skip-appimage] +# [--skip-appimage] [--bundle-deps DIR] +# +# --bundle-deps DIR packs the deps zips found in DIR into the AppImage (never +# the tarball), so it runs with no network on first launch. DIR must hold the +# release assets for the version app/assets/deps-version.json names, each with +# its .sha256.json: the platform bundle, and on x64 the v2 delta as well. set -e VERSION="1.0.0" SKIP_BUILD=false SKIP_APPIMAGE=false +BUNDLE_DEPS="" ARCH="" # Pinned, and checksummed below: this binary assembles what users run. @@ -30,9 +36,10 @@ while [[ $# -gt 0 ]]; do --skip-build) SKIP_BUILD=true; shift ;; --arch) ARCH="$2"; shift 2 ;; --skip-appimage) SKIP_APPIMAGE=true; shift ;; + --bundle-deps) BUNDLE_DEPS="$(cd "$2" && pwd)"; shift 2 ;; *) echo "Unknown option: $1" - echo "Usage: $0 --version X.Y.Z [--skip-build] [--arch x64|arm64] [--skip-appimage]" + echo "Usage: $0 --version X.Y.Z [--skip-build] [--arch x64|arm64] [--skip-appimage] [--bundle-deps DIR]" exit 1 ;; esac @@ -135,9 +142,13 @@ STEP=$((STEP + 1)) # Create AppImage # # The AppImage holds exactly the tree the tarball does, under usr/lib/vapourbox, -# so the two cannot drift. Nothing inside it is written to at runtime: deps and -# add-ons install under $XDG_DATA_HOME, which is also why the processing -# dependencies stay a first-launch download rather than being packed in here. +# so the two cannot drift — plus, with --bundle-deps, the deps zips. Nothing +# inside it is written to at runtime: deps and add-ons install under +# $XDG_DATA_HOME, and a bundled zip is only ever read, by DependencyManager, +# which extracts it there instead of downloading it. The zips are bundled as +# published, not unpacked: the app verifies each against its sidecar and +# installs it through the same path as a download, and an older CPU gets the +# v2 delta extracted over the same bundle (see "CPU tiers" in CLAUDE.md). APPIMAGE_FILE="" if ! $SKIP_APPIMAGE; then echo "[$STEP/$TOTAL_STEPS] Creating AppImage..." @@ -199,9 +210,42 @@ if ! $SKIP_APPIMAGE; then rm -rf "$APPDIR" mkdir -p "$APPDIR/usr/lib" \ "$APPDIR/usr/share/applications" \ + "$APPDIR/usr/share/metainfo" \ "$APPDIR/usr/share/icons/hicolor/256x256/apps" cp -r "$PACKAGE_DIR" "$APPDIR/usr/lib/vapourbox" + # Bundled deps. The app looks for exactly the file names its own + # deps-version.json derives, so a zip of any other version would be ~240 MB + # of dead weight that nothing reports; require the right ones, and check + # each against its sidecar now rather than have every user's first launch + # discover a bad copy and fall back to downloading. + if [ -n "$BUNDLE_DEPS" ]; then + DEPS_VERSION=$(sed -n 's/.*"version": *"\([^"]*\)".*/\1/p' \ + "$PROJECT_ROOT/app/assets/deps-version.json" | head -1) + BUNDLED_ZIPS=("VapourBox-deps-$DEPS_VERSION-linux-$ARCH.zip") + if [ "$ARCH" = "x64" ]; then + BUNDLED_ZIPS+=("VapourBox-deps-$DEPS_VERSION-linux-$ARCH-v2-delta.zip") + fi + mkdir -p "$APPDIR/usr/lib/vapourbox/bundled-deps" + for ZIP in "${BUNDLED_ZIPS[@]}"; do + if [ ! -f "$BUNDLE_DEPS/$ZIP" ] || [ ! -f "$BUNDLE_DEPS/$ZIP.sha256.json" ]; then + echo "ERROR: --bundle-deps: $BUNDLE_DEPS has no $ZIP (and its .sha256.json)." + echo " The app expects deps $DEPS_VERSION (app/assets/deps-version.json)." + exit 1 + fi + WANT=$(sed -n 's/.*"sha256": *"\([^"]*\)".*/\1/p' "$BUNDLE_DEPS/$ZIP.sha256.json") + if ! echo "$WANT $BUNDLE_DEPS/$ZIP" | sha256sum -c - >/dev/null; then + echo "ERROR: --bundle-deps: $ZIP does not match its .sha256.json." + exit 1 + fi + cp "$BUNDLE_DEPS/$ZIP" "$BUNDLE_DEPS/$ZIP.sha256.json" \ + "$APPDIR/usr/lib/vapourbox/bundled-deps/" + echo " Bundled $ZIP" + done + else + echo " No --bundle-deps: this AppImage will download its deps on first launch." + fi + install -m 755 "$LINUX_PACKAGING/AppRun" "$APPDIR/AppRun" # appimagetool wants the desktop file and its icon at the root; the copies # under usr/share are what desktop-integration tools install from. @@ -211,24 +255,57 @@ if ! $SKIP_APPIMAGE; then cp "$LINUX_PACKAGING/$APP_ID.png" "$APPDIR/usr/share/icons/hicolor/256x256/apps/" ln -s "$APP_ID.png" "$APPDIR/.DirIcon" + # AppStream metadata, with this release stamped in. + sed -e "s/@VERSION@/$VERSION/g" -e "s/@DATE@/$(date -u +%Y-%m-%d)/g" \ + "$LINUX_PACKAGING/$APP_ID.appdata.xml" \ + > "$APPDIR/usr/share/metainfo/$APP_ID.appdata.xml" + # Validated here, offline, when the tools are present (CI installs them): + # appimagetool's own check goes to the network for the screenshot, whose + # URL is this release's tag and does not exist until the release does. + if command -v desktop-file-validate >/dev/null 2>&1; then + desktop-file-validate "$APPDIR/$APP_ID.desktop" + fi + if command -v appstreamcli >/dev/null 2>&1; then + appstreamcli validate --no-net "$APPDIR/usr/share/metainfo/$APP_ID.appdata.xml" + fi + + # Named the way AppImages are: App-version-arch, with the machine's own + # architecture name and no "linux" (every AppImage is for Linux; the + # AppImage catalog flags it). The tarball keeps linux-$ARCH. + # # "latest" follows GitHub's Latest release, which is why a deps or whisper # release must never be marked Latest (see CLAUDE.md). The wildcard stands # in for the version; the arch stays literal so x64 never updates to arm64. - APPIMAGE_FILE="$PACKAGE_NAME.AppImage" - UPDATE_INFO="gh-releases-zsync|${GITHUB_REPO%%/*}|${GITHUB_REPO##*/}|latest|VapourBox-*-linux-$ARCH.AppImage.zsync" + APPIMAGE_FILE="VapourBox-$VERSION-$AI_ARCH.AppImage" + UPDATE_INFO="gh-releases-zsync|${GITHUB_REPO%%/*}|${GITHUB_REPO##*/}|latest|VapourBox-*-$AI_ARCH.AppImage.zsync" + # 1.2.0, the only release under the old name, embedded the pattern + # VapourBox-*-linux-.AppImage.zsync. Publishing this release's .zsync + # under that name as well lets a 1.2.0 AppImage find it: the file's URL + # header is relative and names the new AppImage, which then carries the new + # pattern. Drop this once 1.2.0 is no longer worth updating from. + LEGACY_ZSYNC="VapourBox-$VERSION-linux-$ARCH.AppImage.zsync" # appimagetool writes the .zsync into the working directory. It is itself # an AppImage; extract-and-run spares the build host from needing FUSE. cd "$DIST_DIR" - rm -f "$APPIMAGE_FILE" "$APPIMAGE_FILE.zsync" + rm -f "$APPIMAGE_FILE" "$APPIMAGE_FILE.zsync" "$LEGACY_ZSYNC" ARCH="$AI_ARCH" APPIMAGE_EXTRACT_AND_RUN=1 VERSION="$VERSION" \ - "$APPIMAGETOOL" --updateinformation "$UPDATE_INFO" "$APPDIR" "$APPIMAGE_FILE" + "$APPIMAGETOOL" --no-appstream --updateinformation "$UPDATE_INFO" \ + "$APPDIR" "$APPIMAGE_FILE" if [ ! -s "$APPIMAGE_FILE" ] || [ ! -s "$APPIMAGE_FILE.zsync" ]; then echo "ERROR: appimagetool did not produce both $APPIMAGE_FILE and $APPIMAGE_FILE.zsync" exit 1 fi chmod +x "$APPIMAGE_FILE" + # The bridge only works if the .zsync points at the AppImage by a relative + # name; an absolute or differently named URL would send 1.2.0 nowhere. + if ! grep -a -q "^URL: $APPIMAGE_FILE\$" "$APPIMAGE_FILE.zsync"; then + echo "ERROR: $APPIMAGE_FILE.zsync does not name $APPIMAGE_FILE as its URL;" + echo " the copy published as $LEGACY_ZSYNC would not update 1.2.0." + exit 1 + fi + cp "$APPIMAGE_FILE.zsync" "$LEGACY_ZSYNC" rm -rf "$APPDIR" STEP=$((STEP + 1)) @@ -260,6 +337,7 @@ if [ -n "$APPIMAGE_FILE" ]; then echo " Size: $(file_size_mb "$APPIMAGE_FILE") MB" echo " SHA256: $(file_sha256 "$APPIMAGE_FILE")" echo " Update: $DIST_DIR/$APPIMAGE_FILE.zsync (upload it beside the AppImage)" + echo " Bridge: $DIST_DIR/$LEGACY_ZSYNC (upload it too; updates 1.2.0)" echo "" fi echo " Tarball: $DIST_DIR/$TAR_FILE" @@ -275,4 +353,8 @@ echo " tar -xzf $TAR_FILE" echo " cd $PACKAGE_NAME" echo " ./vapourbox" echo "" -echo "Note: Dependencies will be downloaded on first launch." +if [ -n "$APPIMAGE_FILE" ] && [ -n "$BUNDLE_DEPS" ]; then + echo "Note: the AppImage carries its dependencies; the tarball downloads them on first launch." +else + echo "Note: Dependencies will be downloaded on first launch." +fi diff --git a/Scripts/release.sh b/Scripts/release.sh index fa95031..d71043c 100755 --- a/Scripts/release.sh +++ b/Scripts/release.sh @@ -306,8 +306,8 @@ RELEASE_NOTES="## VapourBox $APP_VERSION - **Windows**: \`VapourBox-$APP_VERSION-windows-x64-setup.exe\` (or the portable \`.zip\`) - **macOS (Apple Silicon)**: \`VapourBox-$APP_VERSION-macos-arm64.zip\` (contains signed & notarized DMG) - **macOS (Intel)**: \`VapourBox-$APP_VERSION-macos-x64.zip\` (contains signed & notarized DMG) -- **Linux (x64)**: \`VapourBox-$APP_VERSION-linux-x64.AppImage\` (or the \`.tar.gz\`) -- **Linux (arm64)**: \`VapourBox-$APP_VERSION-linux-arm64.AppImage\` (or the \`.tar.gz\`) +- **Linux (x64)**: \`VapourBox-$APP_VERSION-x86_64.AppImage\` (or the \`.tar.gz\`) +- **Linux (arm64)**: \`VapourBox-$APP_VERSION-aarch64.AppImage\` (or the \`.tar.gz\`) ### Installation - **Windows**: Run the installer — or extract the zip and run \`vapourbox.exe\` diff --git a/app/assets/deps-version.json b/app/assets/deps-version.json index 07387b3..17386ec 100644 --- a/app/assets/deps-version.json +++ b/app/assets/deps-version.json @@ -1,6 +1,7 @@ { - "version": "1.12.0", - "releaseTag": "deps-v1.12.0", - "releaseDate": "2026-10-04", + "version": "1.13.0", + "releaseTag": "deps-v1.13.0", + "releaseDate": "2026-10-09", + "tierFormat": "delta", "githubRepo": "StuartCameronCode/VapourBox" } diff --git a/app/lib/services/dependency_manager.dart b/app/lib/services/dependency_manager.dart index cad0dd9..411ec7b 100644 --- a/app/lib/services/dependency_manager.dart +++ b/app/lib/services/dependency_manager.dart @@ -98,14 +98,23 @@ class DepsVersionInfo { final Map platforms; final String githubRepo; + /// How the release ships the v2 CPU tier of an x64 platform. Absent means a + /// second full bundle (`…-x64-v2.zip`, deps 1.11.0–1.12.0); `delta` means a + /// small zip of the files that differ (`…-x64-v2-delta.zip`, deps 1.13.0 on), + /// extracted over the v3 bundle. See [DependencyManager.installAssetIds]. + final String? tierFormat; + DepsVersionInfo({ required this.version, required this.releaseTag, this.releaseDate, required this.platforms, required this.githubRepo, + this.tierFormat, }); + bool get tierIsDelta => tierFormat == 'delta'; + factory DepsVersionInfo.fromJson(Map json) { final platforms = {}; final platformsJson = json['platforms'] as Map? ?? {}; @@ -121,6 +130,7 @@ class DepsVersionInfo { releaseDate: json['releaseDate'] as String?, platforms: platforms, githubRepo: json['githubRepo'] as String? ?? 'StuartCameronCode/VapourBox', + tierFormat: json['tierFormat'] as String?, ); } @@ -260,6 +270,81 @@ class DependencyManager { static String assetIdFor(String platformId, String? tier) => tier == 'v2' ? '$platformId-v2' : platformId; + /// The release assets that make up an install, in extraction order. + /// + /// One zip, except for the v2 tier of a release whose + /// [DepsVersionInfo.tierFormat] is `delta`: that is the platform's v3 bundle + /// followed by `-v2-delta`, the handful of files built differently + /// for older CPUs, extracted over it. So every x64 machine fetches the same + /// bundle and only an older one fetches anything more. + static List installAssetIds(String platformId, String? tier, + {required bool delta}) => + tier == 'v2' && delta + ? [platformId, '$platformId-v2-delta'] + : [assetIdFor(platformId, tier)]; + + /// Test seam: replaces [bundledDepsDirectory]. + Directory? bundledDepsDirOverride; + + /// Where a package may ship deps zips so the first launch needs no network: + /// `bundled-deps/` beside the executable. The Linux AppImage fills it + /// (Scripts/package-linux.sh --bundle-deps); nothing else does yet. + /// + /// Deliberately one fixed path inside the app's own tree, never a search: a + /// zip found here is extracted and its binaries run, so it gets exactly the + /// trust the executable next to it has, and no more places than that. + Directory bundledDepsDirectory() => + bundledDepsDirOverride ?? + Directory(path.join( + path.dirname(Platform.resolvedExecutable), 'bundled-deps')); + + /// A bundled copy of [filename], or null if the package ships none or it + /// does not match the sidecar shipped beside it. + /// + /// The sidecar is required, not best-effort as it is for a download: with no + /// network there is nothing else to say the zip is whole, and falling back to + /// the download costs nothing when the bundled copy is unusable. + static Future findBundledZip(Directory dir, String filename) async { + final zip = File(path.join(dir.path, filename)); + final sidecar = File('${zip.path}.sha256.json'); + if (!await zip.exists() || !await sidecar.exists()) return null; + try { + final expected = + (jsonDecode(await sidecar.readAsString()) as Map)['sha256']; + if (expected is! String || expected.isEmpty) return null; + final actual = await sha256OfFile(zip); + if (actual != expected) { + print('DependencyManager: bundled $filename does not match its ' + 'sidecar ($actual, expected $expected) - ignoring it'); + return null; + } + return zip; + } catch (e) { + print('DependencyManager: could not use bundled $filename ($e)'); + return null; + } + } + + /// The sha256 of the bundle a v2 delta was cut against, read from the + /// `version.json` inside the delta, or null if it records none. + static String? deltaBaseSha256(Archive delta) { + for (final file in delta) { + if (file.isFile && _isVersionFile(file.name)) { + try { + final json = jsonDecode(utf8.decode(file.content as List)); + final sha = json is Map ? json['baseSha256'] : null; + return sha is String && sha.isNotEmpty ? sha : null; + } catch (_) { + return null; + } + } + } + return null; + } + + static bool _isVersionFile(String entryName) => + entryName.replaceAll('\\', '/') == 'version.json'; + String? _cachedTier; bool _tierResolved = false; @@ -658,65 +743,32 @@ class DependencyManager { final tier = await depsTier(); final assetId = assetIdFor(platformId, tier); - // Construct download URL from release tag (filename is derived). The asset - // carries the tier; the install directory does not. - final downloadUrl = expected.getDownloadUrl(assetId); - final filename = expected.filenameFor(assetId); - - print('DependencyManager: Downloading from $downloadUrl'); + // What to install, in extraction order: one bundle, or for the v2 tier of + // a delta-format release the v3 bundle and then the delta over it. The + // assets carry the tier; the install directory does not. + final assetIds = + installAssetIds(platformId, tier, delta: expected.tierIsDelta); + final filenames = [for (final id in assetIds) expected.filenameFor(id)]; _progressController.add(DownloadProgress( bytesReceived: 0, totalBytes: 0, - status: 'Connecting...', + status: 'Preparing...', )); - // Fetch the integrity sidecar (sha256) uploaded next to the zip. Verification - // is best-effort: if the sidecar is missing/unreadable we still install (the - // download is over HTTPS), matching prior behaviour when no hash was set. - final expectedSha256 = - await _fetchExpectedSha256(expected.getManifestUrl(assetId)); - - // The zip is downloaded into a stable cache path rather than a throwaway - // temp directory, and kept if anything after the download fails. Everything - // past this point — extraction, verification, the swap — can fail for - // reasons that have nothing to do with the bytes we just fetched (issue - // #87), and making the user re-fetch ~200 MB to retry a rename is a poor - // trade for the disk the zip occupies until the install succeeds. - final tempFile = File(await _cachedDownloadPath(filename)); + // Zips this install fetched into the cache, as opposed to ones the package + // shipped. Only these are ours to delete. + final zips = []; + final cached = []; try { - // Reuse the cached zip only when the sidecar gave us a hash to check it - // against. Without one an interrupted download is indistinguishable from - // a complete one, and extracting a truncated zip would report a corrupt - // bundle rather than the missing bytes. - var reusable = false; - if (await tempFile.exists()) { - if (expectedSha256 != null) { - _progressController.add(DownloadProgress( - bytesReceived: 0, - totalBytes: 0, - status: 'Checking the downloaded file...', - )); - reusable = await _sha256OfFile(tempFile) == expectedSha256; - print('DependencyManager: cached download ${reusable ? 'matches the ' - 'expected hash - skipping the download' : 'does not match the ' - 'expected hash - downloading again'}'); - } - if (!reusable) { - await tempFile.delete().catchError((_) => tempFile); - } - } - - if (!reusable) { - await _downloadFile( - downloadUrl, - tempFile, - expectedSha256: expectedSha256, - ); + for (final id in assetIds) { + final zip = await _obtainZip(expected, id, keep: filenames); + zips.add(zip.file); + if (!zip.bundled) cached.add(zip.file); } - // Extract. _extractZip emits per-file extraction progress; this initial + // Extract. extractBundle emits per-file extraction progress; this initial // event (0/0 -> indeterminate) covers the synchronous decode that precedes // the first file write. _progressController.add(DownloadProgress( @@ -725,7 +777,10 @@ class DependencyManager { status: 'Extracting...', )); - final downloadedBytes = await tempFile.length(); + var downloadedBytes = 0; + for (final zip in zips) { + downloadedBytes += await zip.length(); + } // Build the new install alongside the current one and swap it in, rather // than deleting the current one and extracting over the top. The old path @@ -765,7 +820,18 @@ class DependencyManager { final hadPrevious = await depsDir.exists(); final target = hadPrevious ? staging : depsDir; - await _extractZip(tempFile, targetOverride: target); + // A delta is only ever applied over the bundle it was cut against. A + // mismatch means the release was half-updated or a zip is not what its + // name says; either way the result would be a v2 install that is really + // v3 with the wrong plugin dropped in. + if (zips.length > 1) { + await _checkDeltaMatchesBase(base: zips.first, delta: zips.last); + } + + for (var i = 0; i < zips.length; i++) { + await extractBundle(zips[i], target, + overlay: i > 0, finalize: i == zips.length - 1); + } // Prove the install is actually usable before declaring success. The // quarantine strip above can fail — silently, and it cannot succeed at all @@ -869,16 +935,113 @@ class DependencyManager { print('DependencyManager: Installation complete'); - // Only now is the zip dead weight. On any failure above it is deliberately - // left in place so Retry can skip the download. - await tempFile.delete().catchError((_) => tempFile); + // Only now are the downloaded zips dead weight. On any failure above + // they are deliberately left in place so Retry can skip the download. A + // zip the package shipped is never ours to delete. + for (final zip in cached) { + await zip.delete().catchError((_) => zip); + } } catch (e) { - print('DependencyManager: install failed ($e) - keeping the downloaded ' - 'zip at ${tempFile.path} so a retry can reuse it'); + if (cached.isNotEmpty) { + print('DependencyManager: install failed ($e) - keeping the downloaded ' + '${cached.map((f) => f.path).join(', ')} so a retry can reuse ' + 'what was fetched'); + } rethrow; } } + /// One zip of an install: the copy the package shipped if it has a good one, + /// else the cached download, else a fresh download. + Future<({File file, bool bundled})> _obtainZip( + DepsVersionInfo expected, + String assetId, { + required List keep, + }) async { + final filename = expected.filenameFor(assetId); + + final bundled = await findBundledZip(bundledDepsDirectory(), filename); + if (bundled != null) { + print('DependencyManager: using the bundled $filename'); + return (file: bundled, bundled: true); + } + + final downloadUrl = expected.getDownloadUrl(assetId); + print('DependencyManager: Downloading from $downloadUrl'); + _progressController.add(DownloadProgress( + bytesReceived: 0, + totalBytes: 0, + status: 'Connecting...', + )); + + // Fetch the integrity sidecar (sha256) uploaded next to the zip. Verification + // is best-effort: if the sidecar is missing/unreadable we still install (the + // download is over HTTPS), matching prior behaviour when no hash was set. + final expectedSha256 = + await _fetchExpectedSha256(expected.getManifestUrl(assetId)); + + // The zip is downloaded into a stable cache path rather than a throwaway + // temp directory, and kept if anything after the download fails. Everything + // past this point — extraction, verification, the swap — can fail for + // reasons that have nothing to do with the bytes we just fetched (issue + // #87), and making the user re-fetch ~200 MB to retry a rename is a poor + // trade for the disk the zip occupies until the install succeeds. + final tempFile = File(await _cachedDownloadPath(filename, keep: keep)); + + // Reuse the cached zip only when the sidecar gave us a hash to check it + // against. Without one an interrupted download is indistinguishable from + // a complete one, and extracting a truncated zip would report a corrupt + // bundle rather than the missing bytes. + var reusable = false; + if (await tempFile.exists()) { + if (expectedSha256 != null) { + _progressController.add(DownloadProgress( + bytesReceived: 0, + totalBytes: 0, + status: 'Checking the downloaded file...', + )); + reusable = await sha256OfFile(tempFile) == expectedSha256; + print('DependencyManager: cached download ${reusable ? 'matches the ' + 'expected hash - skipping the download' : 'does not match the ' + 'expected hash - downloading again'}'); + } + if (!reusable) { + await tempFile.delete().catchError((_) => tempFile); + } + } + + if (!reusable) { + await _downloadFile( + downloadUrl, + tempFile, + expectedSha256: expectedSha256, + ); + } + return (file: tempFile, bundled: false); + } + + /// Refuse a v2 delta that was not cut against [base]. + Future _checkDeltaMatchesBase( + {required File base, required File delta}) async { + final wanted = + deltaBaseSha256(ZipDecoder().decodeBytes(await delta.readAsBytes())); + final actual = await sha256OfFile(base); + if (wanted == actual) return; + // The cached delta is the likelier stale half (it is tiny and the bundle + // was just verified against its own sidecar), so drop it and let a retry + // fetch it again. + if (!path.isWithin(bundledDepsDirectory().path, delta.path)) { + await delta.delete().catchError((_) => delta); + } + throw DependencyInstallException( + 'The components for older processors do not match the main bundle ' + '(they were built against ${wanted ?? 'an unrecorded bundle'}, but the ' + 'bundle is $actual).', + remedy: 'Try again in a few minutes - a new release may still be ' + 'uploading. If it keeps happening, please report it.', + ); + } + /// What to suggest the user try, given the error that ended the install. /// /// One fixed line of connection advice was actively misleading for anything @@ -928,14 +1091,19 @@ class DependencyManager { /// A fixed name under the (user-configurable) temp directory, so a retry can /// find it. Anything else in there is another version's leftovers and is /// pruned, which is what stops the cache growing without bound. - Future _cachedDownloadPath(String filename) async { + /// + /// [keep] names the other files this install needs: a v2 install has two + /// zips, and fetching the second must not prune the first. + Future _cachedDownloadPath(String filename, + {Iterable keep = const []}) async { final dir = Directory(path.join( (await TempDirectoryService.instance.resolve()).path, 'vapourbox-deps-cache')); if (!await dir.exists()) await dir.create(recursive: true); try { await for (final entry in dir.list()) { - if (entry is File && path.basename(entry.path) != filename) { + final name = path.basename(entry.path); + if (entry is File && name != filename && !keep.contains(name)) { await entry.delete().catchError((_) => entry); } } @@ -946,7 +1114,7 @@ class DependencyManager { } /// Lowercase hex sha256 of [file], in the same form the sidecar publishes. - Future _sha256OfFile(File file) async { + static Future sha256OfFile(File file) async { final digest = await sha256.bind(file.openRead()).first; return digest.toString(); } @@ -1116,23 +1284,34 @@ class DependencyManager { } } - /// Extract a zip file to the deps directory. - Future _extractZip(File zipFile, {Directory? targetOverride}) async { - final depsDir = targetOverride ?? await getDepsDirectory(); - + /// Extract a deps zip into [depsDir]. + /// + /// [overlay] extracts over what is already there instead of replacing it, + /// which is how a v2 delta lands on the v3 bundle. [finalize] runs the + /// once-per-install steps (the macOS quarantine strip and re-sign); pass + /// false for every zip of a multi-zip install but the last. + /// + /// A zip's own `version.json` is never written. [_writeInstalledVersion] + /// writes that file, last, as the mark of a complete install; letting the + /// zip's copy through would make a half-extracted tree look installed, and + /// a v2 install look like v3 until the delta landed. + @visibleForTesting + Future extractBundle(File zipFile, Directory depsDir, + {bool overlay = false, bool finalize = true}) async { // Create parent directory if needed final parentDir = depsDir.parent; if (!await parentDir.exists()) { await parentDir.create(recursive: true); } - // Remove existing deps directory - if (await depsDir.exists()) { - await depsDir.delete(recursive: true); + if (!overlay) { + // Remove existing deps directory + if (await depsDir.exists()) { + await depsDir.delete(recursive: true); + } + await depsDir.create(recursive: true); } - await depsDir.create(recursive: true); - // Read and extract zip final bytes = await zipFile.readAsBytes(); final archive = ZipDecoder().decodeBytes(bytes); @@ -1160,6 +1339,7 @@ class DependencyManager { emitExtractProgress(); for (final file in archive) { + if (_isVersionFile(file.name)) continue; final filePath = path.join(depsDir.path, file.name); if (file.isFile) { @@ -1187,7 +1367,7 @@ class DependencyManager { emitExtractProgress(); // On macOS, remove quarantine attribute and re-sign binaries for Gatekeeper - if (Platform.isMacOS) { + if (finalize && Platform.isMacOS) { await _removeQuarantine(depsDir.path); await _codesignBinaries(depsDir.path); } diff --git a/app/lib/views/dependency_download_dialog.dart b/app/lib/views/dependency_download_dialog.dart index 31c14e0..40b3a54 100644 --- a/app/lib/views/dependency_download_dialog.dart +++ b/app/lib/views/dependency_download_dialog.dart @@ -100,17 +100,17 @@ class _DependencyDownloadDialogState extends State { switch (widget.status) { case DependencyStatus.missing: return 'VapourBox requires additional components to process videos.\n\n' - 'These will be downloaded automatically.'; + 'These will be installed automatically.'; case DependencyStatus.outdated: return 'A new version of the processing components is available.\n\n' 'Updating to ensure compatibility.'; case DependencyStatus.wrongTier: return 'The installed processing components were built for a ' 'different processor.\n\n' - 'Downloading the version for this computer.'; + 'Installing the version for this computer.'; case DependencyStatus.corrupted: return 'Some processing components are damaged or incomplete.\n\n' - 'Re-downloading to fix the issue.'; + 'Reinstalling to fix the issue.'; case DependencyStatus.newerThanExpected: // main.dart keeps these and warns instead of opening this dialog; // handled so the status can never fall through to the default. diff --git a/app/test/deps_delta_install_test.dart b/app/test/deps_delta_install_test.dart new file mode 100644 index 0000000..e08712c --- /dev/null +++ b/app/test/deps_delta_install_test.dart @@ -0,0 +1,252 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:archive/archive.dart'; +import 'package:crypto/crypto.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as p; +import 'package:vapourbox/services/dependency_manager.dart'; + +/// From deps 1.13.0 the v2 CPU tier is not a second full bundle: it is the v3 +/// bundle plus a small delta extracted over it, and a package may ship either +/// zip so the first launch needs no network (the Linux AppImage does). These +/// pin which zips make up an install, when a shipped zip is trusted, and that +/// extracting a delta over a bundle produces the v2 tree and nothing else. +void main() { + List zipOf(Map files) { + final archive = Archive(); + files.forEach((name, content) { + final bytes = utf8.encode(content); + archive.addFile(ArchiveFile(name, bytes.length, bytes)); + }); + return ZipEncoder().encode(archive)!; + } + + late Directory tmp; + setUp(() => tmp = Directory.systemTemp.createTempSync('vb_delta_test_')); + tearDown(() => tmp.deleteSync(recursive: true)); + + group('installAssetIds', () { + test('a delta-format v2 install is the v3 bundle, then the delta', () { + for (final id in ['macos-x64', 'windows-x64', 'linux-x64']) { + expect(DependencyManager.installAssetIds(id, 'v2', delta: true), + [id, '$id-v2-delta']); + } + }); + + test('v3 and ARM installs are one bundle in either format', () { + for (final delta in [true, false]) { + expect(DependencyManager.installAssetIds('linux-x64', 'v3', delta: delta), + ['linux-x64']); + expect( + DependencyManager.installAssetIds('linux-arm64', null, delta: delta), + ['linux-arm64']); + } + }); + + test('a release without tierFormat still ships v2 as a full bundle', () { + // Deps 1.11.0-1.12.0. An app pinned to one of those must keep asking for + // the asset that release actually has. + expect(DependencyManager.installAssetIds('linux-x64', 'v2', delta: false), + ['linux-x64-v2']); + }); + + test('the delta resolves to its own zip and sidecar', () { + // Must agree with what Scripts/make-deps-delta.py writes. + final info = DepsVersionInfo.fromJson({ + 'version': '1.13.0', + 'releaseTag': 'deps-v1.13.0', + 'tierFormat': 'delta', + 'githubRepo': 'StuartCameronCode/VapourBox', + }); + expect(info.tierIsDelta, isTrue); + expect(info.filenameFor('linux-x64-v2-delta'), + 'VapourBox-deps-1.13.0-linux-x64-v2-delta.zip'); + expect( + info.getManifestUrl('linux-x64-v2-delta'), + 'https://github.com/StuartCameronCode/VapourBox/releases/download/' + 'deps-v1.13.0/VapourBox-deps-1.13.0-linux-x64-v2-delta.zip.sha256.json'); + }); + + test('tierFormat defaults to full bundles', () { + final info = DepsVersionInfo.fromJson({'version': '1.12.0'}); + expect(info.tierIsDelta, isFalse); + }); + }); + + group('findBundledZip', () { + const name = 'VapourBox-deps-9.9.9-linux-x64.zip'; + + void writeZip(List bytes, {String? sidecarSha}) { + File(p.join(tmp.path, name)).writeAsBytesSync(bytes); + if (sidecarSha != null) { + File(p.join(tmp.path, '$name.sha256.json')) + .writeAsStringSync(jsonEncode({'filename': name, 'sha256': sidecarSha})); + } + } + + test('a zip matching the sidecar shipped beside it is used', () async { + final bytes = zipOf({'ffmpeg/ffmpeg': 'x'}); + writeZip(bytes, sidecarSha: sha256.convert(bytes).toString()); + final found = await DependencyManager.findBundledZip(tmp, name); + expect(found?.path, p.join(tmp.path, name)); + }); + + test('nothing shipped means download', () async { + expect(await DependencyManager.findBundledZip(tmp, name), isNull); + expect( + await DependencyManager.findBundledZip( + Directory(p.join(tmp.path, 'absent')), name), + isNull); + }); + + test('a zip with no sidecar is not trusted', () async { + // Offline there is nothing else to say the zip is whole. + writeZip(zipOf({'ffmpeg/ffmpeg': 'x'})); + expect(await DependencyManager.findBundledZip(tmp, name), isNull); + }); + + test('a zip that does not match its sidecar falls back to download', + () async { + writeZip(zipOf({'ffmpeg/ffmpeg': 'x'}), sidecarSha: '0' * 64); + expect(await DependencyManager.findBundledZip(tmp, name), isNull); + }); + + test('an unreadable sidecar falls back to download', () async { + writeZip(zipOf({'ffmpeg/ffmpeg': 'x'})); + File(p.join(tmp.path, '$name.sha256.json')).writeAsStringSync('not json'); + expect(await DependencyManager.findBundledZip(tmp, name), isNull); + }); + }); + + group('deltaBaseSha256', () { + Archive deltaWith(String versionJson) => ZipDecoder().decodeBytes( + zipOf({'vapoursynth/plugins/libzsmooth.so': 'v2', 'version.json': versionJson})); + + test('reads the bundle the delta was cut against', () { + expect( + DependencyManager.deltaBaseSha256( + deltaWith(jsonEncode({'tier': 'v2', 'baseSha256': 'abc123'}))), + 'abc123'); + }); + + test('a delta that records no base matches nothing', () { + // null never equals a real hash, so such a delta is always refused. + expect(DependencyManager.deltaBaseSha256(deltaWith('{"tier":"v2"}')), isNull); + expect(DependencyManager.deltaBaseSha256(deltaWith('garbage')), isNull); + expect( + DependencyManager.deltaBaseSha256( + ZipDecoder().decodeBytes(zipOf({'a': 'b'}))), + isNull); + }); + }); + + group('extractBundle', () { + final manager = DependencyManager.instance; + + Future zipFile(String name, Map files) async { + final f = File(p.join(tmp.path, name)); + await f.writeAsBytes(zipOf(files)); + return f; + } + + String read(Directory d, String rel) => + File(p.joinAll([d.path, ...rel.split('/')])).readAsStringSync(); + + test('a delta replaces only the files it carries', () async { + final deps = Directory(p.join(tmp.path, 'deps')); + final base = await zipFile('base.zip', { + 'ffmpeg/ffmpeg': 'ffmpeg', + 'vapoursynth/plugins/libzsmooth.so': 'haswell', + 'vapoursynth/plugins/libmvtools.so': 'mvtools', + 'version.json': '{"version":"9.9.9","tier":"v3"}', + }); + final delta = await zipFile('delta.zip', { + 'vapoursynth/plugins/libzsmooth.so': 'x86_64_v2', + 'version.json': '{"version":"9.9.9","tier":"v2","baseSha256":"x"}', + }); + + await manager.extractBundle(base, deps, finalize: false); + await manager.extractBundle(delta, deps, overlay: true, finalize: false); + + expect(read(deps, 'vapoursynth/plugins/libzsmooth.so'), 'x86_64_v2'); + expect(read(deps, 'vapoursynth/plugins/libmvtools.so'), 'mvtools'); + expect(read(deps, 'ffmpeg/ffmpeg'), 'ffmpeg'); + }); + + test('a zip never writes version.json; that is the commit marker', + () async { + // The bundle's own copy says v3. Extracted, it would make an install + // interrupted before the delta look like a complete v3 one on a v2 CPU. + final deps = Directory(p.join(tmp.path, 'deps')); + final base = await zipFile('base.zip', { + 'ffmpeg/ffmpeg': 'ffmpeg', + 'version.json': '{"version":"9.9.9","tier":"v3"}', + }); + final delta = await zipFile('delta.zip', { + 'ffmpeg/ffmpeg': 'v2', + 'version.json': '{"version":"9.9.9","tier":"v2"}', + }); + await manager.extractBundle(base, deps, finalize: false); + expect(File(p.join(deps.path, 'version.json')).existsSync(), isFalse); + await manager.extractBundle(delta, deps, overlay: true, finalize: false); + expect(File(p.join(deps.path, 'version.json')).existsSync(), isFalse); + }); + + test('without overlay the previous tree is replaced, not merged', () async { + final deps = Directory(p.join(tmp.path, 'deps'))..createSync(); + File(p.join(deps.path, 'stale.so')).writeAsStringSync('old'); + final base = await zipFile('base.zip', {'ffmpeg/ffmpeg': 'ffmpeg'}); + await manager.extractBundle(base, deps, finalize: false); + expect(File(p.join(deps.path, 'stale.so')).existsSync(), isFalse); + }); + }); + + group('the shipped pointer and manifest', () { + String repoFile(List parts) { + var dir = Directory.current; + while (!Directory(p.join(dir.path, 'worker')).existsSync() || + !Directory(p.join(dir.path, 'app')).existsSync()) { + dir = dir.parent; + } + return File(p.joinAll([dir.path, ...parts])).readAsStringSync(); + } + + test('every tier file is a plugin its platform is required to ship', () { + final manifest = jsonDecode(repoFile(['Scripts', 'deps-expected-plugins.json'])) + as Map; + final tierFiles = manifest['_tierFiles'] as Map; + // A tiered platform with no tier files would publish an empty delta and + // hand older CPUs the v3 plugins. + expect(tierFiles.keys.toSet(), {'macos-x64', 'windows-x64', 'linux-x64'}); + for (final entry in tierFiles.entries) { + final files = (entry.value as List).cast(); + expect(files, isNotEmpty, reason: entry.key); + final plugins = (manifest[entry.key] as List).cast().toSet(); + final pluginDir = entry.key.startsWith('windows') + ? 'vapoursynth/vs-plugins/' + : 'vapoursynth/plugins/'; + for (final f in files) { + expect(f, startsWith(pluginDir), reason: entry.key); + expect(plugins, contains(f.substring(pluginDir.length)), + reason: '${entry.key}: $f is not in the platform plugin list'); + } + } + }); + + test('no deps workflow can publish a full v2 bundle', () { + // The v2 build is an intermediate. A release holding both it and the + // delta would leave two different answers to "what is the v2 tier". + for (final os in ['macos', 'windows', 'linux']) { + final workflow = + repoFile(['.github', 'workflows', 'build-deps-$os.yml']); + expect(workflow, contains(r'"dist/$STEM-v2-delta.zip"'), reason: os); + expect(workflow, contains(r'"dist/$STEM-v2-delta.zip.sha256.json"'), + reason: os); + expect(workflow, isNot(contains('-x64*.zip')), reason: os); + expect(workflow, isNot(contains(r'dist/${{ env.ASSET }}.zip" \')), + reason: '$os uploads a matrix build straight to the release'); + } + }); + }); +} diff --git a/app/test/packaging_test.dart b/app/test/packaging_test.dart index 4c4ef4d..36d48cf 100644 --- a/app/test/packaging_test.dart +++ b/app/test/packaging_test.dart @@ -144,17 +144,39 @@ void main() { final package = _read(root, ['Scripts', 'package-linux.sh']); test('names the .zsync the script itself produces', () { - expect(package, contains(r'APPIMAGE_FILE="$PACKAGE_NAME.AppImage"')); - expect( - package, contains(r'PACKAGE_NAME="VapourBox-$VERSION-linux-$ARCH"')); + expect(package, + contains(r'APPIMAGE_FILE="VapourBox-$VERSION-$AI_ARCH.AppImage"')); expect( package, - contains(r'|latest|VapourBox-*-linux-$ARCH.AppImage.zsync"'), + contains(r'|latest|VapourBox-*-$AI_ARCH.AppImage.zsync"'), reason: 'the update pattern must be the output filename with the ' 'version wildcarded, plus .zsync', ); }); + test('the AppImage is named App-version-arch, without "linux"', () { + // The AppImage catalog flags "linux" in an AppImage's file name, and + // wants the machine's own architecture name. The tarball is not an + // AppImage and keeps linux-. + expect(package, contains('AI_ARCH="x86_64"')); + expect(package, contains('AI_ARCH="aarch64"')); + expect( + package, contains(r'PACKAGE_NAME="VapourBox-$VERSION-linux-$ARCH"')); + expect(package, contains(r'TAR_FILE="$PACKAGE_NAME.tar.gz"')); + }); + + test('1.2.0 can still find its update under the name it embedded', () { + // 1.2.0 shipped as VapourBox-1.2.0-linux-.AppImage and looks for + // VapourBox-*-linux-.AppImage.zsync on the latest release. That + // file must keep being published, as a copy of the real .zsync, or + // every 1.2.0 AppImage silently stops updating. + expect( + package, + contains( + r'LEGACY_ZSYNC="VapourBox-$VERSION-linux-$ARCH.AppImage.zsync"')); + expect(package, contains(r'cp "$APPIMAGE_FILE.zsync" "$LEGACY_ZSYNC"')); + }); + test('the release upload carries the AppImage and its .zsync', () { final upload = _read(root, ['Scripts', 'ci-build-and-release.sh']); final finds = upload @@ -168,12 +190,56 @@ void main() { } final workflow = _read(root, ['.github', 'workflows', 'build-linux.yml']); - for (final arch in ['x64', 'arm64']) { - expect(workflow, contains('-linux-$arch.AppImage\n')); - expect(workflow, contains('-linux-$arch.AppImage.zsync\n')); + for (final arch in {'x64': 'x86_64', 'arm64': 'aarch64'}.entries) { + expect(workflow, contains('-${arch.value}.AppImage\n')); + expect(workflow, contains('-${arch.value}.AppImage.zsync\n')); + expect(workflow, contains('-linux-${arch.key}.AppImage.zsync\n'), + reason: 'the bridge .zsync for 1.2.0 must be uploaded too'); + expect(workflow, isNot(contains('-linux-${arch.key}.AppImage\n'))); } }); + test('the AppImage bundles the deps zips; the tarball does not', () { + final workflow = _read(root, ['.github', 'workflows', 'build-linux.yml']); + expect('--bundle-deps bundled-deps'.allMatches(workflow).length, 2, + reason: 'both architectures must bundle'); + // Only x64 is tiered, so only x64 carries a v2 delta. + expect('-v2-delta.zip.sha256.json'.allMatches(workflow).length, 1); + // The zips go into the AppDir after the tarball's tree was copied into + // it, never into the tree both are made from. + expect(package, contains(r'"$APPDIR/usr/lib/vapourbox/bundled-deps"')); + expect(package, isNot(contains(r'"$PACKAGE_DIR/bundled-deps'))); + // The directory name is the one DependencyManager looks in. + final manager = + _read(root, ['app', 'lib', 'services', 'dependency_manager.dart']); + expect(manager, contains("'bundled-deps'")); + }); + + test('ships AppStream metadata for the application id', () { + final cmake = _read(root, ['app', 'linux', 'CMakeLists.txt']); + final id = RegExp(r'set\(APPLICATION_ID "([^"]+)"\)') + .firstMatch(cmake)! + .group(1)!; + // .appdata.xml: the only name the AppImage catalog's lint looks for. + final appdata = _read(root, ['packaging', 'linux', '$id.appdata.xml']); + expect(appdata, contains('$id')); + expect(appdata, + contains('$id.desktop')); + expect(appdata, contains('')); + // Stamped at package time, so neither can go stale. + expect(appdata, contains('')); + expect(appdata, contains('/v@VERSION@/docs/images/screenshot.png')); + expect( + File(p.join(root, 'docs', 'images', 'screenshot.png')).existsSync(), + isTrue); + expect(package, + contains(r'"$APPDIR/usr/share/metainfo/$APP_ID.appdata.xml"')); + // appimagetool's own validation fetches the screenshot, whose tag does + // not exist until the release is published. + expect(package, contains('--no-appstream')); + expect(package, contains('appstreamcli validate --no-net')); + }); + test('the desktop entry and icon are named for the application id', () { final cmake = _read(root, ['app', 'linux', 'CMakeLists.txt']); final id = RegExp(r'set\(APPLICATION_ID "([^"]+)"\)') diff --git a/app/test/support/worker_harness.dart b/app/test/support/worker_harness.dart index 53aa932..efb6219 100644 --- a/app/test/support/worker_harness.dart +++ b/app/test/support/worker_harness.dart @@ -760,44 +760,58 @@ class WorkerHarness { return DepsVersionInfo.fromJson(json); } - /// Download + extract the pinned deps zip into [destDir]. + /// Download + extract the pinned deps into [destDir], composed exactly as + /// the app composes them: one bundle, or for a v2 tier of a delta-format + /// release the v3 bundle with the v2 delta extracted over it + /// (`DependencyManager.installAssetIds`). static Future _downloadDeps(String destDir) async { final info = _loadDepsVersion(); final tier = await depsTier(); final assetId = DependencyManager.assetIdFor(platform, tier); - final url = info.getDownloadUrl(assetId); - final expectedSha = await _fetchSidecarSha(info.getManifestUrl(assetId)); + final assetIds = DependencyManager.installAssetIds(platform, tier, + delta: info.tierIsDelta); final tmp = await Directory.systemTemp.createTemp('vb_deps_'); - final zip = File(p.join(tmp.path, info.filenameFor(assetId))); try { - // ignore: avoid_print - print('WorkerHarness: downloading $url'); - final bytes = await _httpGetBytes(url); - if (expectedSha != null) { - final actual = sha256.convert(bytes).toString(); - if (actual != expectedSha) { - throw StateError('deps sha256 mismatch (expected $expectedSha, got $actual)'); - } - } - await zip.writeAsBytes(bytes); - final dest = Directory(destDir); if (dest.existsSync()) dest.deleteSync(recursive: true); dest.createSync(recursive: true); - final archive = ZipDecoder().decodeBytes(bytes); - for (final f in archive) { - final outPath = p.join(destDir, f.name); - if (f.isFile) { - final outFile = File(outPath); - outFile.parent.createSync(recursive: true); - outFile.writeAsBytesSync(f.content as List); - if (!Platform.isWindows && _isExecutable(f.name)) { - await Process.run('chmod', ['+x', outPath]); - } + String? baseSha; + for (final id in assetIds) { + final url = info.getDownloadUrl(id); + final expectedSha = await _fetchSidecarSha(info.getManifestUrl(id)); + // ignore: avoid_print + print('WorkerHarness: downloading $url'); + final bytes = await _httpGetBytes(url); + final actual = sha256.convert(bytes).toString(); + if (expectedSha != null && actual != expectedSha) { + throw StateError( + 'deps sha256 mismatch for $id (expected $expectedSha, got $actual)'); + } + + final archive = ZipDecoder().decodeBytes(bytes); + if (id == assetIds.first) { + baseSha = actual; } else { - Directory(outPath).createSync(recursive: true); + final wanted = DependencyManager.deltaBaseSha256(archive); + if (wanted != baseSha) { + throw StateError('$id was cut against bundle $wanted, but the ' + 'bundle downloaded is $baseSha'); + } + } + for (final f in archive) { + final outPath = p.join(destDir, f.name); + if (f.isFile) { + final outFile = File(outPath); + outFile.parent.createSync(recursive: true); + outFile.writeAsBytesSync(f.content as List); + if (!Platform.isWindows && _isExecutable(f.name)) { + await Process.run('chmod', ['+x', outPath]); + } + } else { + Directory(outPath).createSync(recursive: true); + } } } diff --git a/docs/BUILDING.md b/docs/BUILDING.md index 22c4740..8e9ab12 100644 --- a/docs/BUILDING.md +++ b/docs/BUILDING.md @@ -277,17 +277,30 @@ already-assembled package directory. ### Linux ```bash -./Scripts/package-linux.sh --version 1.0.0 [--skip-build] [--arch x64|arm64] +./Scripts/package-linux.sh --version 1.0.0 [--skip-build] [--arch x64|arm64] \ + [--skip-appimage] [--bundle-deps DIR] ``` -`--arch` defaults to the host. Output, all from the same tree: +`--arch` defaults to the host. Output, all from the same tree (`` is +`x86_64` or `aarch64`): -- `dist/VapourBox-1.0.0-linux-.AppImage` — the primary download -- `dist/VapourBox-1.0.0-linux-.AppImage.zsync` — its delta-update index; must be uploaded beside it +- `dist/VapourBox-1.0.0-.AppImage` — the primary download +- `dist/VapourBox-1.0.0-.AppImage.zsync` — its delta-update index; must be uploaded beside it +- `dist/VapourBox-1.0.0-linux-.AppImage.zsync` — a copy of that index under the name 1.2.0 embedded, so a 1.2.0 AppImage can still update; upload it too - `dist/VapourBox-1.0.0-linux-.tar.gz` — the fallback The AppImage step needs `zsyncmake` (`sudo apt install zsync`) and a host of the target architecture; it downloads a pinned, checksummed `appimagetool` into `dist/.tools/` (or uses `$APPIMAGETOOL`). `--skip-appimage` builds the tarball alone. +With `desktop-file-validate` and `appstreamcli` installed (`sudo apt install +desktop-file-utils appstream`) it also validates the desktop entry and the +AppStream metadata (`packaging/linux/app.vapourbox.VapourBox.appdata.xml`). + +`--bundle-deps DIR` packs the dependency zips into the AppImage so its first +launch needs no network; release builds always pass it. `DIR` must hold the +release assets for the deps version `app/assets/deps-version.json` names, each +with its `.sha256.json`: `VapourBox-deps--linux-.zip`, and on x64 also +`VapourBox-deps--linux-x64-v2-delta.zip`. Without the flag the AppImage +downloads its dependencies on first launch, as the tarball always does. ### macOS ```bash diff --git a/docs/ENGINEERING_NOTES.md b/docs/ENGINEERING_NOTES.md index 6cec031..108f649 100644 --- a/docs/ENGINEERING_NOTES.md +++ b/docs/ENGINEERING_NOTES.md @@ -384,6 +384,46 @@ Note the version parser accepts `n9.0.1` and `9.0.1` and deliberately **rejects a `master` build** (`N-125978-...`), so reverting any platform to an unpinned master URL is a red build rather than a silent regression. +### The v2 tier became a delta, and the AppImage started carrying its deps (2026-10-09, deps 1.13.0) + +**What prompted it.** The AppImage catalog (appimage.github.io) picked up the +1.2.0 AppImage on its own and its test failed +([PR #9839](https://github.com/AppImage/appimage.github.io/pull/9839)). The +catalog launches each AppImage under `firejail --net=none`, screenshots it and +OCRs the screenshot for error text. VapourBox started fine, tried to download +its deps, failed DNS, and showed "Installation Failed" — which the OCR read. +Two warnings rode along: "linux" in the file name, and no AppStream metadata. + +**Why not just bundle the two existing zips.** The linux-x64 bundle is 236 MB +and so was its v2 twin; an AppImage that works on any x64 CPU would have +carried both, ~470 MB, for a difference that on Linux is one file. The tiers +were already required to list identical plugins +(`deps-expected-plugins.json`), and the tier blocks in `download-deps-*` only +touch zsmooth (and MVTools on macOS), so v2 as "v3 plus a few files" was +already true; only the packaging said otherwise. + +**The delta is declared, not diffed.** The first idea was to build both tiers +and publish whatever differs. That fails on contact: two builds of the same +sources on two runners are not byte-identical, so the diff is most of the +bundle. `_tierFiles` names the files instead, and `make-deps-delta.py` checks +the two things a diff would have told us that still matter — the builds hold +the same set of paths (so overwriting is enough), and each tier file really +does differ (so the tier switch did something). + +**What is unverified as of writing.** Everything Linux- and Windows-side was +written on macOS: the `delta-x64` jobs, `install-deps-bundle.sh`, the bundling +and validation in `package-linux.sh`, and the bridge `.zsync`. In particular: + +- that the composed macOS v2 tree runs — v2 MVTools is built against the v2 + build's own tree, and is now dropped onto the v3 bundle's `libfftw3f`; +- that `appstreamcli` on ubuntu-22.04 (0.15) accepts the appdata file; +- that AppImageUpdate, started from a 1.2.0 AppImage, follows the old-name + `.zsync` to the renamed file. zsync's `URL:` header is relative, so it should; + it has not been run. + +The first `build-deps-*` and `build-linux.yml` runs are the test of the first +two. The third needs a Linux machine and a 1.2.0 AppImage. + ### x86 bundles split into CPU tiers (issue #92, 2026-09-25) **The report.** A Mac Pro 5,1 (Xeon X5690, Westmere — SSE4.2, no AVX at all) diff --git a/packaging/linux/app.vapourbox.VapourBox.appdata.xml b/packaging/linux/app.vapourbox.VapourBox.appdata.xml new file mode 100644 index 0000000..3b1383f --- /dev/null +++ b/packaging/linux/app.vapourbox.VapourBox.appdata.xml @@ -0,0 +1,84 @@ + + + + app.vapourbox.VapourBox + app.vapourbox.VapourBox.desktop + + VapourBox + Deinterlace, restore and convert video with VapourSynth and QTGMC + Stuart Cameron + + CC0-1.0 + GPL-3.0-or-later + + +

+ VapourBox is a desktop app for archiving and converting video: tape + captures, camcorder footage, film scans, DVDs, and the digital files you + already have. It runs VapourSynth, QTGMC and FFmpeg, the tools the video + restoration community already uses, without requiring you to write a + script. +

+
    +
  • Converts DV, AVCHD, MPEG-2, MXF and ordinary MP4, MKV, AVI and MOV files, DVD folders and discs
    • +
  • Deinterlaces with QTGMC and reverses telecine, after detecting how the source is stored
    • +
  • Optional cleanup filters for tape noise, colour bleed, film dust and scratches, ghosting and more
    • +
  • Presets named after the source: VHS, DV camcorder tape, DVD and broadcast, anime, 8mm and Super 8 film
    • +
  • A side-by-side preview of the source and the processed result, and a queue for batches
    • +
  • Encodes to H.264, HEVC, ProRes and lossless formats, with hardware encoders where available
    • +
  • Transcribes speech to subtitles
    • +
+

+ The AppImage includes the processing components, so it works without a + network connection. It needs glibc 2.39 or newer (Ubuntu 24.04, + Debian 13). +

+ + + + AudioVideo + Video + + + deinterlace + QTGMC + VapourSynth + VHS + DV + DVD + convert + + + https://github.com/StuartCameronCode/VapourBox + https://github.com/StuartCameronCode/VapourBox/issues + + + + An interlaced capture with the original and the deinterlaced result side by side + https://raw.githubusercontent.com/StuartCameronCode/VapourBox/v@VERSION@/docs/images/screenshot.png + + + + + + + + https://github.com/StuartCameronCode/VapourBox/releases/tag/v@VERSION@ + + + From a91773144893d18eb08ae7025e8db64f9bb8e6a1 Mon Sep 17 00:00:00 2001 From: Stuart Cameron Date: Mon, 12 Oct 2026 01:04:38 +1100 Subject: [PATCH 2/2] perf(deps): extract a bundle without a chmod process per file First launch of the bundled-deps AppImage took ~130 s on linux-arm64 under Xvfb, nearly all of it parked at 28% of "Extracting...". Two causes, both in extractBundle: - one `chmod +x` process per file whose name has no dot. The bundled Python's terminfo matches that test: 2,847 spawns for the 1.13.0 linux-arm64 bundle, 2,798 of them terminfo entries. Now collected and run in batches of 500. - three awaited async calls per file on the UI isolate, ~14 ms a file under software rendering. Writes are now synchronous, yielding to the UI at each 2 MB progress tick. The same launch now reaches the main window in ~16-22 s. The set of files made executable is unchanged. Also ignore bundled-deps/, where the --bundle-deps zips are downloaded. Co-Authored-By: Claude Opus 5.5 --- .gitignore | 2 ++ app/lib/services/dependency_manager.dart | 32 +++++++++++++++++++----- app/test/deps_delta_install_test.dart | 26 +++++++++++++++++++ 3 files changed, 54 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index fc912d0..c37ace7 100644 --- a/.gitignore +++ b/.gitignore @@ -70,6 +70,8 @@ deps/ # Build Outputs # ============================================================================= dist/ +# Deps zips fetched for package-linux.sh --bundle-deps +bundled-deps/ *.app *.dmg *.exe diff --git a/app/lib/services/dependency_manager.dart b/app/lib/services/dependency_manager.dart index 411ec7b..0576848 100644 --- a/app/lib/services/dependency_manager.dart +++ b/app/lib/services/dependency_manager.dart @@ -1338,30 +1338,50 @@ class DependencyManager { emitExtractProgress(); + // Marked executable in batches after the loop. One chmod process per file + // was most of a first launch on Linux: the name test below matches every + // terminfo entry in the bundled Python (~2,800 files). + final executables = []; + + // Written synchronously, yielding to the UI at each progress tick. The + // bundle is thousands of tiny files, and awaiting each create and write + // costs an event-loop turn apiece: under software rendering that was + // ~14 ms a file, 40 s of a first launch spent on 2 MB of terminfo. + final createdDirs = {}; + void ensureDir(String dir) { + if (createdDirs.add(dir)) Directory(dir).createSync(recursive: true); + } + for (final file in archive) { if (_isVersionFile(file.name)) continue; final filePath = path.join(depsDir.path, file.name); if (file.isFile) { - final outFile = File(filePath); - await outFile.parent.create(recursive: true); - await outFile.writeAsBytes(file.content as List); + ensureDir(path.dirname(filePath)); + File(filePath).writeAsBytesSync(file.content as List); - // Set executable permission on Unix if (!Platform.isWindows && _isExecutable(file.name)) { - await Process.run('chmod', ['+x', filePath]); + executables.add(filePath); } extracted += file.size; if (extracted - lastEmitted >= emitEvery) { lastEmitted = extracted; emitExtractProgress(); + await Future.delayed(Duration.zero); } } else { - await Directory(filePath).create(recursive: true); + ensureDir(filePath); } } + // Set executable permission on Unix. Chunked to stay far below ARG_MAX. + const chmodBatch = 500; + for (var i = 0; i < executables.length; i += chmodBatch) { + final end = (i + chmodBatch).clamp(0, executables.length); + await Process.run('chmod', ['+x', ...executables.sublist(i, end)]); + } + // Final 100% extraction tick. extracted = totalBytes; emitExtractProgress(); diff --git a/app/test/deps_delta_install_test.dart b/app/test/deps_delta_install_test.dart index e08712c..9c61549 100644 --- a/app/test/deps_delta_install_test.dart +++ b/app/test/deps_delta_install_test.dart @@ -200,6 +200,32 @@ void main() { await manager.extractBundle(base, deps, finalize: false); expect(File(p.join(deps.path, 'stale.so')).existsSync(), isFalse); }); + + test('executables are marked executable, past one chmod batch', () async { + // chmod runs in batches of 500 after the files are written; 1,200 + // extensionless names (as the bundled Python's terminfo has) cross two + // batch boundaries. + final deps = Directory(p.join(tmp.path, 'deps')); + final base = await zipFile('base.zip', { + 'ffmpeg/ffmpeg': 'ffmpeg', + 'vapoursynth/vspipe': 'vspipe', + 'vapoursynth/plugins/libmvtools.so': 'mvtools', + for (var i = 0; i < 1200; i++) 'python/share/terminfo/x/entry$i': 't', + }); + await manager.extractBundle(base, deps, finalize: false); + + bool executable(String rel) => + File(p.joinAll([deps.path, ...rel.split('/')])).statSync().mode & + 0x49 != + 0; + expect(executable('ffmpeg/ffmpeg'), isTrue); + expect(executable('vapoursynth/vspipe'), isTrue); + expect(executable('python/share/terminfo/x/entry0'), isTrue); + expect(executable('python/share/terminfo/x/entry499'), isTrue); + expect(executable('python/share/terminfo/x/entry500'), isTrue); + expect(executable('python/share/terminfo/x/entry1199'), isTrue); + expect(executable('vapoursynth/plugins/libmvtools.so'), isFalse); + }, skip: Platform.isWindows ? 'no executable bit on Windows' : null); }); group('the shipped pointer and manifest', () {