diff --git a/.github/scripts/fetch-deps-bundle.sh b/.github/scripts/fetch-deps-bundle.sh index 37a3678..7ba371a 100755 --- a/.github/scripts/fetch-deps-bundle.sh +++ b/.github/scripts/fetch-deps-bundle.sh @@ -1,11 +1,16 @@ #!/usr/bin/env bash # -# Fetch one platform's dependency bundle for a CI run and print the path of the +# Fetch ONE dependency release asset for a CI run and print the path of the # downloaded .zip on stdout. Everything else goes to stderr, so the caller can # do: # # ZIP=$(bash .github/scripts/fetch-deps-bundle.sh macos-arm64 deps-v1.9.0 1.9.0) # +# The first argument is an asset id as it appears in the file name: a platform +# (`linux-x64`), or a v2 delta (`linux-x64-v2-delta`). Workflows should not +# call this directly — install-deps-bundle.sh knows which assets make up a +# platform's bundle (a v2 tier is two of them) and extracts them in order. +# # Two sources, chosen by whether $DEPS_RUN_ID is set: # # (default) the published release named by app/assets/deps-version.json. This @@ -45,7 +50,10 @@ TAG="${2:?missing release tag}" VER="${3:?missing version}" if [ -n "${DEPS_RUN_ID:-}" ]; then - rm -rf .deps-artifact + # One directory per asset: installing a v2 tier fetches two, and the second + # must not wipe the first. + ARTIFACT_DIR=".deps-artifact/$PLATFORM" + rm -rf "$ARTIFACT_DIR" ZIP="" FROM_RUN="" @@ -55,13 +63,13 @@ if [ -n "${DEPS_RUN_ID:-}" ]; then for RUN in $(echo "$DEPS_RUN_ID" | tr ',' ' '); do echo "looking for a $PLATFORM artifact in run $RUN" >&2 if gh run download "$RUN" \ - --pattern "VapourBox-deps-*-${PLATFORM}" --dir .deps-artifact >&2 2>/dev/null; then + --pattern "VapourBox-deps-*-${PLATFORM}" --dir "$ARTIFACT_DIR" >&2 2>/dev/null; then # gh nests each artifact in a directory named after it, so glob rather # than assuming a layout. - ZIP=$(find .deps-artifact -name "VapourBox-deps-*-${PLATFORM}.zip" | head -1) + ZIP=$(find "$ARTIFACT_DIR" -name "VapourBox-deps-*-${PLATFORM}.zip" | head -1) if [ -n "$ZIP" ]; then FROM_RUN="$RUN"; break; fi fi - rm -rf .deps-artifact + rm -rf "$ARTIFACT_DIR" done if [ -z "$ZIP" ]; then diff --git a/.github/scripts/install-deps-bundle.sh b/.github/scripts/install-deps-bundle.sh new file mode 100755 index 0000000..1f01d9c --- /dev/null +++ b/.github/scripts/install-deps-bundle.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# +# Fetch a platform's dependency bundle and extract it into a directory: +# +# bash .github/scripts/install-deps-bundle.sh linux-x64-v2 deps-v1.13.0 1.13.0 deps/linux-x64 +# +# The first argument names the bundle the way the test matrix does: a platform +# (`linux-x64`, `macos-arm64`), optionally suffixed `-v2` for the older-CPU tier +# of an x64 platform (issue #92). +# +# A v2 tier is not one download. From deps 1.13.0 (`"tierFormat": "delta"` in +# app/assets/deps-version.json) it is the platform's full bundle plus a small +# delta extracted over it — exactly what DependencyManager does on a user's +# machine, so CI installs it the same way rather than from some full v2 zip no +# user ever receives. This script is the one place in CI that knows that. +# +# Sources and $DEPS_RUN_ID are as documented in fetch-deps-bundle.sh. +set -euo pipefail + +BUNDLE="${1:?usage: install-deps-bundle.sh [-v2] }" +TAG="${2:?missing release tag}" +VER="${3:?missing version}" +DEST="${4:?missing destination directory}" +HERE="$(cd "$(dirname "$0")" && pwd)" +POINTER="$HERE/../../app/assets/deps-version.json" + +ASSETS=("$BUNDLE") +if [[ "$BUNDLE" == *-v2 ]] && grep -q '"tierFormat": *"delta"' "$POINTER"; then + ASSETS=("${BUNDLE%-v2}" "$BUNDLE-delta") +fi + +mkdir -p "$DEST" +for ASSET in "${ASSETS[@]}"; do + ZIP=$(bash "$HERE/fetch-deps-bundle.sh" "$ASSET" "$TAG" "$VER") + [ -f "$ZIP" ] || { echo "::error::no zip for $ASSET" >&2; exit 1; } + echo "extracting $ZIP into $DEST" >&2 + if [ "${RUNNER_OS:-}" = Windows ]; then + # The Windows bundle uses backslash separators; 7-Zip copes, info-zip + # `unzip` warns and exits non-zero. + 7z x "$ZIP" -o"$DEST" -y >/dev/null + else + unzip -q -o "$ZIP" -d "$DEST" + fi + rm -f "$ZIP" +done +rm -rf .deps-artifact + +# The delta carries the version.json that marks the tree as the v2 tier; if it +# did not land last, the tests would silently run the v3 bundle. +if [[ "$BUNDLE" == *-v2 ]] && ! grep -q '"tier": *"v2"' "$DEST/version.json"; then + echo "::error::$DEST/version.json does not say tier v2 after installing $BUNDLE" >&2 + exit 1 +fi diff --git a/.github/workflows/build-deps-linux.yml b/.github/workflows/build-deps-linux.yml index 7047b60..706d332 100644 --- a/.github/workflows/build-deps-linux.yml +++ b/.github/workflows/build-deps-linux.yml @@ -27,7 +27,9 @@ permissions: jobs: # x64 ships in two CPU tiers (issue #92): v3 for x86-64-v3 CPUs, v2 for - # anything older. Same script, one --tier switch. + # anything older. Same script, one --tier switch. Both are built in full, but + # only v3 is published as a bundle: delta-x64 below reduces the v2 build to + # the few files that differ (Scripts/make-deps-delta.py). build-x64: if: ${{ inputs.arch == 'x64' || inputs.arch == 'both' }} runs-on: ubuntu-24.04 @@ -62,18 +64,59 @@ jobs: - name: Package dependencies run: ./Scripts/package-deps-linux.sh --version "${{ inputs.version }}" --arch x64 + # The v2 build is an intermediate ("-v2-build"): nothing downstream may + # mistake it for a publishable bundle. - uses: actions/upload-artifact@v5 with: - name: ${{ env.ASSET }} + name: ${{ env.ASSET }}${{ matrix.tier == 'v2' && '-build' || '' }} + retention-days: ${{ matrix.tier == 'v2' && 3 || 90 }} path: | dist/${{ env.ASSET }}.zip dist/${{ env.ASSET }}.zip.sha256.json - # The v2 bundle's promise is that it runs on CPUs without AVX. Every hosted + # Cut the v2 delta against the v3 bundle, and compose the tree a v2 user + # ends up with (v3 bundle + delta) for the gate to test. + delta-x64: + needs: build-x64 + runs-on: ubuntu-24.04 + env: + STEM: VapourBox-deps-${{ inputs.version }}-linux-x64 + steps: + - uses: actions/checkout@v5 + - uses: actions/download-artifact@v5 + with: + name: ${{ env.STEM }} + path: base + - uses: actions/download-artifact@v5 + with: + name: ${{ env.STEM }}-v2-build + path: v2-build + - name: Make the v2 delta + run: | + python3 Scripts/make-deps-delta.py --platform linux-x64 \ + --version "${{ inputs.version }}" \ + --base "base/$STEM.zip" --v2 "v2-build/$STEM-v2.zip" \ + --out dist --composed composed + - uses: actions/upload-artifact@v5 + with: + name: ${{ env.STEM }}-v2-delta + path: | + dist/${{ env.STEM }}-v2-delta.zip + dist/${{ env.STEM }}-v2-delta.zip.sha256.json + # What the gate probes. Named as the gate expects a v2 bundle to be; never + # uploaded to a release. + - uses: actions/upload-artifact@v5 + with: + name: ${{ env.STEM }}-v2 + retention-days: 3 + path: composed/${{ env.STEM }}-v2.zip + + # The v2 tier's promise is that it runs on CPUs without AVX. Every hosted # runner has AVX2, so prove it by running every plugin under Intel SDE - # emulating Westmere (the Mac Pro 5,1 from issue #92). + # emulating Westmere (the Mac Pro 5,1 from issue #92) — against the v3 bundle + # with the delta applied, which is what such a machine actually installs. gate-x64-v2: - needs: build-x64 + needs: delta-x64 uses: ./.github/workflows/probe-cpu-compat.yml with: artifact_prefix: VapourBox-deps-${{ inputs.version }}- @@ -81,26 +124,32 @@ jobs: gate: true matrix_json: '[{"platform":"linux-x64","runner":"ubuntu-24.04","sde":"sde64","python":"python/bin/python3","chip":"wsm"}]' - # Published only once the v2 gate has passed, so a bundle that would crash on - # an older CPU never reaches a release. + # Published only once the v2 gate has passed, so a delta that would crash on + # an older CPU never reaches a release. Exactly four files: the v3 bundle, + # the v2 delta, and a sidecar each. upload-x64: - needs: [build-x64, gate-x64-v2] + needs: [build-x64, delta-x64, gate-x64-v2] if: inputs.release_tag != '' runs-on: ubuntu-24.04 + env: + STEM: VapourBox-deps-${{ inputs.version }}-linux-x64 steps: - uses: actions/download-artifact@v5 with: - pattern: VapourBox-deps-${{ inputs.version }}-linux-x64* + name: ${{ env.STEM }} + path: dist + - uses: actions/download-artifact@v5 + with: + name: ${{ env.STEM }}-v2-delta path: dist - merge-multiple: true - name: Upload to release env: GH_TOKEN: ${{ github.token }} run: | gh release upload "${{ inputs.release_tag }}" \ --repo "${{ github.repository }}" \ - dist/VapourBox-deps-${{ inputs.version }}-linux-x64*.zip \ - dist/VapourBox-deps-${{ inputs.version }}-linux-x64*.zip.sha256.json --clobber + "dist/$STEM.zip" "dist/$STEM.zip.sha256.json" \ + "dist/$STEM-v2-delta.zip" "dist/$STEM-v2-delta.zip.sha256.json" --clobber build-arm64: if: ${{ inputs.arch == 'arm64' || inputs.arch == 'both' }} diff --git a/.github/workflows/build-deps-macos.yml b/.github/workflows/build-deps-macos.yml index 5b900ea..ddd101f 100644 --- a/.github/workflows/build-deps-macos.yml +++ b/.github/workflows/build-deps-macos.yml @@ -61,10 +61,16 @@ jobs: "dist/VapourBox-deps-${{ inputs.version }}-macos-arm64.zip.sha256.json" --clobber # x64 ships in two CPU tiers (issue #92): v3 for x86-64-v3 CPUs, v2 for - # anything older. Same script, one --tier switch. The v2 bundle is gated by - # package-deps-macos.sh itself (Scripts/check-load-time-simd.py): macOS has - # no Intel SDE, so it proves statically that no plugin runs AVX while it - # loads — and fails before anything below can upload it. + # anything older. Same script, one --tier switch. Both are built in full, but + # only v3 is published as a bundle: delta-x64 below reduces the v2 build to + # the few files that differ (Scripts/make-deps-delta.py). + # + # The v2 build is gated by package-deps-macos.sh itself + # (Scripts/check-load-time-simd.py): macOS has no Intel SDE, so it proves + # statically that no plugin runs AVX while it loads — and fails this job, so + # nothing below can publish a delta cut from it. The check covers every + # plugin the v2 build holds; the ones outside the delta are the same sources + # and binaries the v3 bundle was built from. build-x64: if: ${{ inputs.arch == 'x64' || inputs.arch == 'both' }} runs-on: macos-15-intel # only hosted Intel image (macos-13 retired; last one until ~Aug 2027) @@ -87,18 +93,60 @@ jobs: - name: Package dependencies run: ./Scripts/package-deps-macos.sh --version "${{ inputs.version }}" --arch x64 + # The v2 build is an intermediate ("-v2-build"): nothing downstream may + # mistake it for a publishable bundle. - uses: actions/upload-artifact@v5 with: - name: ${{ env.ASSET }} + name: ${{ env.ASSET }}${{ matrix.tier == 'v2' && '-build' || '' }} + retention-days: ${{ matrix.tier == 'v2' && 3 || 90 }} path: | dist/${{ env.ASSET }}.zip dist/${{ env.ASSET }}.zip.sha256.json + # Cut the v2 delta against the v3 bundle, then publish both. Runs on macOS so + # the composed tree can be put through the same load-time check the v2 build + # passed, this time on exactly what a v2 user ends up with. + delta-x64: + needs: build-x64 + runs-on: macos-15 + env: + STEM: VapourBox-deps-${{ inputs.version }}-macos-x64 + steps: + - uses: actions/checkout@v5 + - uses: actions/download-artifact@v5 + with: + name: ${{ env.STEM }} + path: dist + - uses: actions/download-artifact@v5 + with: + name: ${{ env.STEM }}-v2-build + path: v2-build + - name: Make the v2 delta + run: | + python3 Scripts/make-deps-delta.py --platform macos-x64 \ + --version "${{ inputs.version }}" \ + --base "dist/$STEM.zip" --v2 "v2-build/$STEM-v2.zip" \ + --out dist --composed composed + + - name: Gate the composed v2 tree (no AVX at load time) + run: | + mkdir composed-tree + unzip -q "composed/$STEM-v2.zip" -d composed-tree + python3 Scripts/check-load-time-simd.py composed-tree/vapoursynth/plugins/*.dylib + + - uses: actions/upload-artifact@v5 + with: + name: ${{ env.STEM }}-v2-delta + path: | + dist/${{ env.STEM }}-v2-delta.zip + dist/${{ env.STEM }}-v2-delta.zip.sha256.json + - name: Upload to release if: inputs.release_tag != '' env: GH_TOKEN: ${{ github.token }} run: | gh release upload "${{ inputs.release_tag }}" \ - "dist/${{ env.ASSET }}.zip" \ - "dist/${{ env.ASSET }}.zip.sha256.json" --clobber + --repo "${{ github.repository }}" \ + "dist/$STEM.zip" "dist/$STEM.zip.sha256.json" \ + "dist/$STEM-v2-delta.zip" "dist/$STEM-v2-delta.zip.sha256.json" --clobber diff --git a/.github/workflows/build-deps-windows.yml b/.github/workflows/build-deps-windows.yml index 95f4bf4..14e3b1e 100644 --- a/.github/workflows/build-deps-windows.yml +++ b/.github/workflows/build-deps-windows.yml @@ -24,7 +24,9 @@ permissions: jobs: # x64 ships in two CPU tiers (issue #92): v3 for x86-64-v3 CPUs, v2 for - # anything older. Same script, one -Tier switch. + # anything older. Same script, one -Tier switch. Both are built in full, but + # only v3 is published as a bundle: delta-x64 below reduces the v2 build to + # the few files that differ (Scripts/make-deps-delta.py). build-x64: runs-on: windows-latest strategy: @@ -44,19 +46,61 @@ jobs: shell: pwsh run: ./Scripts/package-deps-windows.ps1 -Version "${{ inputs.version }}" + # The v2 build is an intermediate ("-v2-build"): nothing downstream may + # mistake it for a publishable bundle. - uses: actions/upload-artifact@v5 with: - name: ${{ env.ASSET }} + name: ${{ env.ASSET }}${{ matrix.tier == 'v2' && '-build' || '' }} + retention-days: ${{ matrix.tier == 'v2' && 3 || 90 }} path: | dist/${{ env.ASSET }}.zip dist/${{ env.ASSET }}.zip.sha256.json - # The v2 bundle must run below x86-64-v3, and every hosted runner has AVX2. + # Cut the v2 delta against the v3 bundle, and compose the tree a v2 user + # ends up with (v3 bundle + delta) for the gate to test. Pure zip work, so it + # runs on Linux like the other platforms' delta jobs. + delta-x64: + needs: build-x64 + runs-on: ubuntu-24.04 + env: + STEM: VapourBox-deps-${{ inputs.version }}-windows-x64 + steps: + - uses: actions/checkout@v5 + - uses: actions/download-artifact@v5 + with: + name: ${{ env.STEM }} + path: base + - uses: actions/download-artifact@v5 + with: + name: ${{ env.STEM }}-v2-build + path: v2-build + - name: Make the v2 delta + run: | + python3 Scripts/make-deps-delta.py --platform windows-x64 \ + --version "${{ inputs.version }}" \ + --base "base/$STEM.zip" --v2 "v2-build/$STEM-v2.zip" \ + --out dist --composed composed + - uses: actions/upload-artifact@v5 + with: + name: ${{ env.STEM }}-v2-delta + path: | + dist/${{ env.STEM }}-v2-delta.zip + dist/${{ env.STEM }}-v2-delta.zip.sha256.json + # What the gate probes. Named as the gate expects a v2 bundle to be; never + # uploaded to a release. + - uses: actions/upload-artifact@v5 + with: + name: ${{ env.STEM }}-v2 + retention-days: 3 + path: composed/${{ env.STEM }}-v2.zip + + # The v2 tier must run below x86-64-v3, and every hosted runner has AVX2. # Windows is probed at Sandy Bridge (AVX without AVX2): at pre-AVX chips SDE # cannot emulate what Microsoft's runtime reads from the kernel, so the core - # control fails and nothing could be proven (see probe-cpu-compat.yml). + # control fails and nothing could be proven (see probe-cpu-compat.yml). The + # gate runs against the v3 bundle with the delta applied. gate-x64-v2: - needs: build-x64 + needs: delta-x64 uses: ./.github/workflows/probe-cpu-compat.yml with: artifact_prefix: VapourBox-deps-${{ inputs.version }}- @@ -64,22 +108,28 @@ jobs: gate: true matrix_json: '[{"platform":"windows-x64","runner":"windows-latest","sde":"sde.exe","python":"vapoursynth/python.exe","chip":"snb"}]' - # Published only once the v2 gate has passed. + # Published only once the v2 gate has passed. Exactly four files: the v3 + # bundle, the v2 delta, and a sidecar each. upload-x64: - needs: [build-x64, gate-x64-v2] + needs: [build-x64, delta-x64, gate-x64-v2] if: inputs.release_tag != '' runs-on: ubuntu-24.04 + env: + STEM: VapourBox-deps-${{ inputs.version }}-windows-x64 steps: - uses: actions/download-artifact@v5 with: - pattern: VapourBox-deps-${{ inputs.version }}-windows-x64* + name: ${{ env.STEM }} + path: dist + - uses: actions/download-artifact@v5 + with: + name: ${{ env.STEM }}-v2-delta path: dist - merge-multiple: true - name: Upload to release env: GH_TOKEN: ${{ github.token }} run: | gh release upload "${{ inputs.release_tag }}" \ --repo "${{ github.repository }}" \ - dist/VapourBox-deps-${{ inputs.version }}-windows-x64*.zip \ - dist/VapourBox-deps-${{ inputs.version }}-windows-x64*.zip.sha256.json --clobber + "dist/$STEM.zip" "dist/$STEM.zip.sha256.json" \ + "dist/$STEM-v2-delta.zip" "dist/$STEM-v2-delta.zip.sha256.json" --clobber diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index 74925cd..356098f 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -37,7 +37,7 @@ jobs: libgtk-3-dev liblzma-dev libstdc++-12-dev # zsyncmake writes the .zsync the AppImage's update information names; # desktop-file-validate lets appimagetool check the desktop entry. - sudo apt-get install -y zsync desktop-file-utils + sudo apt-get install -y zsync desktop-file-utils appstream - name: Setup Flutter uses: subosito/flutter-action@v2 @@ -63,21 +63,27 @@ jobs: restore-keys: | ${{ runner.os }}-x64-cargo- - - name: Download Linux dependencies + # The AppImage carries the deps zips so its first launch needs no + # network. They go in as published, sidecars included; package-linux.sh + # checks each against its sidecar and that they are the version + # app/assets/deps-version.json names. + - name: Download the deps to bundle run: | DEPS_VERSION="${{ inputs.deps_tag }}" DEPS_VERSION="${DEPS_VERSION#deps-v}" + BASE="https://github.com/${{ github.repository }}/releases/download/${{ inputs.deps_tag }}" + STEM="VapourBox-deps-${DEPS_VERSION}-linux-x64" - mkdir -p deps/linux-x64 + mkdir -p bundled-deps + for F in "$STEM.zip" "$STEM.zip.sha256.json"; do + curl -fL --retry 3 -o "bundled-deps/$F" "$BASE/$F" + done - DEPS_URL="https://github.com/${{ github.repository }}/releases/download/${{ inputs.deps_tag }}/VapourBox-deps-${DEPS_VERSION}-linux-x64.zip" - echo "Downloading deps from: $DEPS_URL" - curl -L -o deps.zip "$DEPS_URL" - unzip -o deps.zip -d deps/linux-x64 - rm -f deps.zip - - echo "Dependencies extracted:" - ls -la deps/ + # x64 only: the few files older CPUs need instead (see "CPU tiers"). + for F in "$STEM-v2-delta.zip" "$STEM-v2-delta.zip.sha256.json"; do + curl -fL --retry 3 -o "bundled-deps/$F" "$BASE/$F" + done + ls -la bundled-deps - name: Update version numbers run: | @@ -110,7 +116,8 @@ jobs: - name: Package release run: | - ./Scripts/package-linux.sh --version "${{ inputs.version }}" --arch x64 --skip-build + ./Scripts/package-linux.sh --version "${{ inputs.version }}" --arch x64 --skip-build \ + --bundle-deps bundled-deps - name: Upload artifacts uses: actions/upload-artifact@v5 @@ -118,7 +125,8 @@ jobs: name: VapourBox-${{ inputs.version }}-linux-x64 if-no-files-found: error path: | - dist/VapourBox-${{ inputs.version }}-linux-x64.AppImage + dist/VapourBox-${{ inputs.version }}-x86_64.AppImage + dist/VapourBox-${{ inputs.version }}-x86_64.AppImage.zsync dist/VapourBox-${{ inputs.version }}-linux-x64.AppImage.zsync dist/VapourBox-${{ inputs.version }}-linux-x64.tar.gz @@ -137,7 +145,7 @@ jobs: libgtk-3-dev liblzma-dev libstdc++-12-dev # zsyncmake writes the .zsync the AppImage's update information names; # desktop-file-validate lets appimagetool check the desktop entry. - sudo apt-get install -y zsync desktop-file-utils + sudo apt-get install -y zsync desktop-file-utils appstream - name: Setup Flutter run: | @@ -168,21 +176,22 @@ jobs: restore-keys: | ${{ runner.os }}-arm64-cargo- - - name: Download Linux dependencies + # The AppImage carries the deps zips so its first launch needs no + # network. They go in as published, sidecars included; package-linux.sh + # checks each against its sidecar and that they are the version + # app/assets/deps-version.json names. + - name: Download the deps to bundle run: | DEPS_VERSION="${{ inputs.deps_tag }}" DEPS_VERSION="${DEPS_VERSION#deps-v}" + BASE="https://github.com/${{ github.repository }}/releases/download/${{ inputs.deps_tag }}" + STEM="VapourBox-deps-${DEPS_VERSION}-linux-arm64" - mkdir -p deps/linux-arm64 - - DEPS_URL="https://github.com/${{ github.repository }}/releases/download/${{ inputs.deps_tag }}/VapourBox-deps-${DEPS_VERSION}-linux-arm64.zip" - echo "Downloading deps from: $DEPS_URL" - curl -L -o deps.zip "$DEPS_URL" - unzip -o deps.zip -d deps/linux-arm64 - rm -f deps.zip - - echo "Dependencies extracted:" - ls -la deps/ + mkdir -p bundled-deps + for F in "$STEM.zip" "$STEM.zip.sha256.json"; do + curl -fL --retry 3 -o "bundled-deps/$F" "$BASE/$F" + done + ls -la bundled-deps - name: Update version numbers run: | @@ -215,7 +224,8 @@ jobs: - name: Package release run: | - ./Scripts/package-linux.sh --version "${{ inputs.version }}" --arch arm64 --skip-build + ./Scripts/package-linux.sh --version "${{ inputs.version }}" --arch arm64 --skip-build \ + --bundle-deps bundled-deps - name: Upload artifacts uses: actions/upload-artifact@v5 @@ -223,6 +233,7 @@ jobs: name: VapourBox-${{ inputs.version }}-linux-arm64 if-no-files-found: error path: | - dist/VapourBox-${{ inputs.version }}-linux-arm64.AppImage + dist/VapourBox-${{ inputs.version }}-aarch64.AppImage + dist/VapourBox-${{ inputs.version }}-aarch64.AppImage.zsync dist/VapourBox-${{ inputs.version }}-linux-arm64.AppImage.zsync dist/VapourBox-${{ inputs.version }}-linux-arm64.tar.gz diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index b7647d4..8542a5e 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -75,11 +75,8 @@ jobs: DEPS_RUN_ID: ${{ inputs.deps_run_id }} run: | PLATFORM="macos-${{ matrix.arch }}" - mkdir -p "deps/$PLATFORM" - ZIP=$(bash .github/scripts/fetch-deps-bundle.sh "$PLATFORM" \ - "${{ steps.deps.outputs.tag }}" "${{ steps.deps.outputs.ver }}") - unzip -q -o "$ZIP" -d "deps/$PLATFORM" - rm -rf "$ZIP" .deps-artifact + bash .github/scripts/install-deps-bundle.sh "$PLATFORM" \ + "${{ steps.deps.outputs.tag }}" "${{ steps.deps.outputs.ver }}" "deps/$PLATFORM" xattr -cr "deps/$PLATFORM" 2>/dev/null || true # The whisper small model (~466 MB) is architecture-independent; cache it. @@ -149,13 +146,8 @@ jobs: GH_TOKEN: ${{ github.token }} DEPS_RUN_ID: ${{ inputs.deps_run_id }} run: | - mkdir -p deps/windows-x64 - ZIP=$(bash .github/scripts/fetch-deps-bundle.sh windows-x64 \ - "${{ steps.deps.outputs.tag }}" "${{ steps.deps.outputs.ver }}") - # The Windows deps zip uses backslash separators; 7-Zip handles that - # (info-zip `unzip` warns and exits non-zero, tripping `set -e`). - 7z x "$ZIP" -o"deps/windows-x64" -y >/dev/null - rm -rf "$ZIP" .deps-artifact + bash .github/scripts/install-deps-bundle.sh windows-x64 \ + "${{ steps.deps.outputs.tag }}" "${{ steps.deps.outputs.ver }}" deps/windows-x64 - name: Cache whisper model uses: actions/cache@v5 @@ -231,11 +223,8 @@ jobs: GH_TOKEN: ${{ github.token }} DEPS_RUN_ID: ${{ inputs.deps_run_id }} run: | - mkdir -p deps/linux-x64 - ZIP=$(bash .github/scripts/fetch-deps-bundle.sh linux-x64 \ - "${{ steps.deps.outputs.tag }}" "${{ steps.deps.outputs.ver }}") - unzip -q -o "$ZIP" -d deps/linux-x64 - rm -rf "$ZIP" .deps-artifact + bash .github/scripts/install-deps-bundle.sh linux-x64 \ + "${{ steps.deps.outputs.tag }}" "${{ steps.deps.outputs.ver }}" deps/linux-x64 - name: Cache whisper model uses: actions/cache@v5 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 7400e4b..56f40ab 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -87,11 +87,8 @@ jobs: run: | # The tier is in the asset name only; either installs to deps/. PLATFORM="macos-${{ matrix.arch }}" - mkdir -p "deps/$PLATFORM" - ZIP=$(bash .github/scripts/fetch-deps-bundle.sh "${{ matrix.asset }}" \ - "${{ steps.deps.outputs.tag }}" "${{ steps.deps.outputs.ver }}") - unzip -q -o "$ZIP" -d "deps/$PLATFORM" - rm -rf "$ZIP" .deps-artifact + bash .github/scripts/install-deps-bundle.sh "${{ matrix.asset }}" \ + "${{ steps.deps.outputs.tag }}" "${{ steps.deps.outputs.ver }}" "deps/$PLATFORM" xattr -cr "deps/$PLATFORM" 2>/dev/null || true - name: Cache whisper model @@ -168,11 +165,8 @@ jobs: GH_TOKEN: ${{ github.token }} DEPS_RUN_ID: ${{ inputs.deps_run_id }} run: | - mkdir -p deps/windows-x64 - ZIP=$(bash .github/scripts/fetch-deps-bundle.sh "${{ matrix.asset }}" \ - "${{ steps.deps.outputs.tag }}" "${{ steps.deps.outputs.ver }}") - 7z x "$ZIP" -o"deps/windows-x64" -y >/dev/null - rm -rf "$ZIP" .deps-artifact + bash .github/scripts/install-deps-bundle.sh "${{ matrix.asset }}" \ + "${{ steps.deps.outputs.tag }}" "${{ steps.deps.outputs.ver }}" deps/windows-x64 - name: Cache whisper model uses: actions/cache@v5 @@ -252,11 +246,8 @@ jobs: GH_TOKEN: ${{ github.token }} DEPS_RUN_ID: ${{ inputs.deps_run_id }} run: | - mkdir -p deps/linux-x64 - ZIP=$(bash .github/scripts/fetch-deps-bundle.sh "${{ matrix.asset }}" \ - "${{ steps.deps.outputs.tag }}" "${{ steps.deps.outputs.ver }}") - unzip -q -o "$ZIP" -d deps/linux-x64 - rm -rf "$ZIP" .deps-artifact + bash .github/scripts/install-deps-bundle.sh "${{ matrix.asset }}" \ + "${{ steps.deps.outputs.tag }}" "${{ steps.deps.outputs.ver }}" deps/linux-x64 - name: Cache whisper model uses: actions/cache@v5 diff --git a/.github/workflows/probe-cpu-compat.yml b/.github/workflows/probe-cpu-compat.yml index 57a4354..9d68ed6 100644 --- a/.github/workflows/probe-cpu-compat.yml +++ b/.github/workflows/probe-cpu-compat.yml @@ -101,19 +101,21 @@ jobs: run: | mkdir -p "$D" if [ -d .deps-artifact ]; then + # Gate mode: one full zip built earlier in this run. For a v2 tier + # that is the v3 bundle with the delta already applied (see the + # delta-x64 job of the calling build-deps-* workflow). ZIP=$(find .deps-artifact -name "*-$ASSET_ID.zip" | head -1) + [ -f "$ZIP" ] || { echo "no bundle zip for $ASSET_ID"; exit 1; } + if [ "$RUNNER_OS" = Windows ]; then + 7z x "$ZIP" -o"$D" -y >/dev/null + else + unzip -q -o "$ZIP" -d "$D" + fi + rm -rf "$ZIP" .deps-artifact else TAG=$(sed -n 's/.*"releaseTag": *"\([^"]*\)".*/\1/p' app/assets/deps-version.json) - ZIP=$(bash .github/scripts/fetch-deps-bundle.sh "$ASSET_ID" "$TAG" "${TAG#deps-v}") + bash .github/scripts/install-deps-bundle.sh "$ASSET_ID" "$TAG" "${TAG#deps-v}" "$D" fi - [ -f "$ZIP" ] || { echo "no bundle zip for $ASSET_ID"; exit 1; } - if [ "$RUNNER_OS" = Windows ]; then - # The Windows zip uses backslash separators; 7-Zip copes, unzip does not. - 7z x "$ZIP" -o"$D" -y >/dev/null - else - unzip -q -o "$ZIP" -d "$D" - fi - rm -rf "$ZIP" .deps-artifact - name: Fetch the positive control (upstream zsmooth, AVX2-only) run: | diff --git a/.gitignore b/.gitignore index fc912d0..c37ace7 100644 --- a/.gitignore +++ b/.gitignore @@ -70,6 +70,8 @@ deps/ # Build Outputs # ============================================================================= dist/ +# Deps zips fetched for package-linux.sh --bundle-deps +bundled-deps/ *.app *.dmg *.exe diff --git a/CLAUDE.md b/CLAUDE.md index bfa3f73..0087147 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -371,7 +371,10 @@ Adding a filter touches many files. Missing any step causes silent failures (fil present before zipping and fail the build if any are missing, so a dead download URL becomes a red build instead of a silently-incomplete bundle. x64 platforms have a second key per CPU tier (`macos-x64-v2`, …) that must - list the same plugins — `attribution_test.dart` asserts it. + list the same plugins — `attribution_test.dart` asserts it. **If the plugin + is built differently for the v2 tier, also add its path to `_tierFiles`** in + the same file: that list is the entire content of the published v2 delta, so + a tier-specific plugin missing from it ships as its v3 build to older CPUs. - **Check it for load-time SIMD on x64** (see "CPU tiers" below): a prebuilt x86 binary built with `-mavx2`-style flags can SIGILL inside `dlopen` on an older CPU, and autoload makes that every job. Run @@ -671,20 +674,41 @@ disagree. Pin the **series** against a rolling host tag (e.g. BtbN's `n9.0-latest`), never an exact old build — a fixed old tag gets garbage-collected off a rolling `latest` alias and 404s the deps build outright. -**CPU tiers (issue #92): every x86 deps bundle ships twice.** `v3` is built -for x86-64-v3 CPUs (Haswell, 2013, and later); `v2` runs on anything older. -The rules: +**CPU tiers (issue #92): every x86 platform's deps come in two tiers.** `v3` +is built for x86-64-v3 CPUs (Haswell, 2013, and later); `v2` runs on anything +older. From deps 1.13.0 only v3 is a full bundle: v2 is a small **delta** +extracted over it. The rules: - **One decision, in the worker.** `cpu::cpu_tier()` (`worker/src/cpu.rs`) requires the *whole* x86-64-v3 set, not just AVX2. `--probe-cpu` reports it, the app downloads by it, and `ctmf_opt` derives from it. Never re-derive the tier elsewhere (sysctl is wrong under Rosetta); on x86 an unclear answer means `v2`, which only costs speed. `VAPOURBOX_DEPS_TIER=v2|v3` overrides it. -- **The tier lives only in the asset name and version.json.** `v3` keeps the - plain names (`VapourBox-deps-X-macos-x64.zip`), `v2` is suffixed - (`…-macos-x64-v2.zip`); both install to `deps/`. The app re-checks - the installed tier at startup (`DependencyStatus.wrongTier`) and replaces a - mismatch even when it is newer than expected. +- **The tier lives only in which assets are installed and in version.json.** + The v3 bundle keeps the plain name (`VapourBox-deps-X-macos-x64.zip`); the v2 + tier is that same bundle plus `…-macos-x64-v2-delta.zip`, extracted over it. + Either way the result installs to `deps/`. + `DependencyManager.installAssetIds` is the one statement of which zips make + up an install, and `"tierFormat": "delta"` in `deps-version.json` is what + selects this (absent = the 1.11.0–1.12.0 scheme of a second full + `…-x64-v2.zip`). The app re-checks the installed tier at startup + (`DependencyStatus.wrongTier`) and reinstalls on a mismatch — from the + bundle, never by patching the installed tree — even when it is newer than + expected. +- **The delta is the files in `_tierFiles`, nothing else, and it is tied to + its bundle.** `Scripts/make-deps-delta.py` takes exactly the paths listed + per platform under `_tierFiles` in `Scripts/deps-expected-plugins.json` from + a full v2 build. It is deliberately not a diff of the two builds (two builds + of the same sources are not byte-identical, so a diff names nearly every + binary); instead it fails if the builds hold different *sets* of paths + (a pure overwrite could not reproduce v2) or if a tier file is + byte-identical in both (the tier switch did nothing). The delta's + `version.json` records `baseSha256`, and both the app and the test harness + refuse to apply it over any other bundle. +- **No zip's `version.json` is ever extracted by the app.** + `_writeInstalledVersion` writes that file last, as the commit marker. The v3 + bundle's own copy says `tier: v3`; letting it through would make a v2 + install interrupted before its delta look like a finished v3 one. - **One `--tier` block per `download-deps-*` script** maps the tier to build variables; nothing else in the script branches on it. The two tiers differ only where running the bundle on an old CPU proved they must: zsmooth @@ -692,11 +716,21 @@ The rules: runtime dispatch) everywhere, and on macOS MVTools (v2 builds v24 from source with `patches/mvtools-v24-no-avx2.patch`, because Stefan-Olt's build SIGILLs in its AVX2 static initializers inside `dlopen`). -- **A v2 bundle is gated before it can be published**: under Intel SDE by - `probe-cpu-compat.yml` (Linux at Westmere; Windows at Sandy Bridge, since SDE - cannot emulate pre-AVX Windows — Microsoft's runtime reads the host's CPU - features from the kernel), and on macOS statically by - `Scripts/check-load-time-simd.py` inside `package-deps-macos.sh`. +- **A v2 delta is gated before it can be published**, against what a v2 user + ends up with — the v3 bundle with the delta applied, composed by the + `delta-x64` job of each `build-deps-*` workflow — not against the v2 build it + was cut from: under Intel SDE by `probe-cpu-compat.yml` (Linux at Westmere; + Windows at Sandy Bridge, since SDE cannot emulate pre-AVX Windows — + Microsoft's runtime reads the host's CPU features from the kernel), and on + macOS statically by `Scripts/check-load-time-simd.py` (once on the v2 build + inside `package-deps-macos.sh`, again on the composed tree). The full v2 + build is a CI intermediate (artifact `…-x64-v2-build`) and must never reach a + release; `deps_delta_install_test.dart` asserts no workflow can upload it. +- **CI installs a v2 tier the way the app does.** + `.github/scripts/install-deps-bundle.sh` is the one place the workflows learn + that `linux-x64-v2` means "bundle, then delta"; `WorkerHarness._downloadDeps` + is the same for local test runs. Don't unzip a deps asset in a workflow + directly. **CTMF's `opt` (SIMD level) must be chosen by the worker from the CPU, never left at the plugin's own auto-detect (`opt=0`).** Its AVX-512 kernel for 8-bit @@ -1312,7 +1346,9 @@ gh workflow run ci-test.yml --ref \ It's a comma-separated list because macOS/Windows/Linux are three separate `build-deps-*` workflow runs; each CI job tries every ID and takes the first -holding its platform's artifact. The bundle version is deliberately not +holding its platform's artifact (`.github/scripts/install-deps-bundle.sh`, +which for a `-v2` row fetches the bundle and the `-v2-delta` artifact and +extracts one over the other). The bundle version is deliberately not cross-checked against `deps-version.json` — it logs a `::warning::` instead, so a green run against an unreleased bundle can't be mistaken for one against the released bundle. @@ -1337,6 +1373,13 @@ there's nothing to protect. `version.json` is written **last**, as the commit marker. Direction is checked with `compareVersions`, not `!=`/string compare — a deliberately newer install must never be treated as outdated and overwritten. +An install can be more than one zip (the v2 tier is bundle + delta, see "CPU +tiers"): each is extracted into the same target in order (`extractBundle`, +`overlay: true` for all but the first), the macOS quarantine strip and re-sign +run once after the last, and the cache keeps every zip of the current install +(`_cachedDownloadPath(keep:)`) so fetching the second doesn't prune the first. +`extractBundle` reads a whole zip into memory, ~240 MB for the Linux bundle. + On Windows, both renames go through a retry loop (`retryTransientFsOperation`) because a directory rename fails with **errno 5** (not a permissions error) if anything — a scanner, the search indexer — briefly holds a handle inside it @@ -1526,18 +1569,48 @@ placement and a version skew would change chroma per-OS. environment is inherited by the worker and by the ffmpeg/vspipe it spawns from the downloaded deps bundle, so anything set there leaks into them. - **The mount is read-only** (`/tmp/.mount_*`). Nothing may be written next - to the executable on Linux; deps and add-ons already live under - `$XDG_DATA_HOME`, which is also why deps stay a first-launch download - instead of being packed in (they are versioned separately and come in two - CPU tiers). + to the executable on Linux; deps and add-ons install under + `$XDG_DATA_HOME`. + - **The AppImage carries the deps zips; the tarball does not.** + `package-linux.sh --bundle-deps DIR` copies the published bundle (and on + x64 the v2 delta), each with its `.sha256.json`, into + `usr/lib/vapourbox/bundled-deps/`, and `DependencyManager` installs from + there instead of downloading (see "Dependency Versioning"). They go in as + the published zips, not unpacked: the mount is read-only and the install + path, tier choice and verification stay exactly the download's. So a deps + release must exist **before** the app build (`build-linux.yml` fetches it + by `deps_tag`), the script refuses zips of any version but the one + `deps-version.json` names, and a deps-only fix still reaches AppImage + users only through a new AppImage or the normal download fallback. + - **Named `VapourBox-X-x86_64.AppImage` / `-aarch64.AppImage`** — the + AppImage convention, which the catalog (appimage.github.io) checks: the + machine's architecture name and no "linux". The tarball keeps + `linux-`. - **The `.AppImage.zsync` must be uploaded beside every AppImage.** The embedded update information names it - (`gh-releases-zsync|…|latest|VapourBox-*-linux-.AppImage.zsync`), so - one missing is an updater that fails for every user. The script fails if - `zsyncmake` is absent rather than build without it, and - `packaging_test.dart` asserts the release upload picks both files up. + (`gh-releases-zsync|…|latest|VapourBox-*-.AppImage.zsync`), + so one missing is an updater that fails for every user. The script fails + if `zsyncmake` is absent rather than build without it, and + `packaging_test.dart` asserts the release upload picks the files up. `latest` is GitHub's Latest release — one more reason a deps or whisper release must never be marked Latest. + - **A second `.zsync`, under the old name, is a bridge for 1.2.0.** 1.2.0 was + the only release named `…-linux-.AppImage` and embeds + `VapourBox-*-linux-.AppImage.zsync`. Each release therefore also + publishes a copy of its `.zsync` as `VapourBox-X-linux-.AppImage.zsync` + (`LEGACY_ZSYNC`); its relative `URL:` header names the new file. Remove it + only once updating from 1.2.0 no longer matters. **Not yet proven + end-to-end**: it needs one real AppImageUpdate run from a 1.2.0 AppImage. + - **AppStream metadata** is `packaging/linux/app.vapourbox.VapourBox.appdata.xml`, + installed to `usr/share/metainfo/` with the version, date and a + tag-pinned screenshot URL stamped in at package time. It is what the + catalog and software centres list. Named `.appdata.xml` because the + catalog's lint only looks for that name, and limited to tags a 2021 + `appstreamcli` knows because that is what the catalog runs. The script + validates it with `--no-net` and passes `--no-appstream` to appimagetool, + whose own check would fetch the screenshot from a tag that does not exist + until the release is published. **`docs/images/screenshot.png` is the + listing's screenshot** — keep it current. - The desktop file and icon are named for `APPLICATION_ID` (`app.vapourbox.VapourBox`, `app/linux/CMakeLists.txt`), which is what the window reports as its app id; rename one and the running window stops @@ -1554,7 +1627,19 @@ Plugin lists for all platforms: see `deps/` directories or download scripts. ## Dependency Versioning and Auto-Download -Dependencies are versioned separately from the app via `app/assets/deps-version.json` and distributed as separate GitHub releases (tag: `deps-vX.Y.Z`). The app auto-downloads deps on launch if missing, outdated, or built for a different CPU tier (x64 only — see "CPU tiers"; the asset is `DependencyManager.assetIdFor(platformId, tier)`). +Dependencies are versioned separately from the app via `app/assets/deps-version.json` and distributed as separate GitHub releases (tag: `deps-vX.Y.Z`). The app auto-downloads deps on launch if missing, outdated, or built for a different CPU tier (x64 only — see "CPU tiers"; the assets are `DependencyManager.installAssetIds(platformId, tier, …)`). + +**"Download" is the last of three sources.** For each zip an install needs, +`DependencyManager._obtainZip` takes, in order: a copy the package shipped in +`bundled-deps/` beside the executable (`findBundledZip`; today only the Linux +AppImage ships one), the cached download from an earlier attempt, then the +network. The bundled directory is one fixed path inside the app's own tree and +must never become a search: a zip found there is extracted and run, so it gets +the trust of the executable next to it and no more places than that. A bundled +zip **must** match the `.sha256.json` shipped beside it (not best-effort, as a +download's sidecar is — offline there is nothing else to say it is whole); one +that doesn't is ignored and the download happens as usual. A bundled zip is +never deleted or copied; only zips the app fetched are cleaned up. `deps-version.json` is a **slim pointer** — `{version, releaseTag, githubRepo}`. Integrity metadata is **not** stored here: each `package-deps-*` script writes a **sidecar** `.sha256.json` uploaded next to the zip, which the app fetches and verifies at download time (best-effort if the sidecar is missing). **Net effect: a new deps release only needs a `version`/`releaseTag` bump.** @@ -1654,6 +1739,7 @@ Notes: | `Scripts/package-deps-macos.sh` | Package macOS deps | | `Scripts/package-deps-windows.ps1` | Package Windows deps | | `Scripts/package-deps-linux.sh` | Package Linux deps | +| `Scripts/make-deps-delta.py` | Cut an x64 platform's v2 delta from a full v2 build and the v3 bundle (run by the `delta-x64` job of each `build-deps-*` workflow) | | `Scripts/download-deps-linux.sh` | Build Linux deps from source | | `Scripts/run-debug-linux.sh` | Dev build + run for Linux | @@ -1661,8 +1747,8 @@ Notes: 1. **Confirm version** — ask user, update `pubspec.yaml` 2. **Check deps** — run `check-deps-changed.sh`; if changed, bump `version`/`releaseTag` in `deps-version.json`. -3. **Build & package deps in CI** — dispatch `build-deps-{macos,windows,linux}.yml` with `release_tag`. They build both x64 tiers, gate the v2 bundles, and only then upload each zip **and** its `.sha256.json` sidecar. -4. **Check the deps release** — every platform, both tiers of each x64: `macos-arm64`, `macos-x64`, `macos-x64-v2`, `windows-x64`, `windows-x64-v2`, `linux-x64`, `linux-x64-v2`, `linux-arm64`. +3. **Build & package deps in CI** — dispatch `build-deps-{macos,windows,linux}.yml` with `release_tag`. They build both x64 tiers, cut the v2 delta, gate the composed v2 tree, and only then upload each zip **and** its `.sha256.json` sidecar. +4. **Check the deps release** — eight zips, each with a sidecar: the five bundles `macos-arm64`, `macos-x64`, `windows-x64`, `linux-x64`, `linux-arm64`, and the three deltas `macos-x64-v2-delta`, `windows-x64-v2-delta`, `linux-x64-v2-delta`. There must be **no** full `…-x64-v2.zip`. 5. **Test** — fresh install + upgrade test 6. **Create GitHub releases** — deps release first (if changed, tag `deps-vX.Y.Z`), then app release (tag `vX.Y.Z`) @@ -1765,3 +1851,4 @@ Full write-ups (root causes, measurements) for each entry are in | 1.10.0 | 2026-08-31 | **Issue #82**: zsmooth now ships one build per x86 CPU baseline (haswell, x86_64_v2), fixing an illegal-instruction crash on pre-2013 CPUs. **FFmpeg pinned to 9.0 on all four platforms** (they had silently diverged: Windows on an unpinned post-9.0 master, macOS on floating 9.0.1, Linux stuck at 7.1 after BtbN garbage-collected the pinned tag) | | 1.11.0 | 2026-09 | **Issue #92**: every x86 bundle ships in two **CPU tiers** — `v3` (x86-64-v3; the plain asset names) and `v2` (`…-x64-v2`, anything older), chosen by the app from `vapourbox-worker --probe-cpu`. Each bundle carries one autoloaded zsmooth (replacing 1.10.0's two-builds-loaded-by-path). macOS v2 builds MVTools v24 from source without its AVX2 files, whose static initializers SIGILLed inside `dlopen` on pre-AVX Macs. v2 bundles are gated (SDE on Linux/Windows, static check on macOS) before publishing | | 1.12.0 | 2026-10-04 | **Issue #101**: akarin 1.4.1 → **1.5.0** on every platform. The 1.4.1 Linux wheel put JIT code on the `brk` heap, where SELinux denies `PROT_EXEC`, so every job segfaulted on Fedora/RHEL from deps 1.8.0 on. `download-deps-linux.sh` now fails if `libakarin.so` lacks the mmap allocator | +| 1.13.0 | 2026-10 | **Packaging only — every binary is built from the same sources as 1.12.0.** The v2 CPU tier is no longer a second full bundle: each x64 platform publishes its v3 bundle plus `…-x64-v2-delta.zip`, the files in `_tierFiles` (zsmooth; on macOS also MVTools), extracted over it (`"tierFormat": "delta"`). Every x64 machine downloads the same bundle; only an older CPU fetches the few MB more. This is also what lets the Linux AppImage carry its deps for both tiers without doubling in size | diff --git a/README.md b/README.md index 7b4e941..64e5c10 100644 --- a/README.md +++ b/README.md @@ -39,8 +39,8 @@ VapourBox runs [VapourSynth](https://www.vapoursynth.com/), QTGMC and FFmpeg — | macOS (Apple Silicon) | macOS 15 Sequoia | `VapourBox-x.x.x-macos-arm64.dmg` | | macOS (Intel) | macOS 12 Monterey | `VapourBox-x.x.x-macos-x64.dmg` | | Windows (x64) | Windows 10/11 | `VapourBox-x.x.x-windows-x64-setup.exe` | -| Linux (x64) | glibc 2.39 (Ubuntu 24.04, Debian 13) | `VapourBox-x.x.x-linux-x64.AppImage` | -| Linux (arm64) | glibc 2.39 (Ubuntu 24.04, Debian 13) | `VapourBox-x.x.x-linux-arm64.AppImage` | +| Linux (x64) | glibc 2.39 (Ubuntu 24.04, Debian 13) | `VapourBox-x.x.x-x86_64.AppImage` | +| Linux (arm64) | glibc 2.39 (Ubuntu 24.04, Debian 13) | `VapourBox-x.x.x-aarch64.AppImage` | All processing is local. There is no account, no telemetry, and nothing is uploaded. @@ -72,20 +72,20 @@ It installs for the current user only (under `%LOCALAPPDATA%\Programs\VapourBox`
Installing on Linux -1. Download the `.AppImage` for your architecture. -2. Make it executable: `chmod +x VapourBox-x.x.x-linux-x64.AppImage` (or tick *Allow executing file as program* in the file manager's Properties). -3. Run it: `./VapourBox-x.x.x-linux-x64.AppImage` -4. First launch downloads its processing dependencies (~185 MB). +1. Download the `.AppImage` for your architecture (`x86_64` for Intel/AMD, `aarch64` for ARM). +2. Make it executable: `chmod +x VapourBox-x.x.x-x86_64.AppImage` (or tick *Allow executing file as program* in the file manager's Properties). +3. Run it: `./VapourBox-x.x.x-x86_64.AppImage` +4. First launch unpacks the processing dependencies it carries. No download is needed, so it works offline. The AppImage is a single file that needs no installation and can live anywhere. To get a menu entry and icon, open it with an AppImage manager such as [Gear Lever](https://github.com/mijorus/gearlever) or [AppImageLauncher](https://github.com/TheAssassin/AppImageLauncher); those, and [AppImageUpdate](https://github.com/AppImageCommunity/AppImageUpdate), can also update it in place, downloading only what changed. -Each release also carries a plain `.tar.gz` of the same build, for systems where an AppImage can't be mounted: extract it and run `./vapourbox`. +Each release also carries a plain `.tar.gz` of the same build, for systems where an AppImage can't be mounted: extract it and run `./vapourbox`. It is much smaller because it leaves the processing dependencies out, and downloads them on first launch (~240 MB) instead. GPU-accelerated deinterlacing (NNEDI3CL) needs your GPU's OpenCL driver installed. Without it, VapourBox falls back to the CPU automatically.
-**Older Intel and AMD processors** (before about 2013 — no AVX2) are supported on all three platforms. On first launch VapourBox checks the processor and downloads the matching set of processing components; a machine moved to a different CPU gets the right set on its next launch. The older-CPU set runs some filters more slowly, never differently. Every build of that set is checked for pre-AVX processors (e.g. a 2010 Mac Pro) on macOS and Linux; on Windows it is checked for processors with AVX but no AVX2, and older ones are expected to work but can't be tested automatically. +**Older Intel and AMD processors** (before about 2013 — no AVX2) are supported on all three platforms. On first launch VapourBox checks the processor and installs the matching set of processing components; a machine moved to a different CPU gets the right set on its next launch. The older-CPU set runs some filters more slowly, never differently. Every build of that set is checked for pre-AVX processors (e.g. a 2010 Mac Pro) on macOS and Linux; on Windows it is checked for processors with AVX but no AVX2, and older ones are expected to work but can't be tested automatically. ## Output formats diff --git a/Scripts/deps-expected-plugins.json b/Scripts/deps-expected-plugins.json index a0f1784..d3a2740 100644 --- a/Scripts/deps-expected-plugins.json +++ b/Scripts/deps-expected-plugins.json @@ -1,5 +1,18 @@ { "_comment": "Required VapourSynth plugin filenames per platform — the contract for a COMPLETE deps bundle. The package-deps-* scripts assert every file listed here exists in the staged bundle's plugin directory before zipping and FAIL the build if any are missing, so a silently-failed download (e.g. a dead upstream URL) becomes a red build instead of an incomplete bundle shipping. Plugin directory: windows-x64 = vapoursynth/vs-plugins, macos/linux = vapoursynth/plugins. Lists exclude data files (nnedi3 weights) and runtime libs (fftw); they cover the VapourSynth plugin binaries only. Update this when adding or removing a plugin. The x64 platforms ship in two CPU tiers (issue #92): the plain key is the v3 bundle, the -v2 key the bundle for CPUs below x86-64-v3. Both tiers carry the same set of plugins, built differently, so their lists must stay identical (asserted by app/test/attribution_test.dart).", + "_tierFilesComment": "The files that differ between an x64 platform's two CPU tiers, as paths from the bundle root. From deps 1.13.0 the v2 tier is not published as a second full bundle: Scripts/make-deps-delta.py takes exactly these files from the v2 build and publishes them as VapourBox-deps---v2-delta.zip, which the app extracts over the v3 bundle. A file that must differ on older CPUs but is missing here ships as its v3 build and crashes there, so add it in the same change as the download-deps-* tier block that builds it differently. app/test/attribution_test.dart asserts each entry is a plugin the platform lists.", + "_tierFiles": { + "windows-x64": [ + "vapoursynth/vs-plugins/zsmooth.dll" + ], + "macos-x64": [ + "vapoursynth/plugins/libmvtools.dylib", + "vapoursynth/plugins/libzsmooth.dylib" + ], + "linux-x64": [ + "vapoursynth/plugins/libzsmooth.so" + ] + }, "windows-x64": [ "AddGrain.dll", "bifrost.dll", diff --git a/Scripts/make-deps-delta.py b/Scripts/make-deps-delta.py new file mode 100755 index 0000000..3ac4c74 --- /dev/null +++ b/Scripts/make-deps-delta.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +"""Turn a full v2-tier deps bundle into a delta over the v3 bundle. + +From deps 1.13.0 an x64 platform publishes ONE full bundle (the v3 tier, under +the plain asset name) plus a small delta for older CPUs: + + VapourBox-deps--.zip full bundle, v3 tier + VapourBox-deps---v2-delta.zip the files v2 replaces + +The app (and every CI consumer) installs the v2 tier by extracting the delta +over the full bundle. The delta holds exactly the files listed for the platform +under `_tierFiles` in Scripts/deps-expected-plugins.json, taken from a real v2 +build, plus a version.json stamped `tier: v2` — so "unzip the bundle, unzip the +delta over it" yields a correct tree with nothing else to do. + +It is NOT made by diffing the two builds: two builds of the same sources are +not byte-identical, so a diff names nearly every binary. What IS checked is +that the two builds hold the same set of paths, which is what makes a pure +overwrite sufficient (a file only one tier has would need a delete step). + +The delta records the sha256 of the bundle it was cut against (`baseSha256`, +in its version.json and its sidecar). The app refuses to apply it over any +other bundle, so a rebuilt v3 can never be paired with a stale delta. + +Usage: + make-deps-delta.py --platform linux-x64 --version 1.13.0 \\ + --base dist/VapourBox-deps-1.13.0-linux-x64.zip \\ + --v2 build/VapourBox-deps-1.13.0-linux-x64-v2.zip \\ + --out dist [--composed dist-composed] + +--composed also writes the full v2 bundle obtained by applying the delta to the +base (VapourBox-deps---v2.zip). That file is a CI +intermediate for the CPU gate, which must test what a user ends up with rather +than the v2 build the delta was cut from. It is never published. +""" +import argparse +import datetime +import hashlib +import json +import os +import sys +import zipfile + +MANIFEST = os.path.join(os.path.dirname(os.path.abspath(__file__)), + "deps-expected-plugins.json") +VERSION_FILE = "version.json" + + +def norm(name): + # Compress-Archive writes backslash separators on Windows; everything here + # compares and writes forward slashes. + return name.replace("\\", "/") + + +def sha256_of(path): + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def file_entries(zf): + """{normalized path: ZipInfo} for every non-directory entry.""" + return {norm(i.filename): i for i in zf.infolist() if not i.is_dir()} + + +def copy_entry(dst, src, info, name): + """Copy one entry, keeping its mode bits (and so its symlink-ness).""" + out = zipfile.ZipInfo(name, date_time=info.date_time) + out.external_attr = info.external_attr + out.create_system = info.create_system + out.compress_type = zipfile.ZIP_DEFLATED + dst.writestr(out, src.read(info)) + + +def write_sidecar(zip_path, version, extra=None): + sidecar = { + "filename": os.path.basename(zip_path), + "sha256": sha256_of(zip_path), + "size": os.path.getsize(zip_path), + "version": version, + } + sidecar.update(extra or {}) + with open(zip_path + ".sha256.json", "w") as f: + json.dump(sidecar, f, indent=2) + f.write("\n") + return sidecar + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--platform", required=True, help="e.g. linux-x64") + ap.add_argument("--version", required=True) + ap.add_argument("--base", required=True, help="the published v3 bundle zip") + ap.add_argument("--v2", required=True, help="a full v2-tier build zip") + ap.add_argument("--out", required=True, help="directory for the delta zip") + ap.add_argument("--composed", help="also write base+delta as a full zip here") + args = ap.parse_args() + + tier_files = json.load(open(MANIFEST, encoding="utf-8")) \ + .get("_tierFiles", {}).get(args.platform) + if not tier_files: + sys.exit(f"no _tierFiles entry for {args.platform} in {MANIFEST}") + + base_sha = sha256_of(args.base) + version_json = json.dumps({ + "version": args.version, + "tier": "v2", + "baseSha256": base_sha, + "installedAt": datetime.datetime.now(datetime.timezone.utc) + .strftime("%Y-%m-%dT%H:%M:%SZ"), + }, indent=2) + "\n" + + os.makedirs(args.out, exist_ok=True) + stem = f"VapourBox-deps-{args.version}-{args.platform}" + delta_path = os.path.join(args.out, f"{stem}-v2-delta.zip") + + with zipfile.ZipFile(args.base) as base, zipfile.ZipFile(args.v2) as v2: + base_files, v2_files = file_entries(base), file_entries(v2) + + for label, zf, files in (("base", args.base, base_files), ("v2", args.v2, v2_files)): + if VERSION_FILE not in files: + sys.exit(f"{zf} has no {VERSION_FILE}; not a deps bundle") + tier = json.loads(zipfile.ZipFile(zf).read(files[VERSION_FILE])).get("tier", "v3") + want = "v3" if label == "base" else "v2" + if tier != want: + sys.exit(f"{zf} is a {tier} bundle; --{label} must be the {want} one") + + # A pure overwrite only works if neither tier has a file the other lacks. + only_base = sorted(set(base_files) - set(v2_files)) + only_v2 = sorted(set(v2_files) - set(base_files)) + if only_base or only_v2: + for p in only_base: + print(f" only in the v3 bundle: {p}", file=sys.stderr) + for p in only_v2: + print(f" only in the v2 build: {p}", file=sys.stderr) + sys.exit("the two tiers do not hold the same files, so extracting a " + "delta over the v3 bundle cannot reproduce the v2 one") + + missing = [p for p in tier_files if p not in v2_files] + if missing: + sys.exit("_tierFiles names files the v2 build does not contain: " + + ", ".join(missing)) + + # A tier file identical in both builds was not built differently at + # all, which means the tier switch in download-deps-* did nothing. + same = [p for p in tier_files + if base.read(base_files[p]) == v2.read(v2_files[p])] + if same: + sys.exit("these tier files are byte-identical in both tiers, so the " + "v2 build did not actually build them for older CPUs: " + + ", ".join(same)) + + with zipfile.ZipFile(delta_path, "w", zipfile.ZIP_DEFLATED) as delta: + for p in tier_files: + copy_entry(delta, v2, v2_files[p], p) + delta.writestr(VERSION_FILE, version_json) + + if args.composed: + os.makedirs(args.composed, exist_ok=True) + composed_path = os.path.join(args.composed, f"{stem}-v2.zip") + with zipfile.ZipFile(delta_path) as delta, \ + zipfile.ZipFile(composed_path, "w", zipfile.ZIP_DEFLATED) as out: + overlay = file_entries(delta) + for p, info in base_files.items(): + if p not in overlay: + copy_entry(out, base, info, p) + for p, info in overlay.items(): + copy_entry(out, delta, info, p) + write_sidecar(composed_path, args.version) + print(f"composed: {composed_path}") + + sidecar = write_sidecar(delta_path, args.version, + {"tier": "v2", "baseSha256": base_sha}) + print(f"delta: {delta_path} ({sidecar['size']} bytes, " + f"{len(tier_files)} file(s))") + print(f"base: {os.path.basename(args.base)} sha256 {base_sha}") + + +if __name__ == "__main__": + main() diff --git a/Scripts/package-linux.sh b/Scripts/package-linux.sh index 215a3f7..ba711e4 100755 --- a/Scripts/package-linux.sh +++ b/Scripts/package-linux.sh @@ -10,13 +10,19 @@ # architecture — appimagetool is fetched automatically, or set $APPIMAGETOOL # # Usage: ./Scripts/package-linux.sh --version X.Y.Z [--skip-build] [--arch x64|arm64] -# [--skip-appimage] +# [--skip-appimage] [--bundle-deps DIR] +# +# --bundle-deps DIR packs the deps zips found in DIR into the AppImage (never +# the tarball), so it runs with no network on first launch. DIR must hold the +# release assets for the version app/assets/deps-version.json names, each with +# its .sha256.json: the platform bundle, and on x64 the v2 delta as well. set -e VERSION="1.0.0" SKIP_BUILD=false SKIP_APPIMAGE=false +BUNDLE_DEPS="" ARCH="" # Pinned, and checksummed below: this binary assembles what users run. @@ -30,9 +36,10 @@ while [[ $# -gt 0 ]]; do --skip-build) SKIP_BUILD=true; shift ;; --arch) ARCH="$2"; shift 2 ;; --skip-appimage) SKIP_APPIMAGE=true; shift ;; + --bundle-deps) BUNDLE_DEPS="$(cd "$2" && pwd)"; shift 2 ;; *) echo "Unknown option: $1" - echo "Usage: $0 --version X.Y.Z [--skip-build] [--arch x64|arm64] [--skip-appimage]" + echo "Usage: $0 --version X.Y.Z [--skip-build] [--arch x64|arm64] [--skip-appimage] [--bundle-deps DIR]" exit 1 ;; esac @@ -135,9 +142,13 @@ STEP=$((STEP + 1)) # Create AppImage # # The AppImage holds exactly the tree the tarball does, under usr/lib/vapourbox, -# so the two cannot drift. Nothing inside it is written to at runtime: deps and -# add-ons install under $XDG_DATA_HOME, which is also why the processing -# dependencies stay a first-launch download rather than being packed in here. +# so the two cannot drift — plus, with --bundle-deps, the deps zips. Nothing +# inside it is written to at runtime: deps and add-ons install under +# $XDG_DATA_HOME, and a bundled zip is only ever read, by DependencyManager, +# which extracts it there instead of downloading it. The zips are bundled as +# published, not unpacked: the app verifies each against its sidecar and +# installs it through the same path as a download, and an older CPU gets the +# v2 delta extracted over the same bundle (see "CPU tiers" in CLAUDE.md). APPIMAGE_FILE="" if ! $SKIP_APPIMAGE; then echo "[$STEP/$TOTAL_STEPS] Creating AppImage..." @@ -199,9 +210,42 @@ if ! $SKIP_APPIMAGE; then rm -rf "$APPDIR" mkdir -p "$APPDIR/usr/lib" \ "$APPDIR/usr/share/applications" \ + "$APPDIR/usr/share/metainfo" \ "$APPDIR/usr/share/icons/hicolor/256x256/apps" cp -r "$PACKAGE_DIR" "$APPDIR/usr/lib/vapourbox" + # Bundled deps. The app looks for exactly the file names its own + # deps-version.json derives, so a zip of any other version would be ~240 MB + # of dead weight that nothing reports; require the right ones, and check + # each against its sidecar now rather than have every user's first launch + # discover a bad copy and fall back to downloading. + if [ -n "$BUNDLE_DEPS" ]; then + DEPS_VERSION=$(sed -n 's/.*"version": *"\([^"]*\)".*/\1/p' \ + "$PROJECT_ROOT/app/assets/deps-version.json" | head -1) + BUNDLED_ZIPS=("VapourBox-deps-$DEPS_VERSION-linux-$ARCH.zip") + if [ "$ARCH" = "x64" ]; then + BUNDLED_ZIPS+=("VapourBox-deps-$DEPS_VERSION-linux-$ARCH-v2-delta.zip") + fi + mkdir -p "$APPDIR/usr/lib/vapourbox/bundled-deps" + for ZIP in "${BUNDLED_ZIPS[@]}"; do + if [ ! -f "$BUNDLE_DEPS/$ZIP" ] || [ ! -f "$BUNDLE_DEPS/$ZIP.sha256.json" ]; then + echo "ERROR: --bundle-deps: $BUNDLE_DEPS has no $ZIP (and its .sha256.json)." + echo " The app expects deps $DEPS_VERSION (app/assets/deps-version.json)." + exit 1 + fi + WANT=$(sed -n 's/.*"sha256": *"\([^"]*\)".*/\1/p' "$BUNDLE_DEPS/$ZIP.sha256.json") + if ! echo "$WANT $BUNDLE_DEPS/$ZIP" | sha256sum -c - >/dev/null; then + echo "ERROR: --bundle-deps: $ZIP does not match its .sha256.json." + exit 1 + fi + cp "$BUNDLE_DEPS/$ZIP" "$BUNDLE_DEPS/$ZIP.sha256.json" \ + "$APPDIR/usr/lib/vapourbox/bundled-deps/" + echo " Bundled $ZIP" + done + else + echo " No --bundle-deps: this AppImage will download its deps on first launch." + fi + install -m 755 "$LINUX_PACKAGING/AppRun" "$APPDIR/AppRun" # appimagetool wants the desktop file and its icon at the root; the copies # under usr/share are what desktop-integration tools install from. @@ -211,24 +255,57 @@ if ! $SKIP_APPIMAGE; then cp "$LINUX_PACKAGING/$APP_ID.png" "$APPDIR/usr/share/icons/hicolor/256x256/apps/" ln -s "$APP_ID.png" "$APPDIR/.DirIcon" + # AppStream metadata, with this release stamped in. + sed -e "s/@VERSION@/$VERSION/g" -e "s/@DATE@/$(date -u +%Y-%m-%d)/g" \ + "$LINUX_PACKAGING/$APP_ID.appdata.xml" \ + > "$APPDIR/usr/share/metainfo/$APP_ID.appdata.xml" + # Validated here, offline, when the tools are present (CI installs them): + # appimagetool's own check goes to the network for the screenshot, whose + # URL is this release's tag and does not exist until the release does. + if command -v desktop-file-validate >/dev/null 2>&1; then + desktop-file-validate "$APPDIR/$APP_ID.desktop" + fi + if command -v appstreamcli >/dev/null 2>&1; then + appstreamcli validate --no-net "$APPDIR/usr/share/metainfo/$APP_ID.appdata.xml" + fi + + # Named the way AppImages are: App-version-arch, with the machine's own + # architecture name and no "linux" (every AppImage is for Linux; the + # AppImage catalog flags it). The tarball keeps linux-$ARCH. + # # "latest" follows GitHub's Latest release, which is why a deps or whisper # release must never be marked Latest (see CLAUDE.md). The wildcard stands # in for the version; the arch stays literal so x64 never updates to arm64. - APPIMAGE_FILE="$PACKAGE_NAME.AppImage" - UPDATE_INFO="gh-releases-zsync|${GITHUB_REPO%%/*}|${GITHUB_REPO##*/}|latest|VapourBox-*-linux-$ARCH.AppImage.zsync" + APPIMAGE_FILE="VapourBox-$VERSION-$AI_ARCH.AppImage" + UPDATE_INFO="gh-releases-zsync|${GITHUB_REPO%%/*}|${GITHUB_REPO##*/}|latest|VapourBox-*-$AI_ARCH.AppImage.zsync" + # 1.2.0, the only release under the old name, embedded the pattern + # VapourBox-*-linux-.AppImage.zsync. Publishing this release's .zsync + # under that name as well lets a 1.2.0 AppImage find it: the file's URL + # header is relative and names the new AppImage, which then carries the new + # pattern. Drop this once 1.2.0 is no longer worth updating from. + LEGACY_ZSYNC="VapourBox-$VERSION-linux-$ARCH.AppImage.zsync" # appimagetool writes the .zsync into the working directory. It is itself # an AppImage; extract-and-run spares the build host from needing FUSE. cd "$DIST_DIR" - rm -f "$APPIMAGE_FILE" "$APPIMAGE_FILE.zsync" + rm -f "$APPIMAGE_FILE" "$APPIMAGE_FILE.zsync" "$LEGACY_ZSYNC" ARCH="$AI_ARCH" APPIMAGE_EXTRACT_AND_RUN=1 VERSION="$VERSION" \ - "$APPIMAGETOOL" --updateinformation "$UPDATE_INFO" "$APPDIR" "$APPIMAGE_FILE" + "$APPIMAGETOOL" --no-appstream --updateinformation "$UPDATE_INFO" \ + "$APPDIR" "$APPIMAGE_FILE" if [ ! -s "$APPIMAGE_FILE" ] || [ ! -s "$APPIMAGE_FILE.zsync" ]; then echo "ERROR: appimagetool did not produce both $APPIMAGE_FILE and $APPIMAGE_FILE.zsync" exit 1 fi chmod +x "$APPIMAGE_FILE" + # The bridge only works if the .zsync points at the AppImage by a relative + # name; an absolute or differently named URL would send 1.2.0 nowhere. + if ! grep -a -q "^URL: $APPIMAGE_FILE\$" "$APPIMAGE_FILE.zsync"; then + echo "ERROR: $APPIMAGE_FILE.zsync does not name $APPIMAGE_FILE as its URL;" + echo " the copy published as $LEGACY_ZSYNC would not update 1.2.0." + exit 1 + fi + cp "$APPIMAGE_FILE.zsync" "$LEGACY_ZSYNC" rm -rf "$APPDIR" STEP=$((STEP + 1)) @@ -260,6 +337,7 @@ if [ -n "$APPIMAGE_FILE" ]; then echo " Size: $(file_size_mb "$APPIMAGE_FILE") MB" echo " SHA256: $(file_sha256 "$APPIMAGE_FILE")" echo " Update: $DIST_DIR/$APPIMAGE_FILE.zsync (upload it beside the AppImage)" + echo " Bridge: $DIST_DIR/$LEGACY_ZSYNC (upload it too; updates 1.2.0)" echo "" fi echo " Tarball: $DIST_DIR/$TAR_FILE" @@ -275,4 +353,8 @@ echo " tar -xzf $TAR_FILE" echo " cd $PACKAGE_NAME" echo " ./vapourbox" echo "" -echo "Note: Dependencies will be downloaded on first launch." +if [ -n "$APPIMAGE_FILE" ] && [ -n "$BUNDLE_DEPS" ]; then + echo "Note: the AppImage carries its dependencies; the tarball downloads them on first launch." +else + echo "Note: Dependencies will be downloaded on first launch." +fi diff --git a/Scripts/release.sh b/Scripts/release.sh index fa95031..d71043c 100755 --- a/Scripts/release.sh +++ b/Scripts/release.sh @@ -306,8 +306,8 @@ RELEASE_NOTES="## VapourBox $APP_VERSION - **Windows**: \`VapourBox-$APP_VERSION-windows-x64-setup.exe\` (or the portable \`.zip\`) - **macOS (Apple Silicon)**: \`VapourBox-$APP_VERSION-macos-arm64.zip\` (contains signed & notarized DMG) - **macOS (Intel)**: \`VapourBox-$APP_VERSION-macos-x64.zip\` (contains signed & notarized DMG) -- **Linux (x64)**: \`VapourBox-$APP_VERSION-linux-x64.AppImage\` (or the \`.tar.gz\`) -- **Linux (arm64)**: \`VapourBox-$APP_VERSION-linux-arm64.AppImage\` (or the \`.tar.gz\`) +- **Linux (x64)**: \`VapourBox-$APP_VERSION-x86_64.AppImage\` (or the \`.tar.gz\`) +- **Linux (arm64)**: \`VapourBox-$APP_VERSION-aarch64.AppImage\` (or the \`.tar.gz\`) ### Installation - **Windows**: Run the installer — or extract the zip and run \`vapourbox.exe\` diff --git a/app/assets/deps-version.json b/app/assets/deps-version.json index 07387b3..17386ec 100644 --- a/app/assets/deps-version.json +++ b/app/assets/deps-version.json @@ -1,6 +1,7 @@ { - "version": "1.12.0", - "releaseTag": "deps-v1.12.0", - "releaseDate": "2026-10-04", + "version": "1.13.0", + "releaseTag": "deps-v1.13.0", + "releaseDate": "2026-10-09", + "tierFormat": "delta", "githubRepo": "StuartCameronCode/VapourBox" } diff --git a/app/lib/services/dependency_manager.dart b/app/lib/services/dependency_manager.dart index cad0dd9..0576848 100644 --- a/app/lib/services/dependency_manager.dart +++ b/app/lib/services/dependency_manager.dart @@ -98,14 +98,23 @@ class DepsVersionInfo { final Map platforms; final String githubRepo; + /// How the release ships the v2 CPU tier of an x64 platform. Absent means a + /// second full bundle (`…-x64-v2.zip`, deps 1.11.0–1.12.0); `delta` means a + /// small zip of the files that differ (`…-x64-v2-delta.zip`, deps 1.13.0 on), + /// extracted over the v3 bundle. See [DependencyManager.installAssetIds]. + final String? tierFormat; + DepsVersionInfo({ required this.version, required this.releaseTag, this.releaseDate, required this.platforms, required this.githubRepo, + this.tierFormat, }); + bool get tierIsDelta => tierFormat == 'delta'; + factory DepsVersionInfo.fromJson(Map json) { final platforms = {}; final platformsJson = json['platforms'] as Map? ?? {}; @@ -121,6 +130,7 @@ class DepsVersionInfo { releaseDate: json['releaseDate'] as String?, platforms: platforms, githubRepo: json['githubRepo'] as String? ?? 'StuartCameronCode/VapourBox', + tierFormat: json['tierFormat'] as String?, ); } @@ -260,6 +270,81 @@ class DependencyManager { static String assetIdFor(String platformId, String? tier) => tier == 'v2' ? '$platformId-v2' : platformId; + /// The release assets that make up an install, in extraction order. + /// + /// One zip, except for the v2 tier of a release whose + /// [DepsVersionInfo.tierFormat] is `delta`: that is the platform's v3 bundle + /// followed by `-v2-delta`, the handful of files built differently + /// for older CPUs, extracted over it. So every x64 machine fetches the same + /// bundle and only an older one fetches anything more. + static List installAssetIds(String platformId, String? tier, + {required bool delta}) => + tier == 'v2' && delta + ? [platformId, '$platformId-v2-delta'] + : [assetIdFor(platformId, tier)]; + + /// Test seam: replaces [bundledDepsDirectory]. + Directory? bundledDepsDirOverride; + + /// Where a package may ship deps zips so the first launch needs no network: + /// `bundled-deps/` beside the executable. The Linux AppImage fills it + /// (Scripts/package-linux.sh --bundle-deps); nothing else does yet. + /// + /// Deliberately one fixed path inside the app's own tree, never a search: a + /// zip found here is extracted and its binaries run, so it gets exactly the + /// trust the executable next to it has, and no more places than that. + Directory bundledDepsDirectory() => + bundledDepsDirOverride ?? + Directory(path.join( + path.dirname(Platform.resolvedExecutable), 'bundled-deps')); + + /// A bundled copy of [filename], or null if the package ships none or it + /// does not match the sidecar shipped beside it. + /// + /// The sidecar is required, not best-effort as it is for a download: with no + /// network there is nothing else to say the zip is whole, and falling back to + /// the download costs nothing when the bundled copy is unusable. + static Future findBundledZip(Directory dir, String filename) async { + final zip = File(path.join(dir.path, filename)); + final sidecar = File('${zip.path}.sha256.json'); + if (!await zip.exists() || !await sidecar.exists()) return null; + try { + final expected = + (jsonDecode(await sidecar.readAsString()) as Map)['sha256']; + if (expected is! String || expected.isEmpty) return null; + final actual = await sha256OfFile(zip); + if (actual != expected) { + print('DependencyManager: bundled $filename does not match its ' + 'sidecar ($actual, expected $expected) - ignoring it'); + return null; + } + return zip; + } catch (e) { + print('DependencyManager: could not use bundled $filename ($e)'); + return null; + } + } + + /// The sha256 of the bundle a v2 delta was cut against, read from the + /// `version.json` inside the delta, or null if it records none. + static String? deltaBaseSha256(Archive delta) { + for (final file in delta) { + if (file.isFile && _isVersionFile(file.name)) { + try { + final json = jsonDecode(utf8.decode(file.content as List)); + final sha = json is Map ? json['baseSha256'] : null; + return sha is String && sha.isNotEmpty ? sha : null; + } catch (_) { + return null; + } + } + } + return null; + } + + static bool _isVersionFile(String entryName) => + entryName.replaceAll('\\', '/') == 'version.json'; + String? _cachedTier; bool _tierResolved = false; @@ -658,65 +743,32 @@ class DependencyManager { final tier = await depsTier(); final assetId = assetIdFor(platformId, tier); - // Construct download URL from release tag (filename is derived). The asset - // carries the tier; the install directory does not. - final downloadUrl = expected.getDownloadUrl(assetId); - final filename = expected.filenameFor(assetId); - - print('DependencyManager: Downloading from $downloadUrl'); + // What to install, in extraction order: one bundle, or for the v2 tier of + // a delta-format release the v3 bundle and then the delta over it. The + // assets carry the tier; the install directory does not. + final assetIds = + installAssetIds(platformId, tier, delta: expected.tierIsDelta); + final filenames = [for (final id in assetIds) expected.filenameFor(id)]; _progressController.add(DownloadProgress( bytesReceived: 0, totalBytes: 0, - status: 'Connecting...', + status: 'Preparing...', )); - // Fetch the integrity sidecar (sha256) uploaded next to the zip. Verification - // is best-effort: if the sidecar is missing/unreadable we still install (the - // download is over HTTPS), matching prior behaviour when no hash was set. - final expectedSha256 = - await _fetchExpectedSha256(expected.getManifestUrl(assetId)); - - // The zip is downloaded into a stable cache path rather than a throwaway - // temp directory, and kept if anything after the download fails. Everything - // past this point — extraction, verification, the swap — can fail for - // reasons that have nothing to do with the bytes we just fetched (issue - // #87), and making the user re-fetch ~200 MB to retry a rename is a poor - // trade for the disk the zip occupies until the install succeeds. - final tempFile = File(await _cachedDownloadPath(filename)); + // Zips this install fetched into the cache, as opposed to ones the package + // shipped. Only these are ours to delete. + final zips = []; + final cached = []; try { - // Reuse the cached zip only when the sidecar gave us a hash to check it - // against. Without one an interrupted download is indistinguishable from - // a complete one, and extracting a truncated zip would report a corrupt - // bundle rather than the missing bytes. - var reusable = false; - if (await tempFile.exists()) { - if (expectedSha256 != null) { - _progressController.add(DownloadProgress( - bytesReceived: 0, - totalBytes: 0, - status: 'Checking the downloaded file...', - )); - reusable = await _sha256OfFile(tempFile) == expectedSha256; - print('DependencyManager: cached download ${reusable ? 'matches the ' - 'expected hash - skipping the download' : 'does not match the ' - 'expected hash - downloading again'}'); - } - if (!reusable) { - await tempFile.delete().catchError((_) => tempFile); - } - } - - if (!reusable) { - await _downloadFile( - downloadUrl, - tempFile, - expectedSha256: expectedSha256, - ); + for (final id in assetIds) { + final zip = await _obtainZip(expected, id, keep: filenames); + zips.add(zip.file); + if (!zip.bundled) cached.add(zip.file); } - // Extract. _extractZip emits per-file extraction progress; this initial + // Extract. extractBundle emits per-file extraction progress; this initial // event (0/0 -> indeterminate) covers the synchronous decode that precedes // the first file write. _progressController.add(DownloadProgress( @@ -725,7 +777,10 @@ class DependencyManager { status: 'Extracting...', )); - final downloadedBytes = await tempFile.length(); + var downloadedBytes = 0; + for (final zip in zips) { + downloadedBytes += await zip.length(); + } // Build the new install alongside the current one and swap it in, rather // than deleting the current one and extracting over the top. The old path @@ -765,7 +820,18 @@ class DependencyManager { final hadPrevious = await depsDir.exists(); final target = hadPrevious ? staging : depsDir; - await _extractZip(tempFile, targetOverride: target); + // A delta is only ever applied over the bundle it was cut against. A + // mismatch means the release was half-updated or a zip is not what its + // name says; either way the result would be a v2 install that is really + // v3 with the wrong plugin dropped in. + if (zips.length > 1) { + await _checkDeltaMatchesBase(base: zips.first, delta: zips.last); + } + + for (var i = 0; i < zips.length; i++) { + await extractBundle(zips[i], target, + overlay: i > 0, finalize: i == zips.length - 1); + } // Prove the install is actually usable before declaring success. The // quarantine strip above can fail — silently, and it cannot succeed at all @@ -869,16 +935,113 @@ class DependencyManager { print('DependencyManager: Installation complete'); - // Only now is the zip dead weight. On any failure above it is deliberately - // left in place so Retry can skip the download. - await tempFile.delete().catchError((_) => tempFile); + // Only now are the downloaded zips dead weight. On any failure above + // they are deliberately left in place so Retry can skip the download. A + // zip the package shipped is never ours to delete. + for (final zip in cached) { + await zip.delete().catchError((_) => zip); + } } catch (e) { - print('DependencyManager: install failed ($e) - keeping the downloaded ' - 'zip at ${tempFile.path} so a retry can reuse it'); + if (cached.isNotEmpty) { + print('DependencyManager: install failed ($e) - keeping the downloaded ' + '${cached.map((f) => f.path).join(', ')} so a retry can reuse ' + 'what was fetched'); + } rethrow; } } + /// One zip of an install: the copy the package shipped if it has a good one, + /// else the cached download, else a fresh download. + Future<({File file, bool bundled})> _obtainZip( + DepsVersionInfo expected, + String assetId, { + required List keep, + }) async { + final filename = expected.filenameFor(assetId); + + final bundled = await findBundledZip(bundledDepsDirectory(), filename); + if (bundled != null) { + print('DependencyManager: using the bundled $filename'); + return (file: bundled, bundled: true); + } + + final downloadUrl = expected.getDownloadUrl(assetId); + print('DependencyManager: Downloading from $downloadUrl'); + _progressController.add(DownloadProgress( + bytesReceived: 0, + totalBytes: 0, + status: 'Connecting...', + )); + + // Fetch the integrity sidecar (sha256) uploaded next to the zip. Verification + // is best-effort: if the sidecar is missing/unreadable we still install (the + // download is over HTTPS), matching prior behaviour when no hash was set. + final expectedSha256 = + await _fetchExpectedSha256(expected.getManifestUrl(assetId)); + + // The zip is downloaded into a stable cache path rather than a throwaway + // temp directory, and kept if anything after the download fails. Everything + // past this point — extraction, verification, the swap — can fail for + // reasons that have nothing to do with the bytes we just fetched (issue + // #87), and making the user re-fetch ~200 MB to retry a rename is a poor + // trade for the disk the zip occupies until the install succeeds. + final tempFile = File(await _cachedDownloadPath(filename, keep: keep)); + + // Reuse the cached zip only when the sidecar gave us a hash to check it + // against. Without one an interrupted download is indistinguishable from + // a complete one, and extracting a truncated zip would report a corrupt + // bundle rather than the missing bytes. + var reusable = false; + if (await tempFile.exists()) { + if (expectedSha256 != null) { + _progressController.add(DownloadProgress( + bytesReceived: 0, + totalBytes: 0, + status: 'Checking the downloaded file...', + )); + reusable = await sha256OfFile(tempFile) == expectedSha256; + print('DependencyManager: cached download ${reusable ? 'matches the ' + 'expected hash - skipping the download' : 'does not match the ' + 'expected hash - downloading again'}'); + } + if (!reusable) { + await tempFile.delete().catchError((_) => tempFile); + } + } + + if (!reusable) { + await _downloadFile( + downloadUrl, + tempFile, + expectedSha256: expectedSha256, + ); + } + return (file: tempFile, bundled: false); + } + + /// Refuse a v2 delta that was not cut against [base]. + Future _checkDeltaMatchesBase( + {required File base, required File delta}) async { + final wanted = + deltaBaseSha256(ZipDecoder().decodeBytes(await delta.readAsBytes())); + final actual = await sha256OfFile(base); + if (wanted == actual) return; + // The cached delta is the likelier stale half (it is tiny and the bundle + // was just verified against its own sidecar), so drop it and let a retry + // fetch it again. + if (!path.isWithin(bundledDepsDirectory().path, delta.path)) { + await delta.delete().catchError((_) => delta); + } + throw DependencyInstallException( + 'The components for older processors do not match the main bundle ' + '(they were built against ${wanted ?? 'an unrecorded bundle'}, but the ' + 'bundle is $actual).', + remedy: 'Try again in a few minutes - a new release may still be ' + 'uploading. If it keeps happening, please report it.', + ); + } + /// What to suggest the user try, given the error that ended the install. /// /// One fixed line of connection advice was actively misleading for anything @@ -928,14 +1091,19 @@ class DependencyManager { /// A fixed name under the (user-configurable) temp directory, so a retry can /// find it. Anything else in there is another version's leftovers and is /// pruned, which is what stops the cache growing without bound. - Future _cachedDownloadPath(String filename) async { + /// + /// [keep] names the other files this install needs: a v2 install has two + /// zips, and fetching the second must not prune the first. + Future _cachedDownloadPath(String filename, + {Iterable keep = const []}) async { final dir = Directory(path.join( (await TempDirectoryService.instance.resolve()).path, 'vapourbox-deps-cache')); if (!await dir.exists()) await dir.create(recursive: true); try { await for (final entry in dir.list()) { - if (entry is File && path.basename(entry.path) != filename) { + final name = path.basename(entry.path); + if (entry is File && name != filename && !keep.contains(name)) { await entry.delete().catchError((_) => entry); } } @@ -946,7 +1114,7 @@ class DependencyManager { } /// Lowercase hex sha256 of [file], in the same form the sidecar publishes. - Future _sha256OfFile(File file) async { + static Future sha256OfFile(File file) async { final digest = await sha256.bind(file.openRead()).first; return digest.toString(); } @@ -1116,23 +1284,34 @@ class DependencyManager { } } - /// Extract a zip file to the deps directory. - Future _extractZip(File zipFile, {Directory? targetOverride}) async { - final depsDir = targetOverride ?? await getDepsDirectory(); - + /// Extract a deps zip into [depsDir]. + /// + /// [overlay] extracts over what is already there instead of replacing it, + /// which is how a v2 delta lands on the v3 bundle. [finalize] runs the + /// once-per-install steps (the macOS quarantine strip and re-sign); pass + /// false for every zip of a multi-zip install but the last. + /// + /// A zip's own `version.json` is never written. [_writeInstalledVersion] + /// writes that file, last, as the mark of a complete install; letting the + /// zip's copy through would make a half-extracted tree look installed, and + /// a v2 install look like v3 until the delta landed. + @visibleForTesting + Future extractBundle(File zipFile, Directory depsDir, + {bool overlay = false, bool finalize = true}) async { // Create parent directory if needed final parentDir = depsDir.parent; if (!await parentDir.exists()) { await parentDir.create(recursive: true); } - // Remove existing deps directory - if (await depsDir.exists()) { - await depsDir.delete(recursive: true); + if (!overlay) { + // Remove existing deps directory + if (await depsDir.exists()) { + await depsDir.delete(recursive: true); + } + await depsDir.create(recursive: true); } - await depsDir.create(recursive: true); - // Read and extract zip final bytes = await zipFile.readAsBytes(); final archive = ZipDecoder().decodeBytes(bytes); @@ -1159,35 +1338,56 @@ class DependencyManager { emitExtractProgress(); + // Marked executable in batches after the loop. One chmod process per file + // was most of a first launch on Linux: the name test below matches every + // terminfo entry in the bundled Python (~2,800 files). + final executables = []; + + // Written synchronously, yielding to the UI at each progress tick. The + // bundle is thousands of tiny files, and awaiting each create and write + // costs an event-loop turn apiece: under software rendering that was + // ~14 ms a file, 40 s of a first launch spent on 2 MB of terminfo. + final createdDirs = {}; + void ensureDir(String dir) { + if (createdDirs.add(dir)) Directory(dir).createSync(recursive: true); + } + for (final file in archive) { + if (_isVersionFile(file.name)) continue; final filePath = path.join(depsDir.path, file.name); if (file.isFile) { - final outFile = File(filePath); - await outFile.parent.create(recursive: true); - await outFile.writeAsBytes(file.content as List); + ensureDir(path.dirname(filePath)); + File(filePath).writeAsBytesSync(file.content as List); - // Set executable permission on Unix if (!Platform.isWindows && _isExecutable(file.name)) { - await Process.run('chmod', ['+x', filePath]); + executables.add(filePath); } extracted += file.size; if (extracted - lastEmitted >= emitEvery) { lastEmitted = extracted; emitExtractProgress(); + await Future.delayed(Duration.zero); } } else { - await Directory(filePath).create(recursive: true); + ensureDir(filePath); } } + // Set executable permission on Unix. Chunked to stay far below ARG_MAX. + const chmodBatch = 500; + for (var i = 0; i < executables.length; i += chmodBatch) { + final end = (i + chmodBatch).clamp(0, executables.length); + await Process.run('chmod', ['+x', ...executables.sublist(i, end)]); + } + // Final 100% extraction tick. extracted = totalBytes; emitExtractProgress(); // On macOS, remove quarantine attribute and re-sign binaries for Gatekeeper - if (Platform.isMacOS) { + if (finalize && Platform.isMacOS) { await _removeQuarantine(depsDir.path); await _codesignBinaries(depsDir.path); } diff --git a/app/lib/views/dependency_download_dialog.dart b/app/lib/views/dependency_download_dialog.dart index 31c14e0..40b3a54 100644 --- a/app/lib/views/dependency_download_dialog.dart +++ b/app/lib/views/dependency_download_dialog.dart @@ -100,17 +100,17 @@ class _DependencyDownloadDialogState extends State { switch (widget.status) { case DependencyStatus.missing: return 'VapourBox requires additional components to process videos.\n\n' - 'These will be downloaded automatically.'; + 'These will be installed automatically.'; case DependencyStatus.outdated: return 'A new version of the processing components is available.\n\n' 'Updating to ensure compatibility.'; case DependencyStatus.wrongTier: return 'The installed processing components were built for a ' 'different processor.\n\n' - 'Downloading the version for this computer.'; + 'Installing the version for this computer.'; case DependencyStatus.corrupted: return 'Some processing components are damaged or incomplete.\n\n' - 'Re-downloading to fix the issue.'; + 'Reinstalling to fix the issue.'; case DependencyStatus.newerThanExpected: // main.dart keeps these and warns instead of opening this dialog; // handled so the status can never fall through to the default. diff --git a/app/test/deps_delta_install_test.dart b/app/test/deps_delta_install_test.dart new file mode 100644 index 0000000..9c61549 --- /dev/null +++ b/app/test/deps_delta_install_test.dart @@ -0,0 +1,278 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:archive/archive.dart'; +import 'package:crypto/crypto.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as p; +import 'package:vapourbox/services/dependency_manager.dart'; + +/// From deps 1.13.0 the v2 CPU tier is not a second full bundle: it is the v3 +/// bundle plus a small delta extracted over it, and a package may ship either +/// zip so the first launch needs no network (the Linux AppImage does). These +/// pin which zips make up an install, when a shipped zip is trusted, and that +/// extracting a delta over a bundle produces the v2 tree and nothing else. +void main() { + List zipOf(Map files) { + final archive = Archive(); + files.forEach((name, content) { + final bytes = utf8.encode(content); + archive.addFile(ArchiveFile(name, bytes.length, bytes)); + }); + return ZipEncoder().encode(archive)!; + } + + late Directory tmp; + setUp(() => tmp = Directory.systemTemp.createTempSync('vb_delta_test_')); + tearDown(() => tmp.deleteSync(recursive: true)); + + group('installAssetIds', () { + test('a delta-format v2 install is the v3 bundle, then the delta', () { + for (final id in ['macos-x64', 'windows-x64', 'linux-x64']) { + expect(DependencyManager.installAssetIds(id, 'v2', delta: true), + [id, '$id-v2-delta']); + } + }); + + test('v3 and ARM installs are one bundle in either format', () { + for (final delta in [true, false]) { + expect(DependencyManager.installAssetIds('linux-x64', 'v3', delta: delta), + ['linux-x64']); + expect( + DependencyManager.installAssetIds('linux-arm64', null, delta: delta), + ['linux-arm64']); + } + }); + + test('a release without tierFormat still ships v2 as a full bundle', () { + // Deps 1.11.0-1.12.0. An app pinned to one of those must keep asking for + // the asset that release actually has. + expect(DependencyManager.installAssetIds('linux-x64', 'v2', delta: false), + ['linux-x64-v2']); + }); + + test('the delta resolves to its own zip and sidecar', () { + // Must agree with what Scripts/make-deps-delta.py writes. + final info = DepsVersionInfo.fromJson({ + 'version': '1.13.0', + 'releaseTag': 'deps-v1.13.0', + 'tierFormat': 'delta', + 'githubRepo': 'StuartCameronCode/VapourBox', + }); + expect(info.tierIsDelta, isTrue); + expect(info.filenameFor('linux-x64-v2-delta'), + 'VapourBox-deps-1.13.0-linux-x64-v2-delta.zip'); + expect( + info.getManifestUrl('linux-x64-v2-delta'), + 'https://github.com/StuartCameronCode/VapourBox/releases/download/' + 'deps-v1.13.0/VapourBox-deps-1.13.0-linux-x64-v2-delta.zip.sha256.json'); + }); + + test('tierFormat defaults to full bundles', () { + final info = DepsVersionInfo.fromJson({'version': '1.12.0'}); + expect(info.tierIsDelta, isFalse); + }); + }); + + group('findBundledZip', () { + const name = 'VapourBox-deps-9.9.9-linux-x64.zip'; + + void writeZip(List bytes, {String? sidecarSha}) { + File(p.join(tmp.path, name)).writeAsBytesSync(bytes); + if (sidecarSha != null) { + File(p.join(tmp.path, '$name.sha256.json')) + .writeAsStringSync(jsonEncode({'filename': name, 'sha256': sidecarSha})); + } + } + + test('a zip matching the sidecar shipped beside it is used', () async { + final bytes = zipOf({'ffmpeg/ffmpeg': 'x'}); + writeZip(bytes, sidecarSha: sha256.convert(bytes).toString()); + final found = await DependencyManager.findBundledZip(tmp, name); + expect(found?.path, p.join(tmp.path, name)); + }); + + test('nothing shipped means download', () async { + expect(await DependencyManager.findBundledZip(tmp, name), isNull); + expect( + await DependencyManager.findBundledZip( + Directory(p.join(tmp.path, 'absent')), name), + isNull); + }); + + test('a zip with no sidecar is not trusted', () async { + // Offline there is nothing else to say the zip is whole. + writeZip(zipOf({'ffmpeg/ffmpeg': 'x'})); + expect(await DependencyManager.findBundledZip(tmp, name), isNull); + }); + + test('a zip that does not match its sidecar falls back to download', + () async { + writeZip(zipOf({'ffmpeg/ffmpeg': 'x'}), sidecarSha: '0' * 64); + expect(await DependencyManager.findBundledZip(tmp, name), isNull); + }); + + test('an unreadable sidecar falls back to download', () async { + writeZip(zipOf({'ffmpeg/ffmpeg': 'x'})); + File(p.join(tmp.path, '$name.sha256.json')).writeAsStringSync('not json'); + expect(await DependencyManager.findBundledZip(tmp, name), isNull); + }); + }); + + group('deltaBaseSha256', () { + Archive deltaWith(String versionJson) => ZipDecoder().decodeBytes( + zipOf({'vapoursynth/plugins/libzsmooth.so': 'v2', 'version.json': versionJson})); + + test('reads the bundle the delta was cut against', () { + expect( + DependencyManager.deltaBaseSha256( + deltaWith(jsonEncode({'tier': 'v2', 'baseSha256': 'abc123'}))), + 'abc123'); + }); + + test('a delta that records no base matches nothing', () { + // null never equals a real hash, so such a delta is always refused. + expect(DependencyManager.deltaBaseSha256(deltaWith('{"tier":"v2"}')), isNull); + expect(DependencyManager.deltaBaseSha256(deltaWith('garbage')), isNull); + expect( + DependencyManager.deltaBaseSha256( + ZipDecoder().decodeBytes(zipOf({'a': 'b'}))), + isNull); + }); + }); + + group('extractBundle', () { + final manager = DependencyManager.instance; + + Future zipFile(String name, Map files) async { + final f = File(p.join(tmp.path, name)); + await f.writeAsBytes(zipOf(files)); + return f; + } + + String read(Directory d, String rel) => + File(p.joinAll([d.path, ...rel.split('/')])).readAsStringSync(); + + test('a delta replaces only the files it carries', () async { + final deps = Directory(p.join(tmp.path, 'deps')); + final base = await zipFile('base.zip', { + 'ffmpeg/ffmpeg': 'ffmpeg', + 'vapoursynth/plugins/libzsmooth.so': 'haswell', + 'vapoursynth/plugins/libmvtools.so': 'mvtools', + 'version.json': '{"version":"9.9.9","tier":"v3"}', + }); + final delta = await zipFile('delta.zip', { + 'vapoursynth/plugins/libzsmooth.so': 'x86_64_v2', + 'version.json': '{"version":"9.9.9","tier":"v2","baseSha256":"x"}', + }); + + await manager.extractBundle(base, deps, finalize: false); + await manager.extractBundle(delta, deps, overlay: true, finalize: false); + + expect(read(deps, 'vapoursynth/plugins/libzsmooth.so'), 'x86_64_v2'); + expect(read(deps, 'vapoursynth/plugins/libmvtools.so'), 'mvtools'); + expect(read(deps, 'ffmpeg/ffmpeg'), 'ffmpeg'); + }); + + test('a zip never writes version.json; that is the commit marker', + () async { + // The bundle's own copy says v3. Extracted, it would make an install + // interrupted before the delta look like a complete v3 one on a v2 CPU. + final deps = Directory(p.join(tmp.path, 'deps')); + final base = await zipFile('base.zip', { + 'ffmpeg/ffmpeg': 'ffmpeg', + 'version.json': '{"version":"9.9.9","tier":"v3"}', + }); + final delta = await zipFile('delta.zip', { + 'ffmpeg/ffmpeg': 'v2', + 'version.json': '{"version":"9.9.9","tier":"v2"}', + }); + await manager.extractBundle(base, deps, finalize: false); + expect(File(p.join(deps.path, 'version.json')).existsSync(), isFalse); + await manager.extractBundle(delta, deps, overlay: true, finalize: false); + expect(File(p.join(deps.path, 'version.json')).existsSync(), isFalse); + }); + + test('without overlay the previous tree is replaced, not merged', () async { + final deps = Directory(p.join(tmp.path, 'deps'))..createSync(); + File(p.join(deps.path, 'stale.so')).writeAsStringSync('old'); + final base = await zipFile('base.zip', {'ffmpeg/ffmpeg': 'ffmpeg'}); + await manager.extractBundle(base, deps, finalize: false); + expect(File(p.join(deps.path, 'stale.so')).existsSync(), isFalse); + }); + + test('executables are marked executable, past one chmod batch', () async { + // chmod runs in batches of 500 after the files are written; 1,200 + // extensionless names (as the bundled Python's terminfo has) cross two + // batch boundaries. + final deps = Directory(p.join(tmp.path, 'deps')); + final base = await zipFile('base.zip', { + 'ffmpeg/ffmpeg': 'ffmpeg', + 'vapoursynth/vspipe': 'vspipe', + 'vapoursynth/plugins/libmvtools.so': 'mvtools', + for (var i = 0; i < 1200; i++) 'python/share/terminfo/x/entry$i': 't', + }); + await manager.extractBundle(base, deps, finalize: false); + + bool executable(String rel) => + File(p.joinAll([deps.path, ...rel.split('/')])).statSync().mode & + 0x49 != + 0; + expect(executable('ffmpeg/ffmpeg'), isTrue); + expect(executable('vapoursynth/vspipe'), isTrue); + expect(executable('python/share/terminfo/x/entry0'), isTrue); + expect(executable('python/share/terminfo/x/entry499'), isTrue); + expect(executable('python/share/terminfo/x/entry500'), isTrue); + expect(executable('python/share/terminfo/x/entry1199'), isTrue); + expect(executable('vapoursynth/plugins/libmvtools.so'), isFalse); + }, skip: Platform.isWindows ? 'no executable bit on Windows' : null); + }); + + group('the shipped pointer and manifest', () { + String repoFile(List parts) { + var dir = Directory.current; + while (!Directory(p.join(dir.path, 'worker')).existsSync() || + !Directory(p.join(dir.path, 'app')).existsSync()) { + dir = dir.parent; + } + return File(p.joinAll([dir.path, ...parts])).readAsStringSync(); + } + + test('every tier file is a plugin its platform is required to ship', () { + final manifest = jsonDecode(repoFile(['Scripts', 'deps-expected-plugins.json'])) + as Map; + final tierFiles = manifest['_tierFiles'] as Map; + // A tiered platform with no tier files would publish an empty delta and + // hand older CPUs the v3 plugins. + expect(tierFiles.keys.toSet(), {'macos-x64', 'windows-x64', 'linux-x64'}); + for (final entry in tierFiles.entries) { + final files = (entry.value as List).cast(); + expect(files, isNotEmpty, reason: entry.key); + final plugins = (manifest[entry.key] as List).cast().toSet(); + final pluginDir = entry.key.startsWith('windows') + ? 'vapoursynth/vs-plugins/' + : 'vapoursynth/plugins/'; + for (final f in files) { + expect(f, startsWith(pluginDir), reason: entry.key); + expect(plugins, contains(f.substring(pluginDir.length)), + reason: '${entry.key}: $f is not in the platform plugin list'); + } + } + }); + + test('no deps workflow can publish a full v2 bundle', () { + // The v2 build is an intermediate. A release holding both it and the + // delta would leave two different answers to "what is the v2 tier". + for (final os in ['macos', 'windows', 'linux']) { + final workflow = + repoFile(['.github', 'workflows', 'build-deps-$os.yml']); + expect(workflow, contains(r'"dist/$STEM-v2-delta.zip"'), reason: os); + expect(workflow, contains(r'"dist/$STEM-v2-delta.zip.sha256.json"'), + reason: os); + expect(workflow, isNot(contains('-x64*.zip')), reason: os); + expect(workflow, isNot(contains(r'dist/${{ env.ASSET }}.zip" \')), + reason: '$os uploads a matrix build straight to the release'); + } + }); + }); +} diff --git a/app/test/packaging_test.dart b/app/test/packaging_test.dart index 4c4ef4d..36d48cf 100644 --- a/app/test/packaging_test.dart +++ b/app/test/packaging_test.dart @@ -144,17 +144,39 @@ void main() { final package = _read(root, ['Scripts', 'package-linux.sh']); test('names the .zsync the script itself produces', () { - expect(package, contains(r'APPIMAGE_FILE="$PACKAGE_NAME.AppImage"')); - expect( - package, contains(r'PACKAGE_NAME="VapourBox-$VERSION-linux-$ARCH"')); + expect(package, + contains(r'APPIMAGE_FILE="VapourBox-$VERSION-$AI_ARCH.AppImage"')); expect( package, - contains(r'|latest|VapourBox-*-linux-$ARCH.AppImage.zsync"'), + contains(r'|latest|VapourBox-*-$AI_ARCH.AppImage.zsync"'), reason: 'the update pattern must be the output filename with the ' 'version wildcarded, plus .zsync', ); }); + test('the AppImage is named App-version-arch, without "linux"', () { + // The AppImage catalog flags "linux" in an AppImage's file name, and + // wants the machine's own architecture name. The tarball is not an + // AppImage and keeps linux-. + expect(package, contains('AI_ARCH="x86_64"')); + expect(package, contains('AI_ARCH="aarch64"')); + expect( + package, contains(r'PACKAGE_NAME="VapourBox-$VERSION-linux-$ARCH"')); + expect(package, contains(r'TAR_FILE="$PACKAGE_NAME.tar.gz"')); + }); + + test('1.2.0 can still find its update under the name it embedded', () { + // 1.2.0 shipped as VapourBox-1.2.0-linux-.AppImage and looks for + // VapourBox-*-linux-.AppImage.zsync on the latest release. That + // file must keep being published, as a copy of the real .zsync, or + // every 1.2.0 AppImage silently stops updating. + expect( + package, + contains( + r'LEGACY_ZSYNC="VapourBox-$VERSION-linux-$ARCH.AppImage.zsync"')); + expect(package, contains(r'cp "$APPIMAGE_FILE.zsync" "$LEGACY_ZSYNC"')); + }); + test('the release upload carries the AppImage and its .zsync', () { final upload = _read(root, ['Scripts', 'ci-build-and-release.sh']); final finds = upload @@ -168,12 +190,56 @@ void main() { } final workflow = _read(root, ['.github', 'workflows', 'build-linux.yml']); - for (final arch in ['x64', 'arm64']) { - expect(workflow, contains('-linux-$arch.AppImage\n')); - expect(workflow, contains('-linux-$arch.AppImage.zsync\n')); + for (final arch in {'x64': 'x86_64', 'arm64': 'aarch64'}.entries) { + expect(workflow, contains('-${arch.value}.AppImage\n')); + expect(workflow, contains('-${arch.value}.AppImage.zsync\n')); + expect(workflow, contains('-linux-${arch.key}.AppImage.zsync\n'), + reason: 'the bridge .zsync for 1.2.0 must be uploaded too'); + expect(workflow, isNot(contains('-linux-${arch.key}.AppImage\n'))); } }); + test('the AppImage bundles the deps zips; the tarball does not', () { + final workflow = _read(root, ['.github', 'workflows', 'build-linux.yml']); + expect('--bundle-deps bundled-deps'.allMatches(workflow).length, 2, + reason: 'both architectures must bundle'); + // Only x64 is tiered, so only x64 carries a v2 delta. + expect('-v2-delta.zip.sha256.json'.allMatches(workflow).length, 1); + // The zips go into the AppDir after the tarball's tree was copied into + // it, never into the tree both are made from. + expect(package, contains(r'"$APPDIR/usr/lib/vapourbox/bundled-deps"')); + expect(package, isNot(contains(r'"$PACKAGE_DIR/bundled-deps'))); + // The directory name is the one DependencyManager looks in. + final manager = + _read(root, ['app', 'lib', 'services', 'dependency_manager.dart']); + expect(manager, contains("'bundled-deps'")); + }); + + test('ships AppStream metadata for the application id', () { + final cmake = _read(root, ['app', 'linux', 'CMakeLists.txt']); + final id = RegExp(r'set\(APPLICATION_ID "([^"]+)"\)') + .firstMatch(cmake)! + .group(1)!; + // .appdata.xml: the only name the AppImage catalog's lint looks for. + final appdata = _read(root, ['packaging', 'linux', '$id.appdata.xml']); + expect(appdata, contains('$id')); + expect(appdata, + contains('$id.desktop')); + expect(appdata, contains('')); + // Stamped at package time, so neither can go stale. + expect(appdata, contains('')); + expect(appdata, contains('/v@VERSION@/docs/images/screenshot.png')); + expect( + File(p.join(root, 'docs', 'images', 'screenshot.png')).existsSync(), + isTrue); + expect(package, + contains(r'"$APPDIR/usr/share/metainfo/$APP_ID.appdata.xml"')); + // appimagetool's own validation fetches the screenshot, whose tag does + // not exist until the release is published. + expect(package, contains('--no-appstream')); + expect(package, contains('appstreamcli validate --no-net')); + }); + test('the desktop entry and icon are named for the application id', () { final cmake = _read(root, ['app', 'linux', 'CMakeLists.txt']); final id = RegExp(r'set\(APPLICATION_ID "([^"]+)"\)') diff --git a/app/test/support/worker_harness.dart b/app/test/support/worker_harness.dart index 53aa932..efb6219 100644 --- a/app/test/support/worker_harness.dart +++ b/app/test/support/worker_harness.dart @@ -760,44 +760,58 @@ class WorkerHarness { return DepsVersionInfo.fromJson(json); } - /// Download + extract the pinned deps zip into [destDir]. + /// Download + extract the pinned deps into [destDir], composed exactly as + /// the app composes them: one bundle, or for a v2 tier of a delta-format + /// release the v3 bundle with the v2 delta extracted over it + /// (`DependencyManager.installAssetIds`). static Future _downloadDeps(String destDir) async { final info = _loadDepsVersion(); final tier = await depsTier(); final assetId = DependencyManager.assetIdFor(platform, tier); - final url = info.getDownloadUrl(assetId); - final expectedSha = await _fetchSidecarSha(info.getManifestUrl(assetId)); + final assetIds = DependencyManager.installAssetIds(platform, tier, + delta: info.tierIsDelta); final tmp = await Directory.systemTemp.createTemp('vb_deps_'); - final zip = File(p.join(tmp.path, info.filenameFor(assetId))); try { - // ignore: avoid_print - print('WorkerHarness: downloading $url'); - final bytes = await _httpGetBytes(url); - if (expectedSha != null) { - final actual = sha256.convert(bytes).toString(); - if (actual != expectedSha) { - throw StateError('deps sha256 mismatch (expected $expectedSha, got $actual)'); - } - } - await zip.writeAsBytes(bytes); - final dest = Directory(destDir); if (dest.existsSync()) dest.deleteSync(recursive: true); dest.createSync(recursive: true); - final archive = ZipDecoder().decodeBytes(bytes); - for (final f in archive) { - final outPath = p.join(destDir, f.name); - if (f.isFile) { - final outFile = File(outPath); - outFile.parent.createSync(recursive: true); - outFile.writeAsBytesSync(f.content as List); - if (!Platform.isWindows && _isExecutable(f.name)) { - await Process.run('chmod', ['+x', outPath]); - } + String? baseSha; + for (final id in assetIds) { + final url = info.getDownloadUrl(id); + final expectedSha = await _fetchSidecarSha(info.getManifestUrl(id)); + // ignore: avoid_print + print('WorkerHarness: downloading $url'); + final bytes = await _httpGetBytes(url); + final actual = sha256.convert(bytes).toString(); + if (expectedSha != null && actual != expectedSha) { + throw StateError( + 'deps sha256 mismatch for $id (expected $expectedSha, got $actual)'); + } + + final archive = ZipDecoder().decodeBytes(bytes); + if (id == assetIds.first) { + baseSha = actual; } else { - Directory(outPath).createSync(recursive: true); + final wanted = DependencyManager.deltaBaseSha256(archive); + if (wanted != baseSha) { + throw StateError('$id was cut against bundle $wanted, but the ' + 'bundle downloaded is $baseSha'); + } + } + for (final f in archive) { + final outPath = p.join(destDir, f.name); + if (f.isFile) { + final outFile = File(outPath); + outFile.parent.createSync(recursive: true); + outFile.writeAsBytesSync(f.content as List); + if (!Platform.isWindows && _isExecutable(f.name)) { + await Process.run('chmod', ['+x', outPath]); + } + } else { + Directory(outPath).createSync(recursive: true); + } } } diff --git a/docs/BUILDING.md b/docs/BUILDING.md index 22c4740..8e9ab12 100644 --- a/docs/BUILDING.md +++ b/docs/BUILDING.md @@ -277,17 +277,30 @@ already-assembled package directory. ### Linux ```bash -./Scripts/package-linux.sh --version 1.0.0 [--skip-build] [--arch x64|arm64] +./Scripts/package-linux.sh --version 1.0.0 [--skip-build] [--arch x64|arm64] \ + [--skip-appimage] [--bundle-deps DIR] ``` -`--arch` defaults to the host. Output, all from the same tree: +`--arch` defaults to the host. Output, all from the same tree (`` is +`x86_64` or `aarch64`): -- `dist/VapourBox-1.0.0-linux-.AppImage` — the primary download -- `dist/VapourBox-1.0.0-linux-.AppImage.zsync` — its delta-update index; must be uploaded beside it +- `dist/VapourBox-1.0.0-.AppImage` — the primary download +- `dist/VapourBox-1.0.0-.AppImage.zsync` — its delta-update index; must be uploaded beside it +- `dist/VapourBox-1.0.0-linux-.AppImage.zsync` — a copy of that index under the name 1.2.0 embedded, so a 1.2.0 AppImage can still update; upload it too - `dist/VapourBox-1.0.0-linux-.tar.gz` — the fallback The AppImage step needs `zsyncmake` (`sudo apt install zsync`) and a host of the target architecture; it downloads a pinned, checksummed `appimagetool` into `dist/.tools/` (or uses `$APPIMAGETOOL`). `--skip-appimage` builds the tarball alone. +With `desktop-file-validate` and `appstreamcli` installed (`sudo apt install +desktop-file-utils appstream`) it also validates the desktop entry and the +AppStream metadata (`packaging/linux/app.vapourbox.VapourBox.appdata.xml`). + +`--bundle-deps DIR` packs the dependency zips into the AppImage so its first +launch needs no network; release builds always pass it. `DIR` must hold the +release assets for the deps version `app/assets/deps-version.json` names, each +with its `.sha256.json`: `VapourBox-deps--linux-.zip`, and on x64 also +`VapourBox-deps--linux-x64-v2-delta.zip`. Without the flag the AppImage +downloads its dependencies on first launch, as the tarball always does. ### macOS ```bash diff --git a/docs/ENGINEERING_NOTES.md b/docs/ENGINEERING_NOTES.md index 6cec031..108f649 100644 --- a/docs/ENGINEERING_NOTES.md +++ b/docs/ENGINEERING_NOTES.md @@ -384,6 +384,46 @@ Note the version parser accepts `n9.0.1` and `9.0.1` and deliberately **rejects a `master` build** (`N-125978-...`), so reverting any platform to an unpinned master URL is a red build rather than a silent regression. +### The v2 tier became a delta, and the AppImage started carrying its deps (2026-10-09, deps 1.13.0) + +**What prompted it.** The AppImage catalog (appimage.github.io) picked up the +1.2.0 AppImage on its own and its test failed +([PR #9839](https://github.com/AppImage/appimage.github.io/pull/9839)). The +catalog launches each AppImage under `firejail --net=none`, screenshots it and +OCRs the screenshot for error text. VapourBox started fine, tried to download +its deps, failed DNS, and showed "Installation Failed" — which the OCR read. +Two warnings rode along: "linux" in the file name, and no AppStream metadata. + +**Why not just bundle the two existing zips.** The linux-x64 bundle is 236 MB +and so was its v2 twin; an AppImage that works on any x64 CPU would have +carried both, ~470 MB, for a difference that on Linux is one file. The tiers +were already required to list identical plugins +(`deps-expected-plugins.json`), and the tier blocks in `download-deps-*` only +touch zsmooth (and MVTools on macOS), so v2 as "v3 plus a few files" was +already true; only the packaging said otherwise. + +**The delta is declared, not diffed.** The first idea was to build both tiers +and publish whatever differs. That fails on contact: two builds of the same +sources on two runners are not byte-identical, so the diff is most of the +bundle. `_tierFiles` names the files instead, and `make-deps-delta.py` checks +the two things a diff would have told us that still matter — the builds hold +the same set of paths (so overwriting is enough), and each tier file really +does differ (so the tier switch did something). + +**What is unverified as of writing.** Everything Linux- and Windows-side was +written on macOS: the `delta-x64` jobs, `install-deps-bundle.sh`, the bundling +and validation in `package-linux.sh`, and the bridge `.zsync`. In particular: + +- that the composed macOS v2 tree runs — v2 MVTools is built against the v2 + build's own tree, and is now dropped onto the v3 bundle's `libfftw3f`; +- that `appstreamcli` on ubuntu-22.04 (0.15) accepts the appdata file; +- that AppImageUpdate, started from a 1.2.0 AppImage, follows the old-name + `.zsync` to the renamed file. zsync's `URL:` header is relative, so it should; + it has not been run. + +The first `build-deps-*` and `build-linux.yml` runs are the test of the first +two. The third needs a Linux machine and a 1.2.0 AppImage. + ### x86 bundles split into CPU tiers (issue #92, 2026-09-25) **The report.** A Mac Pro 5,1 (Xeon X5690, Westmere — SSE4.2, no AVX at all) diff --git a/packaging/linux/app.vapourbox.VapourBox.appdata.xml b/packaging/linux/app.vapourbox.VapourBox.appdata.xml new file mode 100644 index 0000000..3b1383f --- /dev/null +++ b/packaging/linux/app.vapourbox.VapourBox.appdata.xml @@ -0,0 +1,84 @@ + + + + app.vapourbox.VapourBox + app.vapourbox.VapourBox.desktop + + VapourBox + Deinterlace, restore and convert video with VapourSynth and QTGMC + Stuart Cameron + + CC0-1.0 + GPL-3.0-or-later + + +

+ VapourBox is a desktop app for archiving and converting video: tape + captures, camcorder footage, film scans, DVDs, and the digital files you + already have. It runs VapourSynth, QTGMC and FFmpeg, the tools the video + restoration community already uses, without requiring you to write a + script. +

+
    +
  • Converts DV, AVCHD, MPEG-2, MXF and ordinary MP4, MKV, AVI and MOV files, DVD folders and discs
    • +
  • Deinterlaces with QTGMC and reverses telecine, after detecting how the source is stored
    • +
  • Optional cleanup filters for tape noise, colour bleed, film dust and scratches, ghosting and more
    • +
  • Presets named after the source: VHS, DV camcorder tape, DVD and broadcast, anime, 8mm and Super 8 film
    • +
  • A side-by-side preview of the source and the processed result, and a queue for batches
    • +
  • Encodes to H.264, HEVC, ProRes and lossless formats, with hardware encoders where available
    • +
  • Transcribes speech to subtitles
    • +
+

+ The AppImage includes the processing components, so it works without a + network connection. It needs glibc 2.39 or newer (Ubuntu 24.04, + Debian 13). +

+ + + + AudioVideo + Video + + + deinterlace + QTGMC + VapourSynth + VHS + DV + DVD + convert + + + https://github.com/StuartCameronCode/VapourBox + https://github.com/StuartCameronCode/VapourBox/issues + + + + An interlaced capture with the original and the deinterlaced result side by side + https://raw.githubusercontent.com/StuartCameronCode/VapourBox/v@VERSION@/docs/images/screenshot.png + + + + + + + + https://github.com/StuartCameronCode/VapourBox/releases/tag/v@VERSION@ + + +