diff --git a/.github/workflows/lame_pins.yml b/.github/workflows/lame_pins.yml new file mode 100644 index 0000000..2e9d39d --- /dev/null +++ b/.github/workflows/lame_pins.yml @@ -0,0 +1,56 @@ +name: LAME Pin Update + +# Keeps assets/lame-pins.json in step with upstream LAME packages. The app reads +# that file from master, so merging the pull request this opens is what updates +# installed copies. It is never merged automatically: a changed hash is exactly +# what a tampered upstream would look like, so a person confirms it first. + +on: + schedule: + - cron: "0 5 * * 1" + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +jobs: + update-pins: + runs-on: ubuntu-24.04 + + steps: + - name: Checkout + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + ref: master + + - name: Refresh pins + run: python3 tools/update_lame_pins.py | tee pin-changes.txt + + - name: Open pull request + env: + GH_TOKEN: ${{ github.token }} + run: | + if git diff --quiet -- assets/lame-pins.json; then + echo "Pins are current." + exit 0 + fi + branch=chore/lame-pins + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git switch -C "$branch" + git add assets/lame-pins.json + git commit -m "chore(util): refresh LAME download pins" + git push --force origin "$branch" + { + echo "Upstream LAME packages changed. Merging this updates the pins every installed copy uses." + echo + echo "Check each change against its source before merging. Windows (zip) changes have no independent confirmation." + echo + echo '```' + cat pin-changes.txt + echo '```' + } > pr-body.md + gh pr view "$branch" --json state -q .state 2>/dev/null | grep -q OPEN \ + && gh pr edit "$branch" --body-file pr-body.md \ + || gh pr create --base master --head "$branch" --title "chore(util): refresh LAME download pins" --body-file pr-body.md diff --git a/CMakeLists.txt b/CMakeLists.txt index 63080ad..305dcc3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -553,6 +553,7 @@ if(BUILD_TESTING) tests/unit/ControlDeckTests.cpp tests/unit/UiDecisionTests.cpp tests/unit/SessionSerializationTests.cpp + tests/unit/LameDownloaderTests.cpp src/app/ui/SessionManager.cpp tests/unit/AudioIoTests.cpp tests/unit/AnalysisTests.cpp diff --git a/assets/lame-pins.json b/assets/lame-pins.json new file mode 100644 index 0000000..1a6b9f3 --- /dev/null +++ b/assets/lame-pins.json @@ -0,0 +1,41 @@ +{ + "schema": 1, + "sources": [ + { + "platform": "linux-arm", + "type": "deb", + "url": "https://deb.debian.org/debian/pool/main/l/lame/lame_3.100-6+b3_armhf.deb", + "sha256": "b7c5a96b803db07f67b74b7611240979797fb6eab0b4705e23eb0ce639b9078e" + }, + { + "platform": "linux-arm64", + "type": "deb", + "url": "https://deb.debian.org/debian/pool/main/l/lame/lame_3.100-6+b3_arm64.deb", + "sha256": "4c6c6ee693633c846de685902641dbda18ef98f73a136ebef7c09cf6ee683bcd" + }, + { + "platform": "linux-x64", + "type": "deb", + "url": "https://deb.debian.org/debian/pool/main/l/lame/lame_3.100-6+b3_amd64.deb", + "sha256": "feceeb296f9df9340f6e2ce48decce73c9cd6c70e7fe959495eb89ee59bc8d47" + }, + { + "platform": "win32-arm64", + "type": "zip", + "url": "https://www.rarewares.org/files/mp3/lame3.100.1-x64.zip", + "sha256": "9a9c815203316e5203847e93100c6acf0d5d7a5be7744c9018825ded037052e7" + }, + { + "platform": "win32-ia32", + "type": "zip", + "url": "https://www.rarewares.org/files/mp3/lame3.100.1-win32.zip", + "sha256": "2518e1138953c235fb2bfcefbc38883dd04538d6ae0a19692562576ba37bafec" + }, + { + "platform": "win32-x64", + "type": "zip", + "url": "https://www.rarewares.org/files/mp3/lame3.100.1-x64.zip", + "sha256": "9a9c815203316e5203847e93100c6acf0d5d7a5be7744c9018825ded037052e7" + } + ] +} diff --git a/src/util/LameDownloader.cpp b/src/util/LameDownloader.cpp index 7ed0287..4e68a31 100644 --- a/src/util/LameDownloader.cpp +++ b/src/util/LameDownloader.cpp @@ -19,6 +19,7 @@ #include #include "util/FileUtils.h" +#include "util/Sha256.h" #include "util/StringUtils.h" namespace automix::util { @@ -38,11 +39,28 @@ enum class SourceType { Ghcr, }; +// Every source is an archive that ends up executed, so each one carries the +// SHA-256 of the exact file it must deliver. A source with no hash is refused. +// For Ghcr the hash is also the address: Homebrew bottles are stored as blobs +// named by their own SHA-256, so the pinned bottle is fetched directly. struct DownloadSource { SourceType type = SourceType::Zip; std::string url; - std::string ghcrOs; - std::string ghcrArch; + std::string sha256; +}; + +const std::string kGhcrBlobBaseUrl = "https://ghcr.io/v2/homebrew/core/lame/blobs/sha256:"; + +// The pin list on the default branch, kept current by tools/update_lame_pins.py. +// It lets installed copies follow upstream LAME updates without a release. It +// can only name files on the hosts below, so changing it is not enough to make +// the app run something else: the named host must also serve the matching file. +constexpr const char* kPinManifestUrl = + "https://raw.githubusercontent.com/soficis/AutoMixMaster/master/assets/lame-pins.json"; +constexpr int kPinManifestTimeoutMs = 8000; +constexpr const char* kAllowedDownloadPrefixes[] = { + "https://www.rarewares.org/files/mp3/", + "https://deb.debian.org/debian/pool/main/l/lame/", }; struct TempDirectory { @@ -152,65 +170,121 @@ std::string platformKey() { #endif } -std::vector platformSources() { - const auto version = readEnvironment("AUTOMIX_LAME_VERSION").value_or(kDefaultLameVersion); - if (const auto manualUrl = readEnvironment("AUTOMIX_LAME_DOWNLOAD_URL"); manualUrl.has_value()) { - const auto lower = toLower(*manualUrl); - if (lower.ends_with(".zip")) { - return {{SourceType::Zip, *manualUrl, "", ""}}; - } - if (lower.ends_with(".deb")) { - return {{SourceType::Debian, *manualUrl, "", ""}}; - } - return {{SourceType::DirectBinary, *manualUrl, "", ""}}; - } - - const auto key = platformKey(); - if (key == "win32-x64") { +// Sources for `key` at `version`. The hashes are only valid for the default +// version; platformSources() replaces them when the version is overridden. +// Hash origins: Debian's package index for the .deb files, the Homebrew bottle +// index for the Ghcr blobs. rarewares.org publishes no hashes, so the two ZIP +// pins were taken from the files as served on 2026-10-05. +// Only bottles whose encoder runs on its own are usable: the 3.100 macOS ones +// do. Linux bottles never do (their loader path is a Homebrew placeholder), and +// neither do the 4.0 macOS ones (they need Homebrew's libmpg123). +std::vector sourcesForPlatform(const std::string& key, const std::string& version) { + const std::string rarewares = "https://www.rarewares.org/files/mp3/lame" + version; + const std::string debian = "https://deb.debian.org/debian/pool/main/l/lame/lame_" + version + "-6_"; + if (key == "win32-x64" || key == "win32-arm64") { + // There is no ARM64 build to pin; Windows on ARM runs the x64 one. return { - {SourceType::Zip, "https://www.rarewares.org/files/mp3/lame" + version + ".1-x64.zip", "", ""}, + {SourceType::Zip, rarewares + ".1-x64.zip", "9a9c815203316e5203847e93100c6acf0d5d7a5be7744c9018825ded037052e7"}, }; } if (key == "win32-ia32") { return { - {SourceType::Zip, "https://www.rarewares.org/files/mp3/lame" + version + ".1-win32.zip", "", ""}, - }; - } - if (key == "win32-arm64") { - return { - {SourceType::Zip, "https://www.rarewares.org/files/mp3/LAME-" + version + "-Win-ARM64.zip", "", ""}, + {SourceType::Zip, rarewares + ".1-win32.zip", "2518e1138953c235fb2bfcefbc38883dd04538d6ae0a19692562576ba37bafec"}, }; } if (key == "linux-x64") { return { - {SourceType::Debian, "https://deb.debian.org/debian/pool/main/l/lame/lame_" + version + "-6_amd64.deb", "", ""}, - {SourceType::Ghcr, "", "linux", "amd64"}, + {SourceType::Debian, debian + "amd64.deb", "786ba06d2f222661e1f09b610de7b18c60f411a373d4fd3f595ec890f062089e"}, }; } if (key == "linux-arm64") { return { - {SourceType::Debian, "https://deb.debian.org/debian/pool/main/l/lame/lame_" + version + "-6_arm64.deb", "", ""}, - {SourceType::Ghcr, "", "linux", "arm64"}, + {SourceType::Debian, debian + "arm64.deb", "aba5023ffde46709e4bccc9e1c10142a7d77f2884d2a9af84cab6a28f8792bd2"}, }; } if (key == "linux-arm") { return { - {SourceType::Debian, "https://deb.debian.org/debian/pool/main/l/lame/lame_" + version + "-6_armhf.deb", "", ""}, + {SourceType::Debian, debian + "armhf.deb", "f77e72665a30bae6d83ca3719845309a2db12b1adf27c422415c4a12930ea76b"}, }; } if (key == "darwin-x64") { return { - {SourceType::Ghcr, "", "darwin", "amd64"}, + {SourceType::Ghcr, "", "737751faa513a68ac2499bb5cc607bc366e15dab8ff3bff5443567a455af5c3f"}, }; } if (key == "darwin-arm64") { return { - {SourceType::Ghcr, "", "darwin", "arm64"}, + {SourceType::Ghcr, "", "2ff2c6ad3cfd26e1ba53230631e2f04734a4638c344cce50ff0b8fc36b45c403"}, }; } return {}; } +std::optional fetchJson(const std::string& url, + const std::string& extraHeaders, + std::string* detail, + int timeoutMs = 45000); + +// Sources for this platform from the published pin list, or none when it cannot +// be fetched or does not validate. +std::vector publishedSources() { + const auto manifest = fetchJson(kPinManifestUrl, "", nullptr, kPinManifestTimeoutMs); + if (!manifest.has_value()) { + return {}; + } + + std::vector sources; + for (const auto& pin : LameDownloader::parsePinManifest(manifest->dump(), nullptr)) { + if (pin.platformKey != platformKey()) { + continue; + } + const auto type = pin.type == "zip" ? SourceType::Zip : pin.type == "deb" ? SourceType::Debian : SourceType::Ghcr; + sources.push_back({type, type == SourceType::Ghcr ? std::string() : pin.url, pin.sha256}); + } + return sources; +} + +// AUTOMIX_LAME_DOWNLOAD_URL and AUTOMIX_LAME_VERSION point at files the built-in +// hashes do not cover, so they only work together with AUTOMIX_LAME_DOWNLOAD_SHA256. +// With `allowPublished`, the published pin list is tried first and the built-in +// pins stay as the fallback (offline, list unreachable, or a newer build that +// does not run on this machine). +std::vector platformSources(const bool allowPublished = false) { + const auto manualSha = toLower(readEnvironment("AUTOMIX_LAME_DOWNLOAD_SHA256").value_or("")); + if (const auto manualUrl = readEnvironment("AUTOMIX_LAME_DOWNLOAD_URL"); manualUrl.has_value()) { + const auto lower = toLower(*manualUrl); + if (lower.ends_with(".zip")) { + return {{SourceType::Zip, *manualUrl, manualSha}}; + } + if (lower.ends_with(".deb")) { + return {{SourceType::Debian, *manualUrl, manualSha}}; + } + return {{SourceType::DirectBinary, *manualUrl, manualSha}}; + } + + const auto version = readEnvironment("AUTOMIX_LAME_VERSION").value_or(kDefaultLameVersion); + auto sources = sourcesForPlatform(platformKey(), version); + if (version != kDefaultLameVersion) { + for (auto& source : sources) { + source.sha256 = manualSha; + } + return sources; + } + + if (allowPublished && !flagEnabled("AUTOMIX_LAME_SKIP_PIN_UPDATE")) { + auto published = publishedSources(); + for (auto& source : sources) { + const bool known = std::any_of(published.begin(), published.end(), + [&source](const DownloadSource& other) { return other.sha256 == source.sha256; }); + if (!known) { + published.push_back(std::move(source)); + } + } + return published; + } + return sources; +} + bool runProcess(const juce::StringArray& command, const int timeoutMs, std::string* processOutput, @@ -306,13 +380,34 @@ bool downloadToFile(const std::string& url, return true; } +// Downloads a source and refuses it unless it matches its hash. Nothing from the +// file is extracted or run before this returns true. +bool downloadVerified(const std::string& url, + const std::string& sha256, + const std::filesystem::path& outputPath, + const std::string& extraHeaders, + std::string* detail) { + if (!isSha256Hex(sha256)) { + if (detail != nullptr) { + *detail = "No SHA-256 is pinned for this download (set AUTOMIX_LAME_DOWNLOAD_SHA256 when overriding " + "the URL or version): " + url; + } + return false; + } + if (!downloadToFile(url, outputPath, extraHeaders, detail)) { + return false; + } + return LameDownloader::verifyDownload(outputPath, sha256, detail); +} + std::optional fetchJson(const std::string& url, const std::string& extraHeaders, - std::string* detail) { + std::string* detail, + const int timeoutMs) { int statusCode = 0; const auto baseOptions = juce::URL::InputStreamOptions(juce::URL::ParameterHandling::inAddress) - .withConnectionTimeoutMs(45000) + .withConnectionTimeoutMs(timeoutMs) .withNumRedirectsToFollow(8) .withStatusCode(&statusCode); const auto input = juce::URL(url).createInputStream( @@ -561,7 +656,7 @@ bool copyBinaryToCache(const std::filesystem::path& source, const std::filesyste bool installFromZip(const DownloadSource& source, const std::filesystem::path& targetBinary, std::string* detail) { TempDirectory temp("automix_lame_zip"); const auto archivePath = temp.path / "lame.zip"; - if (!downloadToFile(source.url, archivePath, "", detail)) { + if (!downloadVerified(source.url, source.sha256, archivePath, "", detail)) { return false; } @@ -630,7 +725,7 @@ bool installFromDebian(const DownloadSource& source, const std::filesystem::path #else TempDirectory temp("automix_lame_deb"); const auto debPath = temp.path / "lame.deb"; - if (!downloadToFile(source.url, debPath, "", detail)) { + if (!downloadVerified(source.url, source.sha256, debPath, "", detail)) { return false; } @@ -681,8 +776,6 @@ bool installFromDebian(const DownloadSource& source, const std::filesystem::path } bool installFromGhcr(const DownloadSource& source, const std::filesystem::path& targetBinary, std::string* detail) { - const auto version = readEnvironment("AUTOMIX_LAME_VERSION").value_or(kDefaultLameVersion); - const auto tokenJson = fetchJson("https://ghcr.io/token?service=ghcr.io&scope=repository:homebrew/core/lame:pull", "", detail); if (!tokenJson.has_value() || !tokenJson->contains("token")) { if (detail != nullptr && detail->empty()) { @@ -699,67 +792,13 @@ bool installFromGhcr(const DownloadSource& source, const std::filesystem::path& return false; } - const std::string authHeader = "Authorization: Bearer " + token + "\n"; - const auto manifestList = fetchJson( - "https://ghcr.io/v2/homebrew/core/lame/manifests/" + version, - authHeader + "Accept: application/vnd.oci.image.index.v1+json\n", - detail); - if (!manifestList.has_value() || !manifestList->contains("manifests")) { - if (detail != nullptr && detail->empty()) { - *detail = "Failed to fetch GHCR manifest list."; - } - return false; - } - - std::string manifestDigest; - for (const auto& manifest : (*manifestList)["manifests"]) { - const auto platform = manifest.value("platform", nlohmann::json::object()); - if (platform.value("os", "") == source.ghcrOs && platform.value("architecture", "") == source.ghcrArch) { - manifestDigest = manifest.value("digest", ""); - break; - } - } - if (manifestDigest.empty()) { - if (detail != nullptr) { - *detail = "No GHCR manifest found for " + source.ghcrOs + "/" + source.ghcrArch; - } - return false; - } - - const auto manifest = fetchJson( - "https://ghcr.io/v2/homebrew/core/lame/manifests/" + manifestDigest, - authHeader + "Accept: application/vnd.oci.image.manifest.v1+json\n", - detail); - if (!manifest.has_value() || !manifest->contains("layers")) { - if (detail != nullptr && detail->empty()) { - *detail = "Failed to fetch GHCR image manifest."; - } - return false; - } - - std::string layerDigest; - std::string mediaType; - for (const auto& layer : (*manifest)["layers"]) { - mediaType = layer.value("mediaType", ""); - if (mediaType.find("tar") != std::string::npos) { - layerDigest = layer.value("digest", ""); - break; - } - } - if (layerDigest.empty()) { - if (detail != nullptr) { - *detail = "No tar layer found in GHCR image manifest."; - } - return false; - } - TempDirectory temp("automix_lame_ghcr"); - const bool gzipLayer = mediaType.find("gzip") != std::string::npos; - const auto layerPath = temp.path / (gzipLayer ? "layer.tar.gz" : "layer.tar"); - if (!downloadToFile("https://ghcr.io/v2/homebrew/core/lame/blobs/" + layerDigest, - layerPath, - authHeader + "Accept: application/octet-stream\n", - detail)) { + const auto layerPath = temp.path / "layer.tar.gz"; + if (!downloadVerified(kGhcrBlobBaseUrl + source.sha256, + source.sha256, + layerPath, + "Authorization: Bearer " + token + "\nAccept: application/octet-stream\n", + detail)) { return false; } @@ -773,7 +812,7 @@ bool installFromGhcr(const DownloadSource& source, const std::filesystem::path& return false; } - if (!extractTarArchive(layerPath, extractDir, gzipLayer ? "z" : "", "", detail)) { + if (!extractTarArchive(layerPath, extractDir, "z", "", detail)) { return false; } @@ -795,10 +834,114 @@ std::filesystem::path internalCacheBinaryPath() { return appData / "AutoMixMaster" / "codecs" / "lame" / key / binaryName(); } +// The cached binary is only trusted while this file, written after a verified +// install, still holds the binary's own hash. A binary cached before downloads +// were verified has no such file and is fetched again. +std::filesystem::path cacheMarkerPath(const std::filesystem::path& binary) { + return std::filesystem::path(binary.string() + ".sha256"); +} + +void writeCacheMarker(const std::filesystem::path& binary) { + std::ofstream marker(cacheMarkerPath(binary), std::ios::binary | std::ios::trunc); + marker << fileSha256(binary); +} + } // namespace std::filesystem::path LameDownloader::cacheBinaryPath() { return internalCacheBinaryPath(); } +bool LameDownloader::cachedBinaryIsVerified() { + const auto binary = cacheBinaryPath(); + if (!isRegularFile(binary)) { + return false; + } + std::ifstream marker(cacheMarkerPath(binary), std::ios::binary); + std::string recorded; + marker >> recorded; + return isSha256Hex(recorded) && recorded == fileSha256(binary); +} + +std::vector LameDownloader::pinnedSources() { + std::vector pins; + for (const char* key : {"win32-x64", "win32-ia32", "win32-arm64", "linux-x64", "linux-arm64", "linux-arm", + "darwin-x64", "darwin-arm64"}) { + for (const auto& source : sourcesForPlatform(key, kDefaultLameVersion)) { + const char* type = source.type == SourceType::Zip ? "zip" : source.type == SourceType::Debian ? "deb" : "ghcr"; + pins.push_back({key, type, source.type == SourceType::Ghcr ? kGhcrBlobBaseUrl + source.sha256 : source.url, + source.sha256}); + } + } + return pins; +} + +std::vector LameDownloader::parsePinManifest(const std::string& jsonText, + std::string* detail) { + const auto reject = [detail](const std::string& reason) { + if (detail != nullptr) { + *detail = "LAME pin list rejected: " + reason; + } + return std::vector{}; + }; + + const auto json = nlohmann::json::parse(jsonText, nullptr, false); + if (!json.is_object() || json.value("schema", 0) != 1 || !json.contains("sources") || !json["sources"].is_array()) { + return reject("not a schema 1 pin list."); + } + + std::vector pins; + for (const auto& entry : json["sources"]) { + if (!entry.is_object() || !entry.value("platform", nlohmann::json()).is_string() || + !entry.value("type", nlohmann::json()).is_string() || !entry.value("sha256", nlohmann::json()).is_string() || + !entry.value("url", nlohmann::json("")).is_string()) { + return reject("an entry has missing or non-text fields."); + } + + PinnedSource pin; + pin.platformKey = entry["platform"].get(); + pin.type = entry["type"].get(); + pin.sha256 = toLower(entry["sha256"].get()); + if (!isSha256Hex(pin.sha256)) { + return reject("an entry has no valid SHA-256."); + } + + if (pin.type == "ghcr") { + pin.url = kGhcrBlobBaseUrl + pin.sha256; + } else if (pin.type == "zip" || pin.type == "deb") { + pin.url = entry.value("url", ""); + const bool allowedHost = std::any_of(std::begin(kAllowedDownloadPrefixes), std::end(kAllowedDownloadPrefixes), + [&pin](const char* prefix) { return pin.url.rfind(prefix, 0) == 0; }); + const auto fileName = pin.url.substr(pin.url.find_last_of('/') + 1); + const bool plainFile = !fileName.empty() && fileName.ends_with("." + pin.type) && + pin.url.find_first_of("?#\\%") == std::string::npos && + pin.url.find("..") == std::string::npos; + if (!allowedHost || !plainFile) { + return reject("an entry points outside the known download locations: " + pin.url); + } + } else { + return reject("unknown source type '" + pin.type + "'."); + } + pins.push_back(std::move(pin)); + } + return pins; +} + +bool LameDownloader::verifyDownload(const std::filesystem::path& file, + const std::string& expectedSha256, + std::string* detail) { + const auto actual = fileSha256(file); + if (isSha256Hex(expectedSha256) && actual == toLower(expectedSha256)) { + return true; + } + + std::error_code error; + std::filesystem::remove(file, error); + if (detail != nullptr) { + *detail = "SHA-256 mismatch for " + file.filename().string() + " (expected " + expectedSha256 + ", got " + + (actual.empty() ? "unreadable file" : actual) + "); the download was discarded."; + } + return false; +} + bool LameDownloader::isSupportedOnCurrentPlatform() { return !platformSources().empty(); } LameDownloader::DownloadResult LameDownloader::ensureAvailable(const bool forceDownload) { @@ -807,7 +950,7 @@ LameDownloader::DownloadResult LameDownloader::ensureAvailable(const bool forceD DownloadResult result; const auto targetBinary = cacheBinaryPath(); - if (!forceDownload && !flagEnabled("AUTOMIX_LAME_FORCE_DOWNLOAD") && isRegularFile(targetBinary)) { + if (!forceDownload && !flagEnabled("AUTOMIX_LAME_FORCE_DOWNLOAD") && cachedBinaryIsVerified()) { std::string detail; if (ensureExecutable(targetBinary, &detail)) { result.success = true; @@ -822,7 +965,7 @@ LameDownloader::DownloadResult LameDownloader::ensureAvailable(const bool forceD return result; } - const auto sources = platformSources(); + const auto sources = platformSources(true); if (sources.empty()) { result.detail = "No fallback LAME downloader source configured for this platform."; return result; @@ -837,7 +980,7 @@ LameDownloader::DownloadResult LameDownloader::ensureAvailable(const bool forceD case SourceType::DirectBinary: { TempDirectory temp("automix_lame_direct"); const auto downloadedPath = temp.path / binaryName(); - if (downloadToFile(source.url, downloadedPath, "", &attemptDetail)) { + if (downloadVerified(source.url, source.sha256, downloadedPath, "", &attemptDetail)) { installed = copyBinaryToCache(downloadedPath, targetBinary, &attemptDetail); } break; @@ -854,6 +997,7 @@ LameDownloader::DownloadResult LameDownloader::ensureAvailable(const bool forceD } if (installed) { + writeCacheMarker(targetBinary); result.success = true; result.executablePath = targetBinary; result.detail = "Downloaded fallback LAME binary to " + targetBinary.string(); diff --git a/src/util/LameDownloader.h b/src/util/LameDownloader.h index dae5314..2a36db6 100644 --- a/src/util/LameDownloader.h +++ b/src/util/LameDownloader.h @@ -2,6 +2,7 @@ #include #include +#include namespace automix::util { @@ -14,7 +15,23 @@ class LameDownloader { std::string detail; }; + struct PinnedSource { + std::string platformKey; + std::string type; // "zip", "deb" or "ghcr" + std::string url; + std::string sha256; + }; + static std::filesystem::path cacheBinaryPath(); + /// True when the cached binary exists and is the one a verified install left there. + static bool cachedBinaryIsVerified(); + /// Every built-in download with the SHA-256 it must match, for all platforms. + static std::vector pinnedSources(); + /// Parses the published pin list (assets/lame-pins.json). Any entry that is malformed or + /// points outside the known download hosts rejects the whole list (returns empty). + static std::vector parsePinManifest(const std::string& jsonText, std::string* detail); + /// Checks a downloaded file against its pinned SHA-256; a mismatch deletes the file. + static bool verifyDownload(const std::filesystem::path& file, const std::string& expectedSha256, std::string* detail); static bool isSupportedOnCurrentPlatform(); static DownloadResult ensureAvailable(bool forceDownload = false); }; diff --git a/src/util/WavWriter.cpp b/src/util/WavWriter.cpp index 765569b..cfdd6ff 100644 --- a/src/util/WavWriter.cpp +++ b/src/util/WavWriter.cpp @@ -252,8 +252,8 @@ std::optional resolveBundledLameExecutable() { } std::optional findLameExecutable() { - if (const auto downloaded = LameDownloader::cacheBinaryPath(); isRegularFile(downloaded)) { - return downloaded; + if (LameDownloader::cachedBinaryIsVerified()) { + return LameDownloader::cacheBinaryPath(); } if (const auto bundled = resolveBundledLameExecutable(); bundled.has_value()) { diff --git a/tests/unit/LameDownloaderTests.cpp b/tests/unit/LameDownloaderTests.cpp new file mode 100644 index 0000000..3eee753 --- /dev/null +++ b/tests/unit/LameDownloaderTests.cpp @@ -0,0 +1,110 @@ +#include +#include +#include +#include +#include + +#include + +#include "util/LameDownloader.h" +#include "util/Sha256.h" + +using automix::util::LameDownloader; + +TEST_CASE("Every LAME download source is pinned to a SHA-256 over HTTPS", "[util][lame]") { + const auto pins = LameDownloader::pinnedSources(); + REQUIRE(!pins.empty()); + + std::set platforms; + for (const auto& pin : pins) { + INFO(pin.platformKey << " " << pin.url); + CHECK(automix::util::isSha256Hex(pin.sha256)); + CHECK(pin.url.rfind("https://", 0) == 0); + platforms.insert(pin.platformKey); + } + for (const char* key : {"win32-x64", "win32-ia32", "win32-arm64", "linux-x64", "linux-arm64", "linux-arm", + "darwin-x64", "darwin-arm64"}) { + INFO(key); + CHECK(platforms.count(key) == 1); + } +} + +TEST_CASE("The published LAME pin list in the repo is valid and names only known platforms", "[util][lame]") { + std::ifstream file(std::filesystem::path(AUTOMIX_SOURCE_DIR) / "assets" / "lame-pins.json", std::ios::binary); + REQUIRE(file.is_open()); + const std::string text((std::istreambuf_iterator(file)), std::istreambuf_iterator()); + + std::string detail; + const auto pins = LameDownloader::parsePinManifest(text, &detail); + INFO(detail); + REQUIRE(!pins.empty()); + + // A platform may be absent: the app then uses its built-in pins. + std::set known; + for (const auto& builtIn : LameDownloader::pinnedSources()) { + known.insert(builtIn.platformKey); + } + for (const auto& pin : pins) { + INFO(pin.platformKey); + CHECK(known.count(pin.platformKey) == 1); + } +} + +TEST_CASE("A LAME pin list that could redirect the download is rejected whole", "[util][lame]") { + const std::string sha(64, 'a'); + const auto list = [](const std::string& entry) { return R"({"schema":1,"sources":[)" + entry + "]}"; }; + const auto zip = [&sha](const std::string& url) { + return R"({"platform":"win32-x64","type":"zip","url":")" + url + R"(","sha256":")" + sha + R"("})"; + }; + const std::string good = zip("https://www.rarewares.org/files/mp3/lame4.0-x64.zip"); + + REQUIRE(LameDownloader::parsePinManifest(list(good), nullptr).size() == 1); + CHECK(LameDownloader::parsePinManifest(list(R"({"platform":"darwin-arm64","type":"ghcr","sha256":")" + sha + R"("})"), + nullptr) + .size() == 1); + + std::string detail; + for (const auto& bad : { + zip("https://evil.example/lame.zip"), + zip("http://www.rarewares.org/files/mp3/lame.zip"), + zip("https://www.rarewares.org.evil.example/files/mp3/lame.zip"), + zip("https://www.rarewares.org/files/mp3/../../x/lame.zip"), + zip("https://www.rarewares.org/files/mp3/lame.exe"), + zip("https://www.rarewares.org/files/mp3/lame.zip?x=.zip"), + std::string(R"({"platform":"win32-x64","type":"zip","url":"https://www.rarewares.org/files/mp3/l.zip","sha256":"abc"})"), + std::string(R"({"platform":"win32-x64","type":"exe","url":"https://www.rarewares.org/files/mp3/l.zip","sha256":")") + + sha + R"("})", + }) { + INFO(bad); + // One bad entry poisons the list even next to a good one. + CHECK(LameDownloader::parsePinManifest(list(good + "," + bad), &detail).empty()); + CHECK(!detail.empty()); + } + CHECK(LameDownloader::parsePinManifest("not json", &detail).empty()); + CHECK(LameDownloader::parsePinManifest(R"({"schema":2,"sources":[]})", &detail).empty()); +} + +TEST_CASE("A LAME download that does not match its pin is rejected and deleted", "[util][lame]") { + const auto dir = std::filesystem::temp_directory_path() / "automix_lame_pin_test"; + std::filesystem::create_directories(dir); + const auto file = dir / "lame.zip"; + const auto write = [&file]() { std::ofstream(file, std::ios::binary) << "not really lame"; }; + + write(); + const auto actual = automix::util::fileSha256(file); + std::string detail; + CHECK(LameDownloader::verifyDownload(file, actual, &detail)); + CHECK(std::filesystem::exists(file)); + + std::string wrong = actual; + wrong[0] = wrong[0] == '0' ? '1' : '0'; + CHECK_FALSE(LameDownloader::verifyDownload(file, wrong, &detail)); + CHECK(detail.find("mismatch") != std::string::npos); + CHECK_FALSE(std::filesystem::exists(file)); + + write(); + CHECK_FALSE(LameDownloader::verifyDownload(file, "", &detail)); + CHECK_FALSE(std::filesystem::exists(file)); + + std::filesystem::remove_all(dir); +} diff --git a/tools/update_lame_pins.py b/tools/update_lame_pins.py new file mode 100644 index 0000000..2d15ca5 --- /dev/null +++ b/tools/update_lame_pins.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""Regenerate assets/lame-pins.json, the list of LAME downloads the app trusts. + +The app fetches that file from the default branch before downloading an MP3 +encoder, so merging a change to it updates every installed copy without a +release. Run by .github/workflows/lame_pins.yml, which opens a pull request +when the output changes; a person reviews it before it is merged. + +Where each hash comes from: + - Debian: the SHA256 field of the `lame` entry in Debian stable's package index. + - Homebrew (macOS): the bottle digests in the GHCR image index for the current + version. Each candidate bottle is downloaded, checked against its digest and + inspected: the app copies only bin/lame out of it, so a bottle whose encoder + still points at a Homebrew path cannot run and is not published. With no + usable bottle the platform is left out and the app uses its built-in pins. + Linux bottles are never usable this way, so Linux relies on Debian. + - rarewares.org (Windows): no index or published hash exists, so the known + files are downloaded and hashed. A changed hash there has no independent + confirmation and needs a careful look before merging. + +A source that cannot be refreshed keeps its current entry and is reported. +""" + +import hashlib +import io +import json +import lzma +import pathlib +import sys +import tarfile +import urllib.request + +MANIFEST = pathlib.Path(__file__).resolve().parent.parent / "assets" / "lame-pins.json" +GHCR = "https://ghcr.io/v2/homebrew/core/lame" +DEBIAN = "https://deb.debian.org/debian/" + +# platform key -> Debian architecture / Homebrew (os, architecture) / rarewares file +DEBIAN_ARCH = {"linux-x64": "amd64", "linux-arm64": "arm64", "linux-arm": "armhf"} +BOTTLE = { + "darwin-x64": ("darwin", "amd64"), + "darwin-arm64": ("darwin", "arm64"), +} +WINDOWS_ZIP = { + "win32-x64": "https://www.rarewares.org/files/mp3/lame3.100.1-x64.zip", + "win32-arm64": "https://www.rarewares.org/files/mp3/lame3.100.1-x64.zip", + "win32-ia32": "https://www.rarewares.org/files/mp3/lame3.100.1-win32.zip", +} + + +def fetch(url, headers=None): + request = urllib.request.Request(url, headers={"User-Agent": "automix-lame-pins", **(headers or {})}) + with urllib.request.urlopen(request, timeout=120) as response: + return response.read() + + +def debian_sources(): + sources = [] + for platform, arch in DEBIAN_ARCH.items(): + index = lzma.decompress(fetch(f"{DEBIAN}dists/stable/main/binary-{arch}/Packages.xz")).decode("utf-8") + stanza = next(s for s in index.split("\n\n") if s.startswith("Package: lame\n")) + fields = dict(line.split(": ", 1) for line in stanza.splitlines() if ": " in line and not line.startswith(" ")) + sources.append({"platform": platform, "type": "deb", "url": DEBIAN + fields["Filename"], "sha256": fields["SHA256"]}) + return sources + + +def os_version(manifest): + digits = "".join(c if c.isdigit() or c == "." else " " for c in manifest["platform"].get("os.version", "0")) + return tuple(int(part) for part in digits.split()[0].split(".") if part) if digits.split() else (0,) + + +def runs_standalone(digest, auth): + blob = fetch(f"{GHCR}/blobs/sha256:{digest}", auth) + if hashlib.sha256(blob).hexdigest() != digest: + raise ValueError(f"bottle {digest[:12]} does not match its digest") + with tarfile.open(fileobj=io.BytesIO(blob)) as bottle: + encoder = next(m for m in bottle.getmembers() if m.name.endswith("/bin/lame")) + return b"@@HOMEBREW" not in bottle.extractfile(encoder).read() + + +def homebrew_sources(): + version = json.loads(fetch("https://formulae.brew.sh/api/formula/lame.json"))["versions"]["stable"] + token = json.loads(fetch("https://ghcr.io/token?service=ghcr.io&scope=repository:homebrew/core/lame:pull"))["token"] + auth = {"Authorization": f"Bearer {token}"} + index = json.loads(fetch(f"{GHCR}/manifests/{version}", {**auth, "Accept": "application/vnd.oci.image.index.v1+json"})) + + sources = [] + for platform, (os_name, arch) in BOTTLE.items(): + bottles = [m for m in index["manifests"] + if m["platform"]["os"] == os_name and m["platform"]["architecture"] == arch] + # The bottle built for the oldest OS release runs on the widest range of machines. + for bottle in sorted(bottles, key=os_version): + digest = bottle["annotations"]["sh.brew.bottle.digest"] + if runs_standalone(digest, auth): + sources.append({"platform": platform, "type": "ghcr", "sha256": digest}) + break + else: + print(f"note: no LAME {version} bottle for {platform} runs on its own; the app keeps its built-in pin") + return sources + + +def windows_sources(current): + sources = [] + hashes = {} + for platform, default_url in WINDOWS_ZIP.items(): + url = next((s["url"] for s in current if s["platform"] == platform and s["type"] == "zip"), default_url) + if url not in hashes: + hashes[url] = hashlib.sha256(fetch(url)).hexdigest() + sources.append({"platform": platform, "type": "zip", "url": url, "sha256": hashes[url]}) + return sources + + +def main(): + current = json.loads(MANIFEST.read_text(encoding="utf-8"))["sources"] if MANIFEST.exists() else [] + sources = [] + failed = False + for name, kind, refresh in (("Debian", "deb", debian_sources), + ("Homebrew", "ghcr", homebrew_sources), + ("rarewares.org", "zip", lambda: windows_sources(current))): + try: + sources += refresh() + except Exception as error: # keep the entries we already trust + failed = True + print(f"warning: could not refresh {name} pins ({error}); keeping the current ones", file=sys.stderr) + sources += [s for s in current if s["type"] == kind] + + for old in current: + new = next((s for s in sources if (s["platform"], s["type"]) == (old["platform"], old["type"])), None) + if new is not None and new["sha256"] != old["sha256"]: + note = " -- NO independent confirmation, review before merging" if old["type"] == "zip" else "" + print(f"changed: {old['platform']} {old['type']} {old['sha256'][:12]} -> {new['sha256'][:12]}{note}") + + sources.sort(key=lambda s: (s["platform"], s["type"])) + MANIFEST.write_text(json.dumps({"schema": 1, "sources": sources}, indent=2) + "\n", encoding="utf-8", newline="\n") + return 1 if failed and not sources else 0 + + +if __name__ == "__main__": + sys.exit(main())