diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..7128de5 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,83 @@ +name: Release + +on: + push: + tags: ["v*"] + workflow_dispatch: + +permissions: + contents: write + +jobs: + release: + name: Build, sign, notarize + runs-on: macos-15 + steps: + - uses: actions/checkout@v4 + + - name: Install Rust toolchain + run: rustup toolchain install stable --profile minimal + + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + rust/target + key: ${{ runner.os }}-release-cargo-${{ hashFiles('rust/Cargo.lock') }} + + - name: Install create-dmg + run: brew install create-dmg + + # The Developer ID certificate is imported into a throwaway keychain so it + # never persists on the runner beyond this job. + - name: Import signing certificate + env: + MACOS_CERTIFICATE: ${{ secrets.MACOS_CERTIFICATE }} + MACOS_CERTIFICATE_PWD: ${{ secrets.MACOS_CERTIFICATE_PWD }} + KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }} + run: | + KEYCHAIN=$RUNNER_TEMP/build.keychain + echo "$MACOS_CERTIFICATE" | base64 --decode > $RUNNER_TEMP/cert.p12 + security create-keychain -p "$KEYCHAIN_PASSWORD" "$KEYCHAIN" + security set-keychain-settings -lut 21600 "$KEYCHAIN" + security unlock-keychain -p "$KEYCHAIN_PASSWORD" "$KEYCHAIN" + security import $RUNNER_TEMP/cert.p12 -k "$KEYCHAIN" \ + -P "$MACOS_CERTIFICATE_PWD" -T /usr/bin/codesign + security set-key-partition-list -S apple-tool:,apple:,codesign: \ + -s -k "$KEYCHAIN_PASSWORD" "$KEYCHAIN" + security list-keychain -d user -s "$KEYCHAIN" login.keychain + rm $RUNNER_TEMP/cert.p12 + + - name: Store notarytool credentials + env: + APPLE_API_KEY: ${{ secrets.APPLE_API_KEY }} + APPLE_API_KEY_ID: ${{ secrets.APPLE_API_KEY_ID }} + APPLE_API_ISSUER: ${{ secrets.APPLE_API_ISSUER }} + run: | + echo "$APPLE_API_KEY" | base64 --decode > $RUNNER_TEMP/AuthKey.p8 + xcrun notarytool store-credentials "patcha-notary" \ + --key $RUNNER_TEMP/AuthKey.p8 \ + --key-id "$APPLE_API_KEY_ID" \ + --issuer "$APPLE_API_ISSUER" \ + --keychain $RUNNER_TEMP/build.keychain + rm $RUNNER_TEMP/AuthKey.p8 + + - name: Build, sign, notarize + env: + PATCHA_SIGN_IDENTITY: ${{ secrets.PATCHA_SIGN_IDENTITY }} + PATCHA_NOTARY_PROFILE: patcha-notary + run: ./build.sh + + - name: Verify notarization + run: | + DMG=$(ls dist/patcha-*.dmg) + spctl -a -vvv -t install "$DMG" + xcrun stapler validate "$DMG" + + - name: Publish release + uses: softprops/action-gh-release@v2 + with: + files: dist/patcha-*.dmg + generate_release_notes: true diff --git a/build.sh b/build.sh index e556961..af16b44 100755 --- a/build.sh +++ b/build.sh @@ -15,25 +15,52 @@ for arg in "$@"; do [[ "$arg" == "--skip-app" ]] && SKIP_APP=true done +# Signing configuration. +# PATCHA_SIGN_IDENTITY - "Developer ID Application: NAME (TEAMID)". When +# unset the build is signed ad-hoc for local testing +# and is NOT distributable. +# PATCHA_NOTARY_PROFILE - notarytool keychain profile name. When unset, +# notarization is skipped. +SIGN_IDENTITY="${PATCHA_SIGN_IDENTITY:-}" +NOTARY_PROFILE="${PATCHA_NOTARY_PROFILE:-}" +APP_ENTITLEMENTS="swift-xcode/patcha/patcha/patcha.entitlements" +HELPER_ENTITLEMENTS="swift-xcode/patcha/patcha/helper.entitlements" + +if [[ -z "$SIGN_IDENTITY" ]]; then + ADHOC=true + SIGN_IDENTITY="-" + SIGN_FLAGS=() +else + ADHOC=false + # A secure timestamp is required for notarization and cannot be added after + # the fact, so it must be part of every signature we produce. + SIGN_FLAGS=(--timestamp) + if ! security find-identity -v -p codesigning | grep -qF "$SIGN_IDENTITY"; then + echo "Error: signing identity not found in keychain: $SIGN_IDENTITY" + security find-identity -v -p codesigning | sed 's/^/ /' + exit 1 + fi +fi + echo "Building patcha ${VERSION}..." # Step 1: build native macOS menu bar app echo "" if $SKIP_APP; then - echo "[1/5] Skipping Patcha.app build (--skip-app)." + echo "[1/7] Skipping Patcha.app build (--skip-app)." if [[ ! -d "dist/Patcha.app" ]]; then echo "Error: dist/Patcha.app not found. Run without --skip-app first." exit 1 fi else - echo "[1/5] Building Patcha.app (Swift menu bar app)..." + echo "[1/7] Building Patcha.app (Swift menu bar app)..." bash swift-xcode/patcha/build_app.sh echo " Patcha.app built." fi # Step 2: compile Swift helper binaries (accessibility helpers) echo "" -echo "[2/5] Compiling Swift helper binaries..." +echo "[2/7] Compiling Swift helper binaries..." mkdir -p data if ! command -v swiftc &>/dev/null; then @@ -69,7 +96,7 @@ echo " Swift helper binaries compiled." # Step 3: fetch the MobileCLIP image-encoder Core ML model (visual pre-filter) echo "" -echo "[3/5] Fetching MobileCLIP model..." +echo "[3/7] Fetching MobileCLIP model..." MLPKG="data/mobileclip_s2_image.mlpackage" if [[ -f "$MLPKG/Data/com.apple.CoreML/weights/weight.bin" ]]; then echo " Model already present, skipping download." @@ -82,29 +109,9 @@ else echo " Model downloaded to $MLPKG" fi -# Step 3b: fetch the FastVLM ONNX captioner model (gist captioning). -# CPU execution provider needs fp32-activation graphs: q4f16 vision/embed run on -# CPU, but the decoder must be the q4 (fp32) graph, not q4f16. -echo " Fetching FastVLM captioner model..." -FVDIR="data/models/fastvlm" -if [[ -f "$FVDIR/onnx/decoder_model_merged_q4.onnx" ]]; then - echo " FastVLM model already present, skipping download." -else - FVBASE="https://huggingface.co/onnx-community/FastVLM-0.5B-ONNX/resolve/main" - mkdir -p "$FVDIR/onnx" - for f in config.json tokenizer.json tokenizer_config.json special_tokens_map.json \ - generation_config.json preprocessor_config.json processor_config.json; do - curl -fsSL "$FVBASE/$f" -o "$FVDIR/$f" - done - curl -fsSL "$FVBASE/onnx/vision_encoder_q4f16.onnx" -o "$FVDIR/onnx/vision_encoder_q4f16.onnx" - curl -fsSL "$FVBASE/onnx/embed_tokens_q4f16.onnx" -o "$FVDIR/onnx/embed_tokens_q4f16.onnx" - curl -fsSL "$FVBASE/onnx/decoder_model_merged_q4.onnx" -o "$FVDIR/onnx/decoder_model_merged_q4.onnx" - echo " FastVLM model downloaded to $FVDIR" -fi - # Step 4: Rust build (replaces PyInstaller) echo "" -echo "[4/5] Building Rust binary..." +echo "[4/7] Building Rust binary..." rm -rf dist/bin mkdir -p dist/bin @@ -114,33 +121,101 @@ chmod +x dist/bin/patcha echo " Rust binary built: dist/bin/patcha" -# Step 5: Assemble .dmg staging area +# Step 5: Assemble the .app payload echo "" -echo "[5/5] Staging .dmg contents..." +echo "[5/7] Staging app payload..." DMG_STAGE="dist/dmg_stage" rm -rf "$DMG_STAGE" mkdir -p "$DMG_STAGE" -cp -r dist/Patcha.app "$DMG_STAGE/" -APP_RES="$DMG_STAGE/Patcha.app/Contents/Resources" +cp -R dist/Patcha.app "$DMG_STAGE/" +STAGED_APP="$DMG_STAGE/Patcha.app" +APP_RES="$STAGED_APP/Contents/Resources" + cp dist/bin/patcha "$APP_RES/" -chmod +x "$APP_RES/patcha" # Swift helper binaries + MobileCLIP model (resolved next to the patcha binary at runtime) cp data/ax_content data/ocr data/mobileclip data/observer "$APP_RES/" -chmod +x "$APP_RES/ax_content" "$APP_RES/ocr" "$APP_RES/mobileclip" "$APP_RES/observer" -cp -r data/mobileclip_s2_image.mlpackage "$APP_RES/" +chmod +x "$APP_RES/patcha" "$APP_RES/ax_content" "$APP_RES/ocr" \ + "$APP_RES/mobileclip" "$APP_RES/observer" +cp -R data/mobileclip_s2_image.mlpackage "$APP_RES/" + +# The FastVLM captioner model (~810 MB) is deliberately NOT bundled. The daemon +# fetches it into ~/.patcha/models/fastvlm on first run; see model_fetch.rs. +# Bundling it would quadruple the .dmg and add two multi-gigabyte notarization +# uploads per release. -# FastVLM captioner model (resolved at resources_dir/models/fastvlm at runtime). -# NOTE: ~0.8 GB — consider download-on-first-run instead of bundling to keep the DMG small. -mkdir -p "$APP_RES/models" -cp -r data/models/fastvlm "$APP_RES/models/" +# PatchaSourceRoot is a dev-only fallback pointing at this machine's checkout. +# Leaving it in would ship a local filesystem path in a public release. +/usr/libexec/PlistBuddy -c "Delete :PatchaSourceRoot" \ + "$STAGED_APP/Contents/Info.plist" 2>/dev/null || true -echo " Contents staged at $DMG_STAGE" +# Extended attributes left by cp/curl make codesign fail with +# "resource fork, Finder information, or similar detritus not allowed". +xattr -cr "$STAGED_APP" -# Create .dmg +echo " Payload staged at $STAGED_APP" + +# Step 6: Sign inside-out. Nested code must be signed before the enclosing +# bundle, otherwise sealing the app captures signatures that no longer match. echo "" -echo "Creating .dmg..." +if $ADHOC; then + echo "[6/7] Signing ad-hoc (local testing only)..." + echo " WARNING: Gatekeeper will reject this bundle on any other machine." + echo " Set PATCHA_SIGN_IDENTITY to a Developer ID Application identity to" + echo " produce a distributable build." +else + echo "[6/7] Signing with: $SIGN_IDENTITY" +fi + +sign_code() { + codesign --force --options runtime "${SIGN_FLAGS[@]}" \ + --sign "$SIGN_IDENTITY" --entitlements "$HELPER_ENTITLEMENTS" "$1" +} + +for binary in patcha ax_content ocr mobileclip observer; do + sign_code "$APP_RES/$binary" +done + +# Nested resource bundles carry no entitlements of their own. +for nested in "$APP_RES"/*.bundle; do + [[ -e "$nested" ]] || continue + codesign --force --options runtime "${SIGN_FLAGS[@]}" \ + --sign "$SIGN_IDENTITY" "$nested" +done + +codesign --force --options runtime "${SIGN_FLAGS[@]}" \ + --sign "$SIGN_IDENTITY" --entitlements "$APP_ENTITLEMENTS" "$STAGED_APP" + +echo " Verifying signature..." +codesign --verify --strict --deep --verbose=2 "$STAGED_APP" + +if ! $ADHOC; then + # Only a real Developer ID signature can satisfy Gatekeeper. Before + # notarization this reports "rejected ... not notarized", which is expected. + spctl -a -vvv -t exec "$STAGED_APP" 2>&1 | sed 's/^/ /' || true +fi + +# Notarize and staple the app. Stapling before building the .dmg means the app +# still validates offline once the user drags it out of the disk image. +if [[ -n "$NOTARY_PROFILE" ]] && ! $ADHOC; then + echo "" + echo " Notarizing app (this uploads the bundle to Apple and can take a while)..." + APP_ZIP="dist/Patcha-notarize.zip" + rm -f "$APP_ZIP" + ditto -c -k --keepParent "$STAGED_APP" "$APP_ZIP" + xcrun notarytool submit "$APP_ZIP" --keychain-profile "$NOTARY_PROFILE" --wait + xcrun stapler staple "$STAGED_APP" + rm -f "$APP_ZIP" + echo " App notarized and stapled." +elif ! $ADHOC; then + echo "" + echo " Skipping notarization (PATCHA_NOTARY_PROFILE not set)." +fi + +# Step 7: Create the .dmg +echo "" +echo "[7/7] Creating .dmg..." DMG_PATH="dist/patcha-${VERSION}.dmg" rm -f "$DMG_PATH" @@ -155,5 +230,25 @@ create-dmg \ "$DMG_PATH" \ "$DMG_STAGE/" +if ! $ADHOC; then + echo " Signing .dmg..." + codesign --force "${SIGN_FLAGS[@]}" --sign "$SIGN_IDENTITY" "$DMG_PATH" +fi + +if [[ -n "$NOTARY_PROFILE" ]] && ! $ADHOC; then + echo " Notarizing .dmg..." + xcrun notarytool submit "$DMG_PATH" --keychain-profile "$NOTARY_PROFILE" --wait + xcrun stapler staple "$DMG_PATH" + echo " Verifying stapled .dmg..." + spctl -a -vvv -t install "$DMG_PATH" 2>&1 | sed 's/^/ /' +fi + echo "" echo "Done: $DMG_PATH" +if $ADHOC; then + echo "" + echo "This is an ad-hoc build and is NOT distributable. To ship a release:" + echo " export PATCHA_SIGN_IDENTITY=\"Developer ID Application: NAME (TEAMID)\"" + echo " export PATCHA_NOTARY_PROFILE=\"patcha-notary\"" + echo "See docs/RELEASING.md for the one-time setup." +fi diff --git a/docs/RELEASING.md b/docs/RELEASING.md new file mode 100644 index 0000000..8462753 --- /dev/null +++ b/docs/RELEASING.md @@ -0,0 +1,99 @@ +# Releasing Patcha + +Patcha ships as a signed, notarized `.dmg` from GitHub Releases. It is not +distributed through the Mac App Store: the app needs Accessibility and Screen +Recording, which are not available to sandboxed apps. + +Distributing outside the App Store still requires an Apple Developer Program +membership and notarization. An unnotarized download is quarantined by +Gatekeeper, and since macOS 15 the Control-click "Open" bypass no longer works +— users must approve the app in System Settings > Privacy & Security. + +## One-time setup + +### 1. Apple Developer Program + +Enroll at ($99/year). The team ID +already configured in the Xcode project is `Z7JCWW3N99`. + +### 2. Developer ID Application certificate + +In Xcode: Settings > Accounts > Manage Certificates > + > Developer ID +Application. Confirm it landed in the keychain: + +```sh +security find-identity -v -p codesigning +``` + +You want a line reading `Developer ID Application: NAME (TEAMID)`. An +`Apple Development` certificate is not sufficient for distribution. + +### 3. notarytool credentials + +Create an App Store Connect API key (Users and Access > Integrations > App Store +Connect API) and store it as a keychain profile: + +```sh +xcrun notarytool store-credentials "patcha-notary" \ + --key ~/private_keys/AuthKey_XXXXXXXX.p8 \ + --key-id XXXXXXXX \ + --issuer XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX +``` + +## Cutting a release + +```sh +export PATCHA_SIGN_IDENTITY="Developer ID Application: NAME (TEAMID)" +export PATCHA_NOTARY_PROFILE="patcha-notary" +./build.sh +``` + +`build.sh` builds the app unsigned, stages the daemon, helper binaries and +models into `Contents/Resources`, then signs inside-out (nested executables +first, app bundle last), notarizes, staples, and builds a signed and stapled +`.dmg` at `dist/patcha-.dmg`. + +Without `PATCHA_SIGN_IDENTITY` the build falls back to an ad-hoc signature. +That is fine for local testing and useless for distribution. + +## Models + +The MobileCLIP visual pre-filter (~68 MB) is bundled in the app. The FastVLM +captioner (~810 MB) is not — the daemon downloads it to +`~/.patcha/models/fastvlm` on first run, because the signed app bundle is +read-only and a bundled copy would mean two multi-gigabyte notarization uploads +per release. + +Gist captioning stays off until the fetch completes; everything else works +immediately. Progress is published to `~/.patcha/model_download.json`. To +pre-seed a machine or recover from a failed fetch: + +```sh +patcha fetch-models # --force re-downloads +``` + +Set `ENABLE_MODEL_AUTO_DOWNLOAD=false` to opt out of the automatic fetch. + +## Verifying before you publish + +```sh +spctl -a -vvv -t install dist/patcha-.dmg +xcrun stapler validate dist/patcha-.dmg +``` + +`spctl` should report `source=Notarized Developer ID`. The real test is +downloading the `.dmg` on a machine that has never built Patcha — quarantine is +applied on download, so a locally built file will not reproduce a user's first +run. + +## Gotchas + +- **Sign after staging, never before.** Copying anything into the bundle after + signing invalidates the seal. `build_app.sh` deliberately builds unsigned. +- **`ENABLE_APP_SANDBOX` must stay `NO`.** The sandbox blocks spawning the + daemon, the Accessibility API, and writes to `~/.patcha`. +- **Usage description strings are mandatory under hardened runtime.** A missing + `NSAppleEventsUsageDescription` or `NSScreenCaptureUsageDescription` crashes + the process that triggers the prompt rather than showing a denial. +- **TCC grants are keyed to the signing identity.** Switching identities resets + a user's Accessibility and Screen Recording approvals. diff --git a/rust/patcha/src/cli/caption_eval.rs b/rust/patcha/src/cli/caption_eval.rs index fd76203..efa972a 100644 --- a/rust/patcha/src/cli/caption_eval.rs +++ b/rust/patcha/src/cli/caption_eval.rs @@ -3,7 +3,7 @@ //! across screen types (editor, browser, design tool, terminal, video, …). use crate::config::Config; -use crate::perception::FastVlmCaptioner; +use crate::perception::{model_fetch, FastVlmCaptioner}; use anyhow::{anyhow, Result}; use clap::Args; use std::path::{Path, PathBuf}; @@ -15,7 +15,10 @@ pub struct CaptionEvalArgs { #[arg(help = "A screenshot image, or a directory of png/jpg images, to caption")] pub images_dir: PathBuf, - #[arg(long, help = "FastVLM model dir (default: /models/fastvlm)")] + #[arg( + long, + help = "FastVLM model dir (default: bundled, else ~/.patcha/models/fastvlm)" + )] pub model_dir: Option, #[arg(long, help = "OCR helper binary (default: /ocr)")] @@ -41,13 +44,13 @@ pub async fn run(args: CaptionEvalArgs, _cfg: Config) -> Result<()> { let res = resources_dir(); let model_dir = args .model_dir - .unwrap_or_else(|| res.join("models").join("fastvlm")); + .unwrap_or_else(|| model_fetch::resolve_model_dir(&res)); let ocr_bin = args.ocr_bin.unwrap_or_else(|| res.join("ocr")); let mut cap = FastVlmCaptioner::new(model_dir.clone(), args.max_new_tokens); if !cap.available() { return Err(anyhow!( - "FastVLM model not found at {model_dir:?} — pass --model-dir or fetch the model" + "FastVLM model not found at {model_dir:?} — run `patcha fetch-models` or pass --model-dir" )); } diff --git a/rust/patcha/src/cli/fetch_models.rs b/rust/patcha/src/cli/fetch_models.rs new file mode 100644 index 0000000..c9c2c72 --- /dev/null +++ b/rust/patcha/src/cli/fetch_models.rs @@ -0,0 +1,37 @@ +//! `patcha fetch-models` — download the FastVLM captioner model up front. +//! +//! The daemon fetches it automatically on first run; this exists for pre-seeding +//! a machine and for recovering when `ENABLE_MODEL_AUTO_DOWNLOAD=false`. + +use anyhow::Result; +use clap::Args; + +use crate::config::Config; +use crate::perception::model_fetch; + +#[derive(Args, Debug)] +pub struct FetchModelsArgs { + #[arg(long, help = "Re-download even if the model is already complete")] + pub force: bool, +} + +pub async fn run(args: FetchModelsArgs, _cfg: Config) -> Result<()> { + let dir = model_fetch::user_model_dir(); + + if args.force && dir.exists() { + std::fs::remove_dir_all(&dir)?; + } + + if model_fetch::is_complete(&dir) { + println!("FastVLM model already present at {}", dir.display()); + return Ok(()); + } + + println!( + "Downloading FastVLM model to {} (~810 MB)...", + dir.display() + ); + model_fetch::ensure_fastvlm(&dir).await?; + println!("Done."); + Ok(()) +} diff --git a/rust/patcha/src/cli/mod.rs b/rust/patcha/src/cli/mod.rs index 40da460..11579a4 100644 --- a/rust/patcha/src/cli/mod.rs +++ b/rust/patcha/src/cli/mod.rs @@ -3,6 +3,7 @@ pub mod caption_eval; pub mod cluster; pub mod collect; pub mod compact; +pub mod fetch_models; pub mod graph; pub mod maintenance; pub mod observe; diff --git a/rust/patcha/src/collectors/accessibility.rs b/rust/patcha/src/collectors/accessibility.rs index e10be07..ddaf0dc 100644 --- a/rust/patcha/src/collectors/accessibility.rs +++ b/rust/patcha/src/collectors/accessibility.rs @@ -2,7 +2,7 @@ use crate::{ collectors::filters::{is_banking_domain, is_incognito_window}, config::Config, models::{Event, EventType}, - perception::{AppEmbeddingCache, FastVlmCaptioner, MobileClipEmbedder}, + perception::{model_fetch, AppEmbeddingCache, FastVlmCaptioner, MobileClipEmbedder}, }; use anyhow::Result; use chrono::{DateTime, Utc}; @@ -70,18 +70,18 @@ impl AccessibilityCollector { None }; + // Constructed even when the model is absent: availability is re-checked + // per caption, so gist starts working as soon as the first-run fetch + // finishes rather than waiting for a daemon restart. let captioner = if cfg.enable_captioner { - let model_dir = resources_dir.join("models").join("fastvlm"); - let candidate = FastVlmCaptioner::new(model_dir.clone(), cfg.caption_max_new_tokens); - if candidate.available() { - Some(candidate) - } else { - tracing::warn!( - "captioner enabled but FastVLM model not found at {:?}; gist disabled", + let model_dir = model_fetch::resolve_model_dir(resources_dir); + if !model_fetch::is_complete(&model_dir) { + tracing::info!( + "FastVLM model not yet present at {:?}; gist stays off until the fetch completes", model_dir ); - None } + Some(FastVlmCaptioner::new(model_dir, cfg.caption_max_new_tokens)) } else { None }; @@ -188,15 +188,18 @@ impl AccessibilityCollector { // 5. Gist: caption only on context switches (new app / new window), while the // screenshot is still on disk. Within-window drift ("same") skips the VLM. let gist = if transition != "same" { - self.captioner.as_mut().and_then(|c| { - match c.caption(&screenshot, &app_name, &window_title, &ocr_text) { - Ok(g) => Some(g), - Err(e) => { - tracing::debug!(error = %e, "captioner failed"); - None - } - } - }) + self.captioner + .as_mut() + .filter(|c| c.available()) + .and_then( + |c| match c.caption(&screenshot, &app_name, &window_title, &ocr_text) { + Ok(g) => Some(g), + Err(e) => { + tracing::debug!(error = %e, "captioner failed"); + None + } + }, + ) } else { None }; diff --git a/rust/patcha/src/config.rs b/rust/patcha/src/config.rs index d837822..8face83 100644 --- a/rust/patcha/src/config.rs +++ b/rust/patcha/src/config.rs @@ -65,6 +65,8 @@ pub struct Config { // Gist captioning (Phase 3) — model resolved from resources_dir/models/fastvlm. pub enable_captioner: bool, pub caption_max_new_tokens: usize, + /// Fetch the ~810 MB FastVLM model on first run when it is not bundled. + pub enable_model_auto_download: bool, // Patcha cloud API pub patcha_api_url: String, @@ -121,6 +123,7 @@ impl Default for Config { background_poll_interval_seconds: 15, enable_captioner: true, caption_max_new_tokens: 56, + enable_model_auto_download: true, patcha_api_url: "https://api.patcha.app".into(), patcha_access_token: String::new(), patcha_refresh_token: String::new(), @@ -254,6 +257,7 @@ impl Config { background_poll_interval_seconds: env_u64("BACKGROUND_POLL_INTERVAL", 15), enable_captioner: env_bool("ENABLE_CAPTIONER", true), caption_max_new_tokens: env_usize("CAPTION_MAX_NEW_TOKENS", 56), + enable_model_auto_download: env_bool("ENABLE_MODEL_AUTO_DOWNLOAD", true), patcha_api_url: env_str("PATCHA_API_URL", "https://api.patcha.app"), patcha_access_token: access_token, patcha_refresh_token: refresh_token, diff --git a/rust/patcha/src/daemon/loop.rs b/rust/patcha/src/daemon/loop.rs index ab7a217..2a45fe7 100644 --- a/rust/patcha/src/daemon/loop.rs +++ b/rust/patcha/src/daemon/loop.rs @@ -24,6 +24,7 @@ use crate::{ hourly::HourlySummarizer, llm::backend, models::Event, + perception::model_fetch, process::EventPreprocessor, }; @@ -139,6 +140,23 @@ pub async fn start(cfg: Config) -> Result<()> { "patcha daemon starting" ); + // The first-run model fetch is ~810 MB, so it runs in the background: the + // daemon starts collecting immediately and the captioner picks the model up + // once it lands. + if cfg.enable_captioner && cfg.enable_model_auto_download { + let model_dir = model_fetch::resolve_model_dir(&res_dir); + if !model_fetch::is_complete(&model_dir) { + tokio::spawn(async move { + if let Err(e) = model_fetch::ensure_fastvlm(&model_dir).await { + tracing::warn!( + error = %e, + "FastVLM model fetch failed; gist captioning stays off" + ); + } + }); + } + } + // ----------------------------------------------------------------------- // Subsystems // ----------------------------------------------------------------------- diff --git a/rust/patcha/src/main.rs b/rust/patcha/src/main.rs index da83472..3628160 100644 --- a/rust/patcha/src/main.rs +++ b/rust/patcha/src/main.rs @@ -34,6 +34,8 @@ enum Commands { about = "Run the FastVLM captioner over a folder of screenshots" )] CaptionEval(cli::caption_eval::CaptionEvalArgs), + #[command(name = "fetch-models", about = "Download the FastVLM captioner model")] + FetchModels(cli::fetch_models::FetchModelsArgs), // Summarization #[command(name = "summarize", about = "Generate daily summary")] @@ -127,6 +129,7 @@ async fn main() -> Result<()> { Commands::Collect(args) => cli::collect::run(args, cfg).await, Commands::Observe(args) => cli::observe::run(args, cfg).await, Commands::CaptionEval(args) => cli::caption_eval::run(args, cfg).await, + Commands::FetchModels(args) => cli::fetch_models::run(args, cfg).await, Commands::Summarize(args) => cli::summarize::run(args, cfg).await, Commands::Search(args) => cli::search::run(args, cfg).await, Commands::Review(args) => cli::review::run(args, cfg).await, diff --git a/rust/patcha/src/perception/captioner.rs b/rust/patcha/src/perception/captioner.rs index e50d892..cb069fc 100644 --- a/rust/patcha/src/perception/captioner.rs +++ b/rust/patcha/src/perception/captioner.rs @@ -54,12 +54,11 @@ impl FastVlmCaptioner { } } - /// Whether the model files are present (so the collector can skip captioning - /// cleanly when the model hasn't been fetched). + /// Whether every model file is present (so the collector can skip captioning + /// cleanly while the first-run fetch is still in flight). Delegates to + /// `model_fetch` so a partly-downloaded model never counts as ready. pub fn available(&self) -> bool { - ["tokenizer.json", &format!("onnx/{DECODER_FILE}")] - .iter() - .all(|f| self.model_dir.join(f).exists()) + super::model_fetch::is_complete(&self.model_dir) } fn load(&self) -> Result { diff --git a/rust/patcha/src/perception/mod.rs b/rust/patcha/src/perception/mod.rs index 950234b..73b14a3 100644 --- a/rust/patcha/src/perception/mod.rs +++ b/rust/patcha/src/perception/mod.rs @@ -8,6 +8,7 @@ mod app_cache; mod captioner; mod embedder; +pub mod model_fetch; pub use app_cache::{cosine, AppEmbeddingCache}; pub use captioner::FastVlmCaptioner; diff --git a/rust/patcha/src/perception/model_fetch.rs b/rust/patcha/src/perception/model_fetch.rs new file mode 100644 index 0000000..a79c996 --- /dev/null +++ b/rust/patcha/src/perception/model_fetch.rs @@ -0,0 +1,225 @@ +//! First-run fetch for the FastVLM captioner model. +//! +//! The model is ~810 MB, so it is downloaded on first run instead of being +//! bundled in the .dmg. It lands in `~/.patcha/models/fastvlm` because the app +//! bundle is code-signed and read-only — writing into `Contents/Resources` +//! would invalidate the signature. + +use anyhow::{anyhow, Context, Result}; +use serde::Serialize; +use std::io::{Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; + +const HF_BASE: &str = "https://huggingface.co/onnx-community/FastVLM-0.5B-ONNX/resolve/main"; +const MAX_ATTEMPTS: usize = 3; + +/// Files the captioner loads, each with a minimum plausible size. The size floor +/// catches a truncated transfer or an HTML error page saved under the model's +/// name, which would otherwise look present and then fail inside ort with an +/// opaque parse error. +/// +/// Note `decoder_model_merged_q4` (not q4f16): the CPU execution provider cannot +/// run the fp16 contrib ops in the q4f16 decoder. See `captioner.rs`. +const FILES: &[(&str, u64)] = &[ + ("config.json", 100), + ("generation_config.json", 100), + ("preprocessor_config.json", 100), + ("processor_config.json", 100), + ("special_tokens_map.json", 100), + ("tokenizer_config.json", 100), + ("tokenizer.json", 1_000_000), + ("onnx/vision_encoder_q4f16.onnx", 200_000_000), + ("onnx/embed_tokens_q4f16.onnx", 200_000_000), + ("onnx/decoder_model_merged_q4.onnx", 250_000_000), +]; + +fn patcha_dir() -> PathBuf { + dirs::home_dir() + .unwrap_or_else(|| PathBuf::from("/tmp")) + .join(".patcha") +} + +/// Where a downloaded model lives. +pub fn user_model_dir() -> PathBuf { + patcha_dir().join("models").join("fastvlm") +} + +/// Resolve the model directory, preferring a copy bundled next to the binary +/// (dev checkouts and any build that still ships one) over the downloaded copy. +pub fn resolve_model_dir(resources_dir: &Path) -> PathBuf { + let bundled = resources_dir.join("models").join("fastvlm"); + if is_complete(&bundled) { + return bundled; + } + user_model_dir() +} + +/// Whether every required file is present and large enough to be real. +pub fn is_complete(dir: &Path) -> bool { + FILES.iter().all(|(rel, min_size)| { + std::fs::metadata(dir.join(rel)) + .map(|m| m.len() >= *min_size) + .unwrap_or(false) + }) +} + +#[derive(Serialize)] +#[serde(tag = "state", rename_all = "snake_case")] +enum Status { + Downloading { + file: String, + file_index: usize, + file_count: usize, + downloaded_bytes: u64, + total_bytes: u64, + }, + Ready, + Failed { + error: String, + }, +} + +/// Progress is published as a file rather than pushed to the app: the daemon is +/// restarted independently of the menu bar app, so a file lets the UI recover +/// the current state whenever it happens to look. +fn write_status(status: &Status) { + let path = patcha_dir().join("model_download.json"); + if let Some(parent) = path.parent() { + let _ = std::fs::create_dir_all(parent); + } + if let Ok(json) = serde_json::to_string(status) { + let _ = std::fs::write(path, json); + } +} + +/// Download any missing model files into `dir`. Idempotent: complete files are +/// left alone, so an interrupted run resumes rather than starting over. +pub async fn ensure_fastvlm(dir: &Path) -> Result<()> { + if is_complete(dir) { + write_status(&Status::Ready); + return Ok(()); + } + + std::fs::create_dir_all(dir.join("onnx")) + .with_context(|| format!("creating model dir {dir:?}"))?; + + let missing: Vec<_> = FILES + .iter() + .filter(|(rel, min_size)| { + std::fs::metadata(dir.join(rel)) + .map(|m| m.len() < *min_size) + .unwrap_or(true) + }) + .collect(); + + let total_files = missing.len(); + tracing::info!( + files = total_files, + dir = ?dir, + "fetching FastVLM captioner model (first run, ~810 MB)" + ); + + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(60 * 60)) + .build()?; + + for (index, (rel, min_size)) in missing.iter().enumerate() { + let mut last_err = None; + let mut ok = false; + + for attempt in 1..=MAX_ATTEMPTS { + match fetch_file(&client, dir, rel, *min_size, index, total_files).await { + Ok(()) => { + ok = true; + break; + } + Err(e) => { + tracing::warn!(file = rel, attempt, error = %e, "model file download failed"); + last_err = Some(e); + } + } + } + + if !ok { + let err = last_err.unwrap_or_else(|| anyhow!("unknown error")); + let msg = format!("{rel}: {err}"); + write_status(&Status::Failed { error: msg.clone() }); + return Err(anyhow!(msg)); + } + } + + write_status(&Status::Ready); + tracing::info!("FastVLM captioner model ready"); + Ok(()) +} + +async fn fetch_file( + client: &reqwest::Client, + dir: &Path, + rel: &str, + min_size: u64, + index: usize, + file_count: usize, +) -> Result<()> { + let dest = dir.join(rel); + let part = dir.join(format!("{rel}.part")); + + // Resume a partial transfer where the server supports it. A 200 response to + // a ranged request means the server ignored the range, so the partial file + // has to be discarded rather than appended to. + let existing = std::fs::metadata(&part).map(|m| m.len()).unwrap_or(0); + let url = format!("{HF_BASE}/{rel}"); + let mut req = client.get(&url); + if existing > 0 { + req = req.header(reqwest::header::RANGE, format!("bytes={existing}-")); + } + + let resp = req.send().await?.error_for_status()?; + let resuming = existing > 0 && resp.status() == reqwest::StatusCode::PARTIAL_CONTENT; + let mut downloaded = if resuming { existing } else { 0 }; + let total = resp.content_length().unwrap_or(0) + downloaded; + + // truncate(false) because a resumed transfer appends; the non-resume path + // truncates explicitly below. + let mut file = std::fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(false) + .open(&part)?; + if resuming { + file.seek(SeekFrom::Start(existing))?; + } else { + file.set_len(0)?; + } + + let mut resp = resp; + let mut since_report = 0u64; + while let Some(chunk) = resp.chunk().await? { + file.write_all(&chunk)?; + downloaded += chunk.len() as u64; + since_report += chunk.len() as u64; + if since_report >= 8 * 1024 * 1024 { + since_report = 0; + write_status(&Status::Downloading { + file: rel.to_string(), + file_index: index + 1, + file_count, + downloaded_bytes: downloaded, + total_bytes: total, + }); + } + } + file.flush()?; + drop(file); + + let size = std::fs::metadata(&part)?.len(); + if size < min_size { + let _ = std::fs::remove_file(&part); + return Err(anyhow!( + "downloaded {size} bytes, expected at least {min_size}" + )); + } + + std::fs::rename(&part, &dest)?; + Ok(()) +} diff --git a/swift-xcode/patcha/Info.plist b/swift-xcode/patcha/Info.plist index 3d9e2cf..b0e5b83 100644 --- a/swift-xcode/patcha/Info.plist +++ b/swift-xcode/patcha/Info.plist @@ -30,16 +30,18 @@ CFBundleVersion 1 LSMinimumSystemVersion - 13.0 + 14.0 LSUIElement NSAccessibilityUsageDescription Patcha needs Accessibility access to track active windows and screen content. + NSAppleEventsUsageDescription + Patcha uses Apple Events to read the title of the frontmost window. NSPrincipalClass NSApplication - PatchaSourceRoot - $(SRCROOT)/../.. NSScreenCaptureUsageDescription Patcha needs Screen Recording to capture on-screen text. + PatchaSourceRoot + $(SRCROOT)/../.. diff --git a/swift-xcode/patcha/build_app.sh b/swift-xcode/patcha/build_app.sh index 479c071..894e53f 100755 --- a/swift-xcode/patcha/build_app.sh +++ b/swift-xcode/patcha/build_app.sh @@ -9,11 +9,16 @@ if ! command -v xcodebuild &>/dev/null; then fi echo "Building Patcha.app (Xcode)..." + +# Built unsigned on purpose: build.sh copies the daemon, helper binaries and +# models into Contents/Resources afterwards, which would invalidate any +# signature applied here. All signing happens in build.sh after staging. xcodebuild \ -project patcha.xcodeproj \ -scheme patcha \ -configuration Release \ -derivedDataPath .build \ + CODE_SIGNING_ALLOWED=NO \ build 2>&1 APP_SRC=".build/Build/Products/Release/patcha.app" @@ -23,7 +28,4 @@ rm -rf "$APP_BUNDLE" mkdir -p "../../dist" cp -r "$APP_SRC" "$APP_BUNDLE" -echo " Signing app bundle (ad-hoc)..." -codesign --force --deep --sign - "$APP_BUNDLE" - -echo " Built: $APP_BUNDLE" +echo " Built (unsigned): $APP_BUNDLE" diff --git a/swift-xcode/patcha/patcha.xcodeproj/project.pbxproj b/swift-xcode/patcha/patcha.xcodeproj/project.pbxproj index 032d0a9..9c520e2 100644 --- a/swift-xcode/patcha/patcha.xcodeproj/project.pbxproj +++ b/swift-xcode/patcha/patcha.xcodeproj/project.pbxproj @@ -196,7 +196,7 @@ GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; LOCALIZATION_PREFERS_STRING_CATALOGS = YES; - MACOSX_DEPLOYMENT_TARGET = 26.2; + MACOSX_DEPLOYMENT_TARGET = 14.0; MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; MTL_FAST_MATH = YES; ONLY_ACTIVE_ARCH = YES; @@ -254,7 +254,7 @@ GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; LOCALIZATION_PREFERS_STRING_CATALOGS = YES; - MACOSX_DEPLOYMENT_TARGET = 26.2; + MACOSX_DEPLOYMENT_TARGET = 14.0; MTL_ENABLE_DEBUG_INFO = NO; MTL_FAST_MATH = YES; SDKROOT = macosx; @@ -305,7 +305,7 @@ COMBINE_HIDPI_IMAGES = YES; CURRENT_PROJECT_VERSION = 1; DEVELOPMENT_TEAM = Z7JCWW3N99; - ENABLE_APP_SANDBOX = YES; + ENABLE_APP_SANDBOX = NO; ENABLE_HARDENED_RUNTIME = YES; ENABLE_PREVIEWS = YES; ENABLE_USER_SELECTED_FILES = readonly; diff --git a/swift-xcode/patcha/patcha/MenuBarController.swift b/swift-xcode/patcha/patcha/MenuBarController.swift index b29c60d..3b62ee6 100644 --- a/swift-xcode/patcha/patcha/MenuBarController.swift +++ b/swift-xcode/patcha/patcha/MenuBarController.swift @@ -32,6 +32,7 @@ private final class NoIconMenuItem: NSMenuItem { private var resumeNowItem: NSMenuItem! private var mcpStatusItem: NSMenuItem! + private var modelStatusItem: NSMenuItem! private var mcpPollTimer: Timer? init(daemonManager: DaemonManager, mcpManager: MCPManager, settingsWindowController: SettingsWindowController, settingsStore: SettingsStore) { @@ -96,6 +97,11 @@ private final class NoIconMenuItem: NSMenuItem { mcpStatusItem.isEnabled = false menu.addItem(mcpStatusItem) + modelStatusItem = NSMenuItem(title: "", action: nil, keyEquivalent: "") + modelStatusItem.isEnabled = false + modelStatusItem.isHidden = true + menu.addItem(modelStatusItem) + menu.addItem(.separator()) menu.addItem(buildPauseSubmenu()) @@ -269,6 +275,35 @@ private final class NoIconMenuItem: NSMenuItem { extension MenuBarController: NSMenuDelegate { func menuWillOpen(_ menu: NSMenu) { refreshPermissions() + refreshModelStatus() resumeNowItem?.isHidden = daemonManager.status != .paused } + + /// The FastVLM captioner model is fetched by the daemon on first run, so the + /// menu reports progress from the status file it writes. + private func refreshModelStatus() { + let path = NSHomeDirectory() + "/.patcha/model_download.json" + guard let data = FileManager.default.contents(atPath: path), + let obj = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let state = obj["state"] as? String else { + modelStatusItem?.isHidden = true + return + } + + switch state { + case "downloading": + let done = (obj["downloaded_bytes"] as? Double) ?? 0 + let total = (obj["total_bytes"] as? Double) ?? 0 + let pct = total > 0 ? Int((done / total) * 100) : 0 + let index = (obj["file_index"] as? Int) ?? 0 + let count = (obj["file_count"] as? Int) ?? 0 + modelStatusItem?.title = "Downloading gist model: \(pct)% (\(index) of \(count))" + modelStatusItem?.isHidden = false + case "failed": + modelStatusItem?.title = "Gist model download failed" + modelStatusItem?.isHidden = false + default: + modelStatusItem?.isHidden = true + } + } } diff --git a/swift-xcode/patcha/patcha/helper.entitlements b/swift-xcode/patcha/patcha/helper.entitlements new file mode 100644 index 0000000..92eead8 --- /dev/null +++ b/swift-xcode/patcha/patcha/helper.entitlements @@ -0,0 +1,10 @@ + + + + + com.apple.security.network.client + + com.apple.security.automation.apple-events + + + diff --git a/swift-xcode/patcha/patcha/patcha.entitlements b/swift-xcode/patcha/patcha/patcha.entitlements index bc04cfb..92eead8 100644 --- a/swift-xcode/patcha/patcha/patcha.entitlements +++ b/swift-xcode/patcha/patcha/patcha.entitlements @@ -4,5 +4,7 @@ com.apple.security.network.client + com.apple.security.automation.apple-events +