From d5491e3d9a08c934d386a177030b971dde51bc17 Mon Sep 17 00:00:00 2001 From: oleost <5670710+oleost@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:44:31 +0200 Subject: [PATCH 01/18] Back to the beta channel after 0.9.2 Co-Authored-By: Claude Opus 5.5 --- visionstate/config.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/visionstate/config.yaml b/visionstate/config.yaml index 5acf0f6..21fcf1a 100644 --- a/visionstate/config.yaml +++ b/visionstate/config.yaml @@ -1,5 +1,5 @@ -name: VisionState -version: "0.9.2" +name: VisionState (beta) +version: "0.9.2b4" slug: visionstate description: Teach a local AI to recognise states (open, closed, …), find objects (people, cars, pets) or read numbers in your camera images url: https://github.com/oleost/VisionState @@ -13,7 +13,7 @@ arch: ingress: true ingress_port: 8099 panel_icon: mdi:eye-check-outline -panel_title: VisionState +panel_title: VisionState β homeassistant_api: true hassio_api: true services: @@ -35,3 +35,5 @@ schema: mqtt_port: port? mqtt_username: str? mqtt_password: password? +environment: + VISIONSTATE_MEDIA: /media/visionstate_beta From b7578299598434ef0b5f6dff976fa636e788645a Mon Sep 17 00:00:00 2001 From: oleost <5670710+oleost@users.noreply.github.com> Date: Mon, 19 Oct 2026 16:34:26 +0200 Subject: [PATCH 02/18] Frontend: source-map-js 1.2.2 (GHSA-68fv-2mgg-jv7q) Co-Authored-By: Claude Opus 5.5 --- visionstate/frontend/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/visionstate/frontend/package-lock.json b/visionstate/frontend/package-lock.json index 51beaf6..4eda88a 100644 --- a/visionstate/frontend/package-lock.json +++ b/visionstate/frontend/package-lock.json @@ -1087,9 +1087,9 @@ } }, "node_modules/source-map-js": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", - "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.2.tgz", + "integrity": "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==", "dev": true, "license": "BSD-3-Clause", "engines": { From 32065efac24c8416832329fe22c8ee7f8000e8ea Mon Sep 17 00:00:00 2001 From: oleost <5670710+oleost@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:48:56 +0200 Subject: [PATCH 03/18] 0.10.0b1: backup of everything, and restoring it Settings -> Backup downloads one ZIP with everything: a consistent copy of the database (SQLite's backup) and the data folders, by the parts of settings.BACKUP_PARTS (sensors and settings always; training images, people and faces, history and downloaded models ticked by default). Camera passwords are masked unless ticked. The download is streamed. Restore replaces everything: the upload is checked and extracted next to the live data and its database migrated there; then the runtime is stopped, its database closed (Database.close, waited for, so its worker threads cannot write into the restored one), its models let go (Runtime.release: two sets at once ran the test VM out of memory), the data swapped and a new runtime started. Any failure puts the old data back; a restore cut off by a stop is undone or finished at the next start. Requests made meanwhile wait for the new runtime. Entities of sensors, classes and people that are not in the backup are removed; the others keep their unique IDs. A newer backup is refused. Nothing added later is missing from a backup: every table and data folder must belong to a part (tests/test_backup.py), and a backup made by each database version is kept in tests/assets/backups and restored by every later version (tests/backup_fixture.py adds the next one). Also: ingress_stream, without which the Supervisor refused uploads over 16 MB (videos and sensor exports too), and Database.init no longer lowers user_version when an older version opens a newer database. Tested between two apps on a Home Assistant VM: 485 MB restored in 9 s, every entity kept its ID, faces recognised right after. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 3 + docs/ARCHITECTURE.md | 11 +- docs/SCOPE.md | 44 +- visionstate/CHANGELOG.md | 21 + visionstate/DOCS.md | 31 +- visionstate/backend/tests/assets/README.md | 4 + .../backups/backup-format1-schema15.zip | Bin 0 -> 20961 bytes visionstate/backend/tests/backup_fixture.py | 156 ++++++ visionstate/backend/tests/test_backup.py | 448 +++++++++++++++++ visionstate/backend/visionstate/api/backup.py | 199 ++++++++ visionstate/backend/visionstate/api/system.py | 7 + visionstate/backend/visionstate/backup.py | 454 ++++++++++++++++++ visionstate/backend/visionstate/db.py | 17 +- .../backend/visionstate/engine/runtime.py | 11 + visionstate/backend/visionstate/main.py | 25 +- visionstate/backend/visionstate/mqtt.py | 13 + visionstate/backend/visionstate/settings.py | 48 ++ visionstate/config.yaml | 5 +- visionstate/frontend/e2e/ui.spec.ts | 35 ++ visionstate/frontend/src/App.svelte | 2 +- visionstate/frontend/src/lib/api.ts | 28 ++ visionstate/frontend/src/lib/app.svelte.ts | 11 +- .../src/lib/components/BackupCard.svelte | 170 +++++++ visionstate/frontend/src/lib/types.ts | 20 + visionstate/frontend/src/lib/ui.ts | 11 +- .../frontend/src/pages/Settings.svelte | 3 + 26 files changed, 1762 insertions(+), 15 deletions(-) create mode 100644 visionstate/backend/tests/assets/backups/backup-format1-schema15.zip create mode 100644 visionstate/backend/tests/backup_fixture.py create mode 100644 visionstate/backend/tests/test_backup.py create mode 100644 visionstate/backend/visionstate/api/backup.py create mode 100644 visionstate/backend/visionstate/backup.py create mode 100644 visionstate/frontend/src/lib/components/BackupCard.svelte diff --git a/CLAUDE.md b/CLAUDE.md index 5c401bf..436f862 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -47,6 +47,9 @@ Every fact has **one** home; other files link to it instead of repeating it. - **Camera credentials never leak:** `redact.py` masks them in logs, errors and exports. - **No hard-coded defaults** outside `settings.py` / the registries; the UI reads them from `GET /api/v1/config`. +- **A backup holds everything:** every table and data folder belongs to a part of + `settings.BACKUP_PARTS` (tests fail otherwise), and a new database version gets a kept backup in + `tests/assets/backups/` (`python -m tests.backup_fixture`) that every later version restores. - Home Assistant state values stay lower-case keys (`open`, `car_parked`); unique IDs and topics never change once released. - Marketing text (README, release notes) promotes VisionState on its own merits — no comparisons diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 2594cb4..c082d5a 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -52,7 +52,7 @@ API as `request.app.state.runtime`. | `db.py` | SQLAlchemy tables and the schema migrations (`MIGRATIONS`, `SCHEMA_VERSION`) | | `storage.py` | Image files on disk: training samples, history frames, thumbnails | | `engine/` | The runtime — see below | -| `api/` | The REST API (`/api/v1/…`), one file per topic: `sensors.py` (CRUD, quality, export), `history.py` (the history: one filter for the list, its count and facets; history frames), `samples.py` (training images), `teach.py` (taught boxes; what is taught from a history frame is written into its row: `remember_taught`), `faces.py` (teaching a face on a sensor; the people and their faces, shared by every sensor), `cameras.py` (camera and entity lists, the light, previews), `review.py` (review queue and answers), `imports.py` (import), `system.py` (UI config, status, AI models and deleting downloaded ones, storage, review rules, review reminder, teaching and face limits), `common.py` (input models and views shared by them) | +| `api/` | The REST API (`/api/v1/…`), one file per topic: `sensors.py` (CRUD, quality, export), `history.py` (the history: one filter for the list, its count and facets; history frames), `samples.py` (training images), `teach.py` (taught boxes; what is taught from a history frame is written into its row: `remember_taught`), `faces.py` (teaching a face on a sensor; the people and their faces, shared by every sensor), `cameras.py` (camera and entity lists, the light, previews), `review.py` (review queue and answers), `imports.py` (import), `backup.py` (backup of everything, restore), `system.py` (UI config, status, AI models and deleting downloaded ones, storage, review rules, review reminder, teaching and face limits), `common.py` (input models and views shared by them) | | `sources.py` | Camera sources: Home Assistant camera, HTTP snapshot, RTSP; address checks and size limits; the Home Assistant REST client (states, services, notify services, the app's page) | | `mqtt.py` | MQTT topics, Home Assistant discovery messages, the bridge (connect, publish, commands) | | `ha_events.py` | Home Assistant WebSocket: state changes of trigger entities, entity registry | @@ -64,6 +64,7 @@ API as `request.app.state.runtime`. | `teach.py` | Object sensors: comparing detected boxes with the boxes the user taught | | `imaging.py` | Decoding, regions (ROI), crops, change signatures, drawing boxes | | `uploads.py`, `bundle.py` | Uploaded files (images, ZIP, video) → frames; sensor export/import bundles | +| `backup.py` | A backup of everything (database copy + files, by the parts of `settings.BACKUP_PARTS`) and restoring one: staged, swapped in, undone on failure | | `redact.py` | Masks credentials in URLs, messages and log lines | ### `engine/` @@ -172,7 +173,13 @@ config in the builder of its kind in `mqtt.py` (`_state_entities`, `_object_enti in Home Assistant hang on them). **A new column**: the field in `db.py`, a new `SCHEMA_VERSION` with its entry in `MIGRATIONS`, and -a test that an old database is upgraded (see `tests/test_entity_ids.py`). +a test that an old database is upgraded (see `tests/test_entity_ids.py`). Then a backup made by the +new version: `python -m tests.backup_fixture` (in `visionstate/backend`), committed — +`tests/test_backup.py` asks for it. + +**A new table or folder of data**: its part in `settings.BACKUP_PARTS` (a folder that is made again +by itself: `BACKUP_LEFT_OUT`), so backups hold it; `tests/test_backup.py` fails until it is there. +A folder is a `Settings` property under the data or media folder. **A new API endpoint**: in the `api/` file of its topic, input as a pydantic model, its function in `lib/api.ts`, its type in `lib/types.ts`. diff --git a/docs/SCOPE.md b/docs/SCOPE.md index 7aed611..250c229 100644 --- a/docs/SCOPE.md +++ b/docs/SCOPE.md @@ -684,8 +684,47 @@ automatically. - Import always creates a new sensor and retrains it; bundles are validated like API input. `manifest.json` is capped (`UPLOAD_LIMITS["max_manifest_mb"]`); an unreadable image in a bundle is skipped and counted (the response's `skipped`), so an import never stops halfway. -- Not implemented: full export of all sensors + global settings, merge/replace import modes, - exporting trained heads (retraining is faster than shipping them). +- Not implemented: a sensor bundle with its trained head (retraining is faster than shipping it). + +### Backup of everything (`backup.py`, `api/backup.py`, Settings → Backup) + +For moving between installations and between the stable, beta and dev apps (which have their own +data and share entities): one ZIP with everything, and a restore that **replaces** everything. +Adding to what is there is left to the sensor bundle above: merging people, IDs and history of two +installations is much code for a rare need. + +- **A copy of the database, not an export of its rows**: SQLite's backup of the live database, + with the rows of the parts left out deleted from the copy. A new table or column is in every + backup without code; `settings.BACKUP_PARTS` names each table's and each folder's part, and + `tests/test_backup.py` fails for one that is in none, so nothing added later goes missing. + Files go along by folder (`files//…`), thumbnails never (made again). +- **Parts** (all ticked at export): sensors and settings (always: rows, trained heads), training + images, people and faces, history (with `reading_stat`), downloaded models. A restore empties + the folders of every part but the models, also when the backup does not have the part (its rows + are gone); models are only added (a missing one is downloaded again). +- **Camera passwords** are masked like in a bundle unless the user ticks them (unticked by + default, `BACKUP["credentials"]`): a full backup is for moving one's own installation, where + typing every password again is the usual pain, but the file then holds them. A restore lists + the sensors whose password was masked. +- **Versions**: `backup.json` has the archive's `format` and the database's `schema`. An older + backup is migrated by the usual migrations; a newer one (format or schema) is refused with what + to do — an older version cannot know what a newer one stored. A backup made by every database + version and format is kept in `tests/assets/backups/` (`tests/backup_fixture.py`, a test fails + when the current one is missing), and every one is restored by every later version's tests. + `Database.init` never lowers `user_version`, so a rollback or such a refused case never makes + data migrations run twice. +- **Safety**: the upload is checked and extracted next to the live data and its database migrated + there first; only then is the runtime stopped, its database closed (its worker threads must not + write into the restored one: `Database.close`, waited for), the live data moved aside and the + restored moved in, and a new runtime started. Any failure puts the old data back; a restore cut + off by a stop is undone (or finished) at the next start (`backup.clean_leftovers`). Requests made + meanwhile wait for the new runtime (`BACKUP["hold_requests_s"]`) instead of failing. Archive paths + outside the known folders are ignored. The download is streamed (no temporary file of its size); + the restore needs the upload and its contents on disk for a moment. Uploads reach the app + streamed (`ingress_stream: true` in `config.yaml`): without it the Supervisor reads a request + into memory and refuses one over 16 MB, which had also kept videos and sensor bundles under that. +- **Home Assistant**: sensors and people that are only in the app before the restore have their + entities removed; those in the backup keep their unique IDs and so their entity IDs. ## 13. Configuration @@ -719,6 +758,7 @@ is part of backups and needs no restart. | **Recognising faces** ✅ | Faces in person boxes (YuNet), people taught by face and shared by every camera (SFace, or AuraFace for a fast PC), a People page, an entity per person saying where they are, unknown person, review questions, export opt-in — phases 1–3 of [`FACE_RECOGNITION_PLAN.md`](FACE_RECOGNITION_PLAN.md), and teaching a face from photos | 0.9.0 (betas 0.8.1b3–b5) | | **Counter limits** ✅ | A counter's limit grows with the time it was away (learned, a fixed rate or per reading), Review moves a counter on, readings held when the camera's picture changes size, training images in the app's storage (backups hold them) | 0.9.1 (betas 0.9.1b1–b6) | | **Reviewing readings** ✅ | A counter's value sent at its first reading, decimals for its rate, a rejected reading that repeats waits in Review once, Review shows a reading's region large and one sensor at a time | 0.9.2 (betas 0.9.2b1–b4) | +| **Backup of everything** | One file with everything (parts to leave out, camera passwords only when ticked), a restore that replaces it and keeps the entities, backups of every database version kept in the tests; uploads over 16 MB through Ingress | beta 0.10.0b1 | | **Better counter readings** | Benchmark, the last value as an expectation, calibrated confidence, extra decimal, self-calibrating regions, wheels-v2, learning per sensor — status per phase in [`WHEEL_READER_PLAN.md`](WHEEL_READER_PLAN.md) | planned | **Open ideas** (not scheduled): more on the History page (export of a filtered selection, a diff --git a/visionstate/CHANGELOG.md b/visionstate/CHANGELOG.md index 049a676..c876c98 100644 --- a/visionstate/CHANGELOG.md +++ b/visionstate/CHANGELOG.md @@ -1,5 +1,26 @@ # Changelog +## 0.10.0b1 + +### New + +- **Backup of everything** (Settings → Backup): one file with your sensors and settings, training + images, people and their faces, history and downloaded AI models — to move VisionState to + another Home Assistant, or between the stable and the beta app. Everything is ticked; untick what + you do not need (each shows its size). Camera passwords are left out unless you tick them. +- **Restore** replaces everything in the app with a backup. Sensors keep their entities and entity + IDs in Home Assistant; what is only in the app before is removed. The backup is checked first, + and if anything goes wrong the app keeps what it had. A backup from an older version is upgraded; + one from a newer version is refused (update the app first). The sensors whose camera password + must be entered again are listed. + +### Fixed + +- Uploads larger than 16 MB (videos, sensor exports) were refused when the app was opened in + Home Assistant. +- An older version started on a newer database (going back to a backup made before an update) no + longer makes the next update repeat its database upgrade. + ## 0.9.2 ### Better diff --git a/visionstate/DOCS.md b/visionstate/DOCS.md index 66d471b..97cd5e4 100644 --- a/visionstate/DOCS.md +++ b/visionstate/DOCS.md @@ -518,11 +518,36 @@ How long history is kept is set under **Settings → Storage** (below). ## Backup, export and import +**Settings → Backup** puts everything in one file and restores it — to move VisionState to +another Home Assistant, from the stable app to the beta app (or back), or to start again from +a known point: + +- **Download backup**: sensors and the app's settings (always), training images, people and + their faces, history and the downloaded AI models — all ticked; untick what you do not want + (each shows its size). Camera passwords are left out unless you tick **Camera passwords** + (anyone with the file can then read them). Large histories make large files. +- **Restore** replaces *everything* in the app with the backup: sensors, people, training images + and history. Sensors in the backup keep their entities (and entity IDs) in Home Assistant; the + sensors and people that are only here are removed. It asks twice before it starts. Download a + backup of the app first if you may want it back. +- What is left out of the backup is gone after a restore, except the AI models: those already + here stay, and missing ones are downloaded again when needed. Without history, a sensor starts + without its "last seen" and past frames; a meter keeps its last value. +- If camera passwords were left out, the result lists the sensors to enter them for again. +- A backup made by an older version is upgraded when it is restored. A backup from a newer + version is refused: update the app first (the beta app is often newer than the stable one). +- The backup is checked before anything is replaced; if the restore fails, the app keeps (or puts + back) what it had. It needs about twice the backup's size of free disk space for a moment. +- Stop the other VisionState app (stable or beta) before you restore into this one: both publish + the same entities. +- The app options (MQTT broker, log level) belong to each app and are not in a backup. + - Settings, the database, trained models and **training images** (also the boxes taught to object sensors and the taught faces) live in the app's own storage and are part of Home Assistant backups of the app, so a restore brings back what you taught together with its images. - **History frames** are stored in `/media/visionstate` (the beta app uses - `/media/visionstate_beta`); they are only in a backup that includes the media folder. + `/media/visionstate_beta`); they are only in a Home Assistant backup that includes the media + folder (a VisionState backup from Settings → Backup has them). - Older versions kept training images in `/media/visionstate/samples`. They are copied to the app's own storage by themselves after the update; the copies in the media folder are removed 30 days later (until then, going back to the backup made before the update still finds them). @@ -552,8 +577,8 @@ How long history is kept is set under **Settings → Storage** (below). New versions are released as beta first. To test them, add `https://github.com/oleost/VisionState#beta` as a repository and install **VisionState (beta)**. -It is a separate app with its own data: move sensors with Export/Import, and run only one of the -two apps at a time (both publish the same entities). +It is a separate app with its own data: move everything with Settings → Backup (or single sensors +with Export/Import), and run only one of the two apps at a time (both publish the same entities). ## AI models diff --git a/visionstate/backend/tests/assets/README.md b/visionstate/backend/tests/assets/README.md index 665c6ff..5c1c16c 100644 --- a/visionstate/backend/tests/assets/README.md +++ b/visionstate/backend/tests/assets/README.md @@ -41,3 +41,7 @@ The demo sensors (`scripts/seed_demo.py`, the README screenshots) use: | `demo_garage.jpg` | A garage door from inside, cut; the script draws it open and half open | [Utility building interior view of wood panel overhead garage doors](https://commons.wikimedia.org/wiki/File:UTILITY_BUILDING_INTERIOR_VIEW_OF_WOOD_PANEL_OVERHEAD_GARAGE_DOORS,_LOOKING_NORTHWEST._-_Tucson_Plant_Materials_Center,_General_Utility_Building,_3241_North_Romero_Road,_HABS_ARIZ,10-TUCSO,32C-7.tif), HABS (public domain) | "Emma" is a made-up name. The water meter is `counter_turning.jpg` above. + +`backups/` holds a backup made by each database version and backup format (`backup-format-schema.zip`), +drawn by our own code (`tests/backup_fixture.py`): every later version's tests restore each of +them. Never change or remove one; add the next with `python -m tests.backup_fixture`. diff --git a/visionstate/backend/tests/assets/backups/backup-format1-schema15.zip b/visionstate/backend/tests/assets/backups/backup-format1-schema15.zip new file mode 100644 index 0000000000000000000000000000000000000000..8551c236e46a65c86152ff38ed93a4427994cbe9 GIT binary patch literal 20961 zcmeI42UHW=*7ye%D~d`JL8AiFL`r~wlw1p-C?Fytg2YgzOHqp8T=jw!6#)@J1Sx`w z7zIJ8kt!W&0s_)|?}Q{X|AY5pLGSnXzHhzvt+$pFPC_Pg&OYbNWdHWwXHSkEUb>7G zf}oYq%Nts!V_eogU(EnP3`-zr4cKc)G_pK@ZnuSlt<8D=X4O2g^&TTs*8Mi;`A&a` z%4BzA$X??wA)VCxx{NvGXpl+E>422@L|2x(r6D6>hi)?dUY~kK%!j>r;CR$b^MT9B zNl}cEXE##9sS2D&68F?iww?BhNs|{&v2V0eO%)v3T9TH~pAvw(VPz9Pr&@fXn6q8{ z0K14h2UN8AVtW(A`io_i9u>)w>Xptm5}WatEJJpszlnmJi&_L-e{em+~mvgVY?LfS|)az=Bkd$sCG3F+ZvZ?_Tr|WPzmDx4QNxvFB^STb# zZ*+4Gon=v8eMX=PBe*28>cYvDJwXbx_}D&53)A@JfJ-ioCHa?9d5_r5Ejjwll(`UH z+7mZ`xq8C@tphI?%pJ^aZ5$kljwZW}4TtXQS#j67yfwQJd@-=lpQoD3Q&UP`H|Rv{ zqx+enr}yh>@`P+qD`4f+IoI4bSlZAm9*i9b(N(>E=E3c|c>=A$S+$JgoxyKxhm15Q zBM)vpMddrVUXoc!Yt#8PF`kBi=N~_LPQ|%((Z+;VfJx z^7+bkx|NCHOzu@k@Nc?fl9fNfl)H@6ld`j#|Bcv@v)f7~m~zFbEXLhPdy)8@*bL{| zh)zW={FVzUafk(Udx%L9xJC%+>O)m^AEu{TBQ>39f)S4RF<)?iH*Y2O!kTf_f;`e5kUdoCQKZ=iUEcY%E?T^Dnss}Hs1$RqU zO>*~9c7}FpN4?8a*d3$Pc_Ztkb?%*e^atc^W-i(vFG*JI+S*uA#O$NCaryqYP8x5y zSBxu{YVI%LeG*$A8aXr4zLjmVW5$k39q8I-dstXGW^JNE)!54qwXvpBUVCWA>$J`a zRkF1k4Tw}fO52(%x9e^$kGdJJ=H^5L{;bSgUcJq!Y95C(X>ZQfXz(I*RX%CE1NAef zj|&F8sB~|#zGl8L{|-$qQ|{QsYvorxAMX!&Pro1;a5vNt7TCjHHTI!gs`~{KZus4a zv3DJp8g~SkpUv9X;-r2~Pix52_WtHPm!dj07Oi-59#*$o9$eb}%1tispPQcU*w%l} zt@d>_W{x#?@Z^k9H8nH!Ag}+y=jjz`u5tIPQV(RFiqpJy!qxt&6~D>_2S3@BAptmT z@lekOjja-Q?hwsy29FC*zgSti)t5ImuHk;kvkWh5OV`Ya4b#P?H)4)85+Iwp5sPLm z%bxt>^5@05)sM%jsE;%U@J2eYG^B2Ibe&Kro0=(St7IC&)%?3PZdf>*DerzAVU*7J-h7&?POdmI)RA;~d+qBzTdga~F3@i!e0=A4 zK4vEIddlPVMIUYamxgoRtv2>7jvFRfiiFy*I2^W_Q>P4i9(w9odEC9~_%W#uj+2v? zij|LPw9Ig8i>W>0D^^JDSvh^>S>41v!vZ zn3>zsO)S0MA970f3|@F(c2zb##`2;kF^iom&UezggK4H9kF3s(KO@BqwB!8NdN{jW z^<03ggQ=^1Ad$!FdI4{gc&xo%inpNk#(Q@~EA>964(;;hN@DTAlnDr7(nx&5fEk}CC;TrBGPJxrRMP)1a*`n2)7y^>qj@3Rr zq4n`2i(Z|Tuxtf~`R@WBVw+;J2M0nNW6xT&2xbyF`~6LR0-@}cgU z$9+X5_t}S@GgzEAn=;VT5scn5-N{UuRNXVIWbk&O2tar%Q8cS6{-z zzNwvd>y%a|JTupOS~yR-yTSdT(j=Di z*zqLJGz4?+uF-3ZLp)1_T84u74CCsBhDQ0`Ztd7~!Civ=hH`XJApeB_Z`wzz`tD-^J}!TR#{=;ET)u6eNTh&$z#r4 z<1n4AD`9H=+r0N0sSm3xjJ?AuR^@MeEYuek_V|v2tgKe4u!W%BF(3OI?5e0$M6H+= zdc`;2%0@xvwzc>Ay#k%S`w~u-3Hof)39{eLALC0ow)0-XlNDxq0(Yi=q#ya9`$_H9q_Q5Lkn@6 z^r3I^kwb#rRV&QymBc1F(lKGPoVj$bi#F|MDpm%c=afTvP57Dm;>o^3mI3jdszz6` zN?MMo3m6~S5g%$G8dTCSu5VDNkCdg~o*B*xFA1uqJk=Mtb!5BF@#LqCj?YPcO%<-G%$I?r10S>XdZZcRdN^Ne&3(-zOqm@{n7k zwW6No$WsZ~2GKLe5sHLDeH?zNn!uXWeS-IP7d1Gli#)Q|PB78dEobX=6pWKmJ4Z%G zbeLx1qu+BabRF6n7S|vkmz!9@GziCdI1k!XNi#i;tmP_lG3a$eTU3~4DmCjkJ|FJ4 zdUsxV-g4<>O%o_H=WwpdiO-iQbn`Y#_DY96?7qqu(YEZJ zig1zb;C(kBoJ4EeC#iB+{767pPqjl=9UTWrJhZ?34Bg+ff@y|xMsIlk@YQH`vCkb? zS9VgMFfsBsflbCn*)M*(WwP}0Hm>XWy)&rC>zTee)MyyjL{&Mh`MCo3&4XU&+0nN3 zTt-*-@^4zXY+vNbUDD<5#d$d+VjXZ1ePWEf_1|mOk&0I5$|DjhVa2W0$1mBrsAcSa zHeMoZNgJQ5s@jF|OZGD7Oq1MEyCcoy?$XDSiLp}NyKOfj=|Qy~J;L!lSjxWQiBO`& zd9{&;X4sF$WdV)}AD>f%g@q3rnluFV*32~GbTVG}95TN5D8On$N7%CcL;IR2e!5(| zujug;wqv+FJ(`P}TIFj?;rRCUDU;1kS#_nXWFZ3rxI#oBf>lMAe@QYI=rn0t%$W?;^S8er#~Shl)Pbv%AQ zE%2tg|H)IU<7+cX-uudh`$P;+g+@}dtnMYoj_@4IG;+N&(wG?aQcbs^n}t zr)$E{ZRO;ilNyZ{8}d_JH>mpuD6#I>5t%8gJT+KCB}!Imn#!NI9d_om8|F@z8xn|` z;6M7ywEMT2cD4V_?!!_~eH=jTIkRMlg!QL1*bNA{k zK^HByX!E=D&h19#oOimN1<8gO-#3FV4jQ_KHm432`_`w#*2!Urq|&B?LdkT-)(iA! z?-i0S7#zvh;>c|~VB*?2`BCK1#yvRy`vZI9a4psP@W>I(X3N5X9=X6ee;;>&+AW4z znOPaL?P>M4v)IUU?c+6%Je^ECuTW)E^v>KjCEI02b3ZvWvw7388}S{L(@j=&%nki( z)pS%`MN7`!TkStM#4g(!a#>c*^pRUXlJ-8TPg?v{XY9!B3Y)um7Ms*+`PKNeqt#S2 z$lWaWiz`|N8e&h7SRBqBjlOzDwAWQ(rcvw}8@cVq58Y-3NMkv12%v#*PXM_1diCG!=`$0Qqim{?^qL4I`(5(GFc{YE{lYbRnY%qWa4E^wzdJDFFBWY zhj$5fuv{r(qg?sB_tGow__x%@6<+4>Ea5)MGvgc4B_&nsCKTX&!L!->#7uaPJ26Irk@?ylOnkH8ncD=#iTjO9dLk&?ou?@(`Oj&WqvL$hE4TDZv)v!-Q;qiF^6E2+Vom&;5QQzr1$@Ou<5Q2l^Bc3!9#Zo3}6K z%O+4v&8)F?D zU}0rrV`F0G*u>7dX(KBe>wF^&OF^AwjH?(KSFx^LvzGNAZ|ECn!%FBB^k6B&4rs{+ zhNT-A&^!nO`pL-fX#*qwv@t9JeO$g`<*LYCd3b@dG&+S)rhySjUN`})VmCnl$+ zXJ#pL^W$QGmi{>`@chGZZ2;q1vTWJXWh>^##jwN)l%*S%F>V)MzHzVS3Zfm$4vEVv zS@-!zzQ|t1gFQCNW@ulznw?j&jc;szXrD&*a|65bFOBT;z`l&@1H`nH0i3+08z4M1 zCL|D*^@cJU&R~ zGCcs_8D>G4W)DQ61G$7%1*zepg2jA3*&25&G<6bP6(-H2t4exvGB2t6^p|#{P}@zo z-v)(>%}~fD8rjr?#E#Hn;pzT4Wgo;1g$T}m$QAOqDopd9!Zh5~=%f)S5jZ%P8-;3` zQRsFB(yT%gub`{7P>!I`Q0?RmR+yBbf28v%uGqykJuzK5z`~b^{Wa98gHh3DLq+&$~4fVzepg1k$W~ zgsE=Nd-6-6iE(?qgYkrL6k2`yPLD?@@iCF25Rg35Qu10;iEWP3#>lGq+c(AClFADg z$<&YsMfeeK&n#6^0&MI@;olrg_lQBEO0l}`tvqm{C!yW1`y!_XwRqAjrBaEF7Ulj( zQ;8LtFFM^rm;C3S=xrLb0u*Y((=#Yx1ITF_3jG%4BRn(Fy~Ic0!8hf*GsCCzo}8#c z3%qd{$*F>S?^;eUZy7adcXP`DmP?{dX9o{jQBG!<(EqI{tFB0O<{d@vq7T^fx; zjNoAm^3kpQJPPeb;0Iu^RLTT{66?`#3PA%YK}6rV;aj<=?pk`)FB?5;I{gjP({J(% z8G0>e@RBLyQ7p(h6P4wFFEa0$<-Zb-WBuS{pklzaezaC0N=FevA&-zY=DE$28njKh zD3q^>LTj)&m`SPLh*|M#DC9%>{Vb^zh1%KApiu8lg1rJf6dFy7D7TL~wk3>-<^w(!Gj84?^4v@hKMYj-L}3jchk%My{C5Y(p9z z)4Zrugx%~9e6M9Ce$=twnytL?qaV`fAI>NFH#+CcStLAm&9Pu$0eb>is?aykJz?_0LOok0We zG*EvkZwb70LvC!LMp(DB^J7Wf0rzwgPub^B9W3zJ=t4v zqg`=VCRg&;O};ku)p&J?CtzC2n7H}MN|9T8hX_2H1i?IR{9!Lat%oO~ukmEv8R+L3 z(wNntiK5W{o&G3vxe}fy3ET$HaZ38p*7?p8$fuVCE(|ZrKLAR+-3xcQKSrTd09*h_ z##Q*yJ)($^-=a3O$4;}tc~Uv>y8~IB?20ug1S;o%>qx>JGrNTYP(+2skAT<6z=J6C zrk6bXWcHa4zI}5WzIO+~eho&0W;%B1>JvAq_aauwnR|x2HTaFiP8ST7YM%=r&9XZ#8HPF(N(PBZ)u3{nVe z18Qpoe4*?9u#&2nJ*b#FO~TG#3@4N~d8I;9Neq zrWQCgBcGV0LsuhE)aM~HDZ`(hMT99S zxJwR&QtIBl)I>7vnCZ&^v;e3&_xIPNUoxv8e09uwCodE-WgcW^wg`=*@n*sG2_~uI zS=p~H<%zsfG%v8b-da~xTj8bzM&jNMQyH7#-q>du7%sCZJITI^4UV0GGuxBkww)(k z58-S}XHWSEkop7je>Ay@h4cu}3IAUF^cV^~+^7?&6OL4%(D=bg4AmKhs=S5Y#m~ON z1AZ1@ku0DMt3oCv$=5Ik08yNGUY=J;`ZIY13jarb6ncKT4)L+*wG5vH+LxD_heFe9 z(g32=PE%lN7Y2AR2TIz^asWi&a}hG21!!?`_oo6`MLh4|p$-&MZ59(Jgd|ZFY>rXh6b=R(n)mI+IMvdV;sn#8E>ITaqj-a*=i&XHT;8}bSy4r@Cl z7wvo7FxuKL4Q?}>)j&Mmxqu6_;S;y=Miocd?S8L7AZu^$KkHJiV67& zEIqkEp6u51Arp^mibLW=X%B&x?8h5MP{PPVB`9QBSDj3&F5o7Ho9K4)thkzIU+pIn zylV?-<*Rt#i?3?vYlMoQJBM)#)p~S})XrE&D;fgC2?E4{%7XBG0L?IBE{r@L3=kLJ z_d&P=Zr^4nN@s*SZv#(i>3bv>9N5#+Av*E@sW~5nxtvFyWCNJd0+3irWd_D}1bm%b zO7Z=s^KCwGzdZ;Q5*dyspV=L%GS?=uJE>4Nj7cX)X3f?GprSlov`L=Y}g8*`$2isC07>cj~x zx=z#}=24Y#m3A4w4vkub2t1nyXTHOk9F-M?O1r?k5U2)O_epoj-=1jO_7$|-uSrw~ z6jFObH{t=Db;m~_w{yVdR-xgzFg)7eEVxYgaQQ9Mn=$}UEA;anQHYh?89SSfHhLsQL~oF#)=_#gIv`Kq^Qe1 zrEB|yc#oM)JNEkZjvAdx!syEkP5_(>R{?Nj;Mpg%!6>pog}h=;Pl(5noQEz-3adg!tgZh zOBXa%X#1{n&>1y`_x`WzDUI#u=Nj<}7f4MBmGgB;)_c4obSKlzSkvvdZ?4?dxj}!3 zRcGk6>tSYPRxPs7;o%dCiuOJC?)8K7Z1dYf)b}5}GAwr2gwHEmngJBfmmlL$Lex;) zNK8x;XJRZXhLaSRGL$uumM{YARXD5+crumvg~MMs{14-BLI}8PzfbL3yqNJf@7FS~ z>%%u9+o~liaD`OsniC&wnj3MiPuCZgW5bMfUZYShP#QJWe|lI33ZoMU(9Pg`U$0(= zLdps5`Ggx|b?Z%arFVMBuQgB-NTJtf+IEN3DQ^gMQmJSeEkdCwNfi3zVO4^f5y1D- z_~d)JFIBo^gk+3;ak$jz*X1yH_-lzE&Ir@h@PGMqYr#U5{AZ7D{j}Ji8~vL$fj{?^ zyt;QF()hK2{q?yo#zyk}fKM2*Th%GTR}DjzsUHCl1YG;mp8TK4qMt(XQu#SlH@BYFCU8=df8A4E-gr@4?8;QDpCBxgwj-RIum zwMgkPymONW>;fkeKh2U12o+DT)5#?><2&Ke3i=;I@T5gulQwOt-HF=GMH8KndrqB) z)0o4f1;kOP-wOT>d5@V8h8ydUN>W7pT*+l}XLv5$h(dGve$uZB<^h>5z8Gyd`Bipo z5jSWG^5Ze%8zj=^KvqeE=t z_%Wt)@Fa_9>Sj7|TCshx1b{w_2i$84F3M3w5N*ub66H7RDTAL|72I@nfceEwN%`8b zJL&FcM~}@=dI-+#DDVYMEiSnF1x-g_2ZLAbE~T6-ud)ln6xLk>aoYQVCbx>7OO8r* zc1F4z%%yF zi#5a=n#!1p%Sf8a8XHQ3hnS>^loZz3M4V{y-vX}Rpx=HaSA0qu`L9o|c&NmBLS^#P zV*MfwHouR-LAjcoJmt%U!4^}Gk7ajTH@|XV2(t?_&D23SM-ZS1C;(KA!e#TOF^?xO zVhN*gZ$9Zs{P;8R@(HFVnVZh^+dtm<(cj&zTG2r=|9!5y(Cvit4VU8HzMNlzao_Zp zScVs^6%=ks(K%pU1H_5AYZHWKv53mrH3+#C=ucmQwoco}Z77s^6qwVr3V#3%Z|40S z9|WSvc8aeTv}XON8SUm~4mu11Khf-2Pe?&-i+eUYvL62U2=+kC3VYzZYmtk;#V^d{ zPMiVe=Y08Mf}S;ZaJ02|`72FIQdV42S_Ug7L&SI`Q(?~O>$nBQhpZkyM&2NERiZHKG5x+yU_>}FczUSF)?0DJ zXh4E{9foVjJ;T;g@5C#zX*xz6&IR6lO&2gz$)B>DH0bIC%HdDhO+#$6Ewlsc_`pIV zi539t&u#8z(_2A6L=lBJ;gAUqn)v6VYrbiRpkAr+ZB0zQ3rwTZOc1kD!e~BzN*3+{ zftOdBzz79#7_DIB7z$C(yUQtI#@5RO|CNRHJ4^%}n;01jrbMvNjOgKT+p$^0Ivu11 zsht>MuP&l;1ilg&N>?<39X7pn9s59KnFa(|%AWu7`x zy1V!;@EG%>&vX{)d zX!{Wwef@muR>;JH?aHvwiT_*SQSc8!C0QIkGV0e@FJi2A(Mu% z@!?Ac2=_lXMNi~ z-nWh!NVADM7y8s*QrgMCwCw0)ZRuKi<+KOok8`7yO5M8%|u;!>tqBaqr_Dk&)` zYiKNKVk$$FG7&e$5{WXB|1I|CujJl^q~-s5`}3bB_s(Db38GBBEu;Ra%QgX~1159AM9ooMMUiiF@KdP=zy)sd7E-#yaEg_vNdZs2paDH>0H z!+Z1lM{^7ZCbWX3>UOce3q18GF732B_k|VAD$Ig8(iY8x^B{c~MC&JA#)%66S$I9o zpNppmAb|4EljD$^s@(%Ivt}Uhj>(CbW`7CH@*;yvM5ITLCP52H5N%hf#lQebG~P?M zKvd+@Vm8}^(-3gWCH&}F@X+cx=mnkp%b+VO+Bf9zr2Hj;FK~g_`-t zX_XV_0aNSV<^Z&Qo09Ap0b*dC(X(1e&U+B!7ryr_uq}di8(t09p8~-$6+RcrHR^ra z(UNh~mt=3uad$cx*Qo#8=g_oHSxX9zR*avffas2J-)2hsJa$7=kQV&3hdT-}z(pz^ zpl=|(smTuuX5BHOWJ0B^1}xAMFsm2FwD8JC9-Z#y62&-Mw6`C>@^Op$heLcpSE2-h z4jF>@O1at@&67GY@m2SC4wZ&dYkjV#8b7T6E(ceIwyR<#{H`tvZY=--{w#(lGRIfh zzH`j~*Ga+#4Ac{|_f*C>XkaP6VNb%%xK4U}8FfpsT9TqV&YGVyyTE;|Gb7uuaoAGN zBJs(w$$?^Zj>?DDh8=TZ_1DXtebfVebmr|HE%`J5Tg*AjJ@D}Yh?_4z#vE1@kWEYq zOO%#2k&u?e8H*Vj0>(*7;!LHmQqodlznrsQ&e?z3Ir|sVx#m5CJ(z0W+Ow0HE-|Dhw#I&k*DAAZYYNutA)0Ssqbj z5Xd%oeeJItV}705#}{Z7;pDHyf<$9sB$h;r18Kx_>^}IqXWN0_7eNca(7fDF1>2TV z2EIDt+a5=rji$S{W9C+YG$&B2@Y<*Ly%8ZdkRIk?*3RWFO5IOAT%N2Kfh!t+5q!S! zMj5@dM`7Rw$(a5G-}#_#KC23Zpq0xEaD*y-1>c05#m{HM4y48jG^g`9^*;66@$|T9 z6`0Q~ukY5!Cq)}T`m|~{h&kC}<}~=+{Rv#3^1XB<|IGJt5gznG%*R9#6;|><6^QxA zy3XR;$VWl2eq8Hw*ok5`4X7S)6&MTkyYX}`Dd*;k{6m0pd@D2nJN4l?X^XKmB)1Ju z+0X{(&Lw4&Xona9iFuX;13$7 zs;64G>zmA9O@H`fO>Dspk*6g>o)acAe1|x*xi9*RgITJzFI{F|x@8wPU+IGcr<(S# z)Xq*Y^R9n@llraA@;|)CEOz1=?1SnU!NS3O+4gw>!P?f?#Olu*G>vVYY^-dF#wNzQ zZEb9vKYeoaFvAj7=ZHQV#;MKsQT@%h<&LG{qU~T zFLgJtEG&e4e~;?oRu1KQhp{4eX*3q>4RUSwC?<|l%Gi^Tr6dAR>l`8BO*VN@^d*Fi=`}HKKml&DOgSW zR~FM2OIRG*|02O27?Xe9N&Sxl{fnh64#j+tB7pyql%EUCES9p^68<8ES@}m&e$Fml lEM>7q`XXiM$byu=*H1?eGXgsug4Tk^J#bUCnA-fi{|EWIIRF3v literal 0 HcmV?d00001 diff --git a/visionstate/backend/tests/backup_fixture.py b/visionstate/backend/tests/backup_fixture.py new file mode 100644 index 0000000..cc7d525 --- /dev/null +++ b/visionstate/backend/tests/backup_fixture.py @@ -0,0 +1,156 @@ +"""A full installation drawn by our own code, and a backup of it (backup.write). + +Used by tests/test_backup.py, and to add a backup made by this version to ``BACKUPS`` — run it +whenever ``db.SCHEMA_VERSION`` or ``settings.BACKUP["format"]`` goes up (the test +``test_a_backup_of_this_version_is_kept`` says so); every kept backup is restored by every later +version's tests: + + python -m tests.backup_fixture +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +from PIL import Image, ImageDraw + +BACKUPS = Path(__file__).parent / "assets" / "backups" + + +def picture(seed: int, size: tuple[int, int] = (160, 90)) -> Image.Image: + image = Image.new("RGB", size, (40 + seed * 30 % 200, 90, 120)) + ImageDraw.Draw(image).rectangle((20 + seed * 7 % 60, 20, 80 + seed * 7 % 60, 70), fill=(220, 210, 60)) + return image + + +def populate(settings) -> dict: # noqa: ANN001 + """One sensor of each kind with what each keeps (training images, a taught box, a person with + a face, history, a counter's watch, a model file), the way the app stores them. Returns + what is in it.""" + from visionstate.db import ( + Database, + Face, + ModelInfo, + Person, + Prediction, + ReadingStat, + ReadingWatch, + Sample, + SampleLabel, + Sensor, + State, + ) + from visionstate.storage import Storage + + settings.ensure_dirs() + db = Database(settings.db_path) + db.init() + storage = Storage(settings) + common = {"interval_s": 60.0, "threshold": 0.7, "debounce": 2, "source_type": "http"} + try: + with db.session() as s: + door = Sensor(slug="garage_door", name="Garage door", source="rtsp://admin:secret@10.0.0.9/live", **common) + door.states = [ + State(key="open", name="Open", color="#2e7d32", position=0), + State(key="closed", name="Closed", color="#c62828", position=1), + ] + drive = Sensor( + slug="driveway", + name="Driveway", + kind="objects", + source="http://cam.local/snap.jpg", + objects={"classes": ["person", "car"], "faces": {"enabled": True}}, + **common, + ) + meter = Sensor( + slug="water_meter", + name="Water meter", + kind="reading", + source="http://cam.local/meter.jpg?user=me&password=pw", + reading={"mode": "counter", "display": "counter", "digits": 7, "decimals": 3}, + **common, + ) + s.add_all([door, drive, meter]) + s.flush() + for n in range(4): + sample = Sample(sensor_id=door.id, filename=storage.save_sample(door.id, picture(n)), origin="snapshot") + s.add(sample) + s.flush() + s.add(SampleLabel(sample_id=sample.id, state_id=door.states[n % 2].id)) + s.add( + Sample( + sensor_id=drive.id, + filename=storage.save_sample(drive.id, picture(9)), + origin="live", + object_label="car", + detected="car", + box=[0.1, 0.2, 0.5, 0.6], + score=0.8, + ) + ) + emma = Person(key="emma", name="Emma") + s.add(emma) + s.flush() + face = {"box": [10.0, 10.0, 40.0, 40.0], "points": [[20.0, 20.0]] * 5, "score": 0.9} + s.add( + Face( + person_id=emma.id, + sensor_id=drive.id, + filename=storage.save_face(picture(5, (64, 64))), + origin="live", + face=face, + ) + ) + for n, (sensor, key) in enumerate(((door, "open"), (drive, "car"), (meter, "reading"))): + s.add( + Prediction( + sensor_id=sensor.id, + state_key=key, + published_key=key if key != "reading" else "89939.512", + confidence=0.9, + probs={}, + frame=storage.save_history(sensor.id, picture(20 + n)), + is_change=True, + ) + ) + s.add(ReadingStat(sensor_id=meter.id, day="2026-10-01", reads=10, accepted=9, rejected={"too_high": 1})) + s.add(ReadingWatch(sensor_id=meter.id, value="89939.512", seen_at=1.0, accepted_at=1.0)) + s.add(ModelInfo(sensor_id=door.id, backbone="test", n_samples=4)) + ids = {"door": door.id, "drive": drive.id, "meter": meter.id} + db.set_setting("storage", {"history_days": 9}) + (settings.heads_dir / f"{ids['door']}.joblib").write_bytes(b"head") + (settings.models_dir / "downloaded.onnx").write_bytes(b"model") + finally: + db.engine.dispose() + return {"slugs": {"garage_door", "driveway", "water_meter"}, "people": {"emma"}, "ids": ids} + + +def make(target: Path) -> Path: + """A backup of ``populate`` in a fresh folder, with everything and without camera passwords.""" + import tempfile + + from visionstate import backup + from visionstate.settings import BACKUP_PARTS, Settings + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + settings = Settings( + data_dir=root / "data", media_dir=root / "media", frontend_dir=root, bundled_models_dir=root + ) + populate(settings) + with target.open("wb") as out: + backup.write(settings, out, set(BACKUP_PARTS), credentials=False) + return target + + +if __name__ == "__main__": + from visionstate.db import SCHEMA_VERSION + from visionstate.settings import BACKUP, VERSION + + BACKUPS.mkdir(parents=True, exist_ok=True) + out = BACKUPS / f"backup-format{BACKUP['format']}-schema{SCHEMA_VERSION}.zip" + if out.exists(): + sys.exit(f"{out.name} is kept already (made by an earlier version): never change a kept backup") + make(out) + print(f"{out} (VisionState {VERSION}, {out.stat().st_size} bytes)", file=sys.stderr) diff --git a/visionstate/backend/tests/test_backup.py b/visionstate/backend/tests/test_backup.py new file mode 100644 index 0000000..94b4147 --- /dev/null +++ b/visionstate/backend/tests/test_backup.py @@ -0,0 +1,448 @@ +"""Backup of everything and restoring it (backup.py, api/backup.py).""" + +from __future__ import annotations + +import io +import json +import sqlite3 +import zipfile +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import select + +from visionstate import backup +from visionstate.db import SCHEMA_VERSION, Base, Database, Face, Prediction, Sample, Sensor +from visionstate.main import create_app +from visionstate.mqtt import MqttBridge +from visionstate.settings import BACKUP, BACKUP_LEFT_OUT, BACKUP_PARTS, Settings +from visionstate.storage import Storage + +from .backup_fixture import BACKUPS, make, populate +from .conftest import MODEL_DIR + + +def make_settings(root: Path) -> Settings: + return Settings( + data_dir=root / "data", media_dir=root / "media", frontend_dir=root / "none", bundled_models_dir=MODEL_DIR + ) + + +@pytest.fixture +def removed(monkeypatch): + """The sensors (slugs) and people (keys) whose entities a restore took away.""" + gone: dict[str, list[str]] = {"sensors": [], "people": [], "classes": []} + + async def remove_discovery(self, sensor): + gone["sensors"].append(sensor.slug) + + async def remove_person(self, key): + gone["people"].append(key) + + async def remove_object_class(self, slug, key, discovery=True): + gone["classes"].append(f"{slug}/{key}") + + monkeypatch.setattr(MqttBridge, "remove_discovery", remove_discovery) + monkeypatch.setattr(MqttBridge, "remove_object_class", remove_object_class) + monkeypatch.setattr(MqttBridge, "remove_person", remove_person) + return gone + + +def files_listed(settings: Settings) -> dict[str, int]: + """Rows of each kind whose file is there (a restore that loses a file shows up here).""" + storage = Storage(settings) + db = Database(settings.db_path) + try: + with db.session() as s: + samples = s.scalars(select(Sample)).all() + faces = s.scalars(select(Face)).all() + frames = s.scalars(select(Prediction).where(Prediction.frame.is_not(None))).all() + return { + "samples": sum(storage.sample_path(r.sensor_id, r.filename).exists() for r in samples), + "samples_missing": sum(not storage.sample_path(r.sensor_id, r.filename).exists() for r in samples), + "faces": sum(storage.face_path(r.filename).exists() for r in faces), + "faces_missing": sum(not storage.face_path(r.filename).exists() for r in faces), + "history": sum(storage.history_path(r.sensor_id, r.frame).exists() for r in frames if r.frame), + "history_missing": sum( + not storage.history_path(r.sensor_id, r.frame).exists() for r in frames if r.frame + ), + } + finally: + db.engine.dispose() + + +# --- Nothing added later is missing from a backup ------------------------------------------- + + +def test_every_table_belongs_to_one_part(): + """A new table in db.py must be put in a part of settings.BACKUP_PARTS (the database goes + along whole; the part says when its rows are left out).""" + listed = [table for part in BACKUP_PARTS.values() for table in part["tables"]] + assert sorted(listed) == sorted(set(listed)), "a table is in two parts" + assert set(listed) == set(Base.metadata.tables), ( + "every table of db.py belongs to exactly one part of settings.BACKUP_PARTS" + ) + + +def test_every_folder_belongs_to_one_part(tmp_path): + """A new folder in Settings (under the data or media folder) must be in a part of + settings.BACKUP_PARTS, or in BACKUP_LEFT_OUT when it is made again by itself.""" + settings = make_settings(tmp_path) + roots = (settings.data_dir, settings.media_dir) + names = [ + name + for name in dir(Settings) + if isinstance(getattr(Settings, name), property) + and isinstance(path := getattr(settings, name), Path) + and any(root in path.parents for root in roots) + and name != "db_path" + ] + listed = [name for part in BACKUP_PARTS.values() for name in part["folders"]] + list(BACKUP_LEFT_OUT) + assert sorted(listed) == sorted(set(listed)), "a folder is listed twice" + assert set(names) == set(listed), "every folder of Settings is in settings.BACKUP_PARTS or BACKUP_LEFT_OUT" + + +def test_after_populating_every_file_is_in_a_known_folder(tmp_path): + """Files written anywhere else in the data or media folder would not be in a backup.""" + settings = make_settings(tmp_path) + populate(settings) + known = [p.resolve() for p in backup.folders(settings).values()] + db_files = {p.name for p in (settings.db_path, settings.db_path.with_suffix(".db-wal"))} + for root in (settings.data_dir, settings.media_dir): + for path in root.rglob("*"): + if path.is_file() and path.name not in db_files and not path.name.endswith("-shm"): + assert any(folder in path.resolve().parents for folder in known), path + + +# --- Export --------------------------------------------------------------------------------- + + +def test_snapshot_leaves_out_parts_and_masks_passwords(tmp_path): + settings = make_settings(tmp_path) + populate(settings) + out = io.BytesIO() + backup.write(settings, out, {"sensors", "samples"}, credentials=False) + archive = zipfile.ZipFile(out) + manifest = json.loads(archive.read("backup.json")) + assert manifest["parts"] == ["samples", "sensors"] + assert manifest["schema"] == SCHEMA_VERSION and manifest["format"] == BACKUP["format"] + assert sorted(manifest["masked"]) == ["Garage door", "Water meter"] + names = archive.namelist() + assert any(n.startswith("files/samples_dir/") for n in names) + assert any(n.startswith("files/heads_dir/") for n in names) # the sensors' part: always + assert not any(n.startswith(("files/history_dir/", "files/faces_dir/", "files/models_dir/")) for n in names) + db_copy = tmp_path / "copy.db" + db_copy.write_bytes(archive.read("visionstate.db")) + con = sqlite3.connect(db_copy) + try: + assert con.execute("SELECT count(*) FROM prediction").fetchone()[0] == 0 + assert con.execute("SELECT count(*) FROM person").fetchone()[0] == 0 + assert con.execute("SELECT count(*) FROM sample").fetchone()[0] == 5 + sources = {r[0] for r in con.execute("SELECT source FROM sensor")} + assert "rtsp://***@10.0.0.9/live" in sources and not any("secret" in s for s in sources) + finally: + con.close() + # With the passwords when asked for. + out = io.BytesIO() + backup.write(settings, out, set(BACKUP_PARTS), credentials=True) + archive = zipfile.ZipFile(out) + assert json.loads(archive.read("backup.json"))["masked"] == [] + assert any(n.startswith("files/models_dir/") for n in archive.namelist()) + + +def test_training_images_moved_already_are_not_in_twice(tmp_path): + settings = make_settings(tmp_path) + populate(settings) + moved = next(settings.samples_dir.rglob("*.jpg")) + old = settings.old_samples_dir / moved.relative_to(settings.samples_dir) + old.parent.mkdir(parents=True) + old.write_bytes(moved.read_bytes()) # the copy older versions still find + (settings.old_samples_dir / "9").mkdir() + (settings.old_samples_dir / "9" / "only-here.jpg").write_bytes(b"x") + out = io.BytesIO() + backup.write(settings, out, set(BACKUP_PARTS), credentials=False) + names = zipfile.ZipFile(out).namelist() + assert "files/old_samples_dir/9/only-here.jpg" in names + assert not any(n.startswith("files/old_samples_dir/") and n.endswith(moved.name) for n in names) + + +def test_export_is_streamed_and_checks_its_parts(tmp_path): + settings = make_settings(tmp_path) + populate(settings) + with TestClient(create_app(settings)) as client: + info = client.get("/api/v1/backup").json() + assert set(info["parts"]) == set(BACKUP_PARTS) and info["parts"]["samples"] > 0 and info["free"] > 0 + cfg = client.get("/api/v1/config").json()["backup"] + assert [p["key"] for p in cfg["parts"]] == list(BACKUP_PARTS) and cfg["credentials"] is False + assert client.get("/api/v1/backup/export?leave_out=sensors").status_code == 422 + assert client.get("/api/v1/backup/export?leave_out=nonsense").status_code == 422 + resp = client.get("/api/v1/backup/export?leave_out=history,models") + assert resp.status_code == 200 + assert "visionstate-backup-" in resp.headers["content-disposition"] + manifest = json.loads(zipfile.ZipFile(io.BytesIO(resp.content)).read("backup.json")) + assert manifest["parts"] == ["faces", "samples", "sensors"] + assert [s["name"] for s in manifest["sensors"]] == ["Garage door", "Driveway", "Water meter"] + + +def test_a_stopped_download_ends_the_writer(tmp_path): + settings = make_settings(tmp_path) + populate(settings) + pieces = backup.stream(settings, set(BACKUP_PARTS), False) + next(pieces) + pieces.close() # the reader went away: the thread stops instead of waiting forever + + +# --- Restore -------------------------------------------------------------------------------- + + +def restore(client, data: bytes): + return client.post("/api/v1/backup/restore", content=data, headers={"content-type": "application/zip"}) + + +def test_restore_replaces_everything(tmp_path, removed): + source = make_settings(tmp_path / "a") + populate(source) + with TestClient(create_app(source)) as client: + data = client.get("/api/v1/backup/export").content # the passwords are masked + target = make_settings(tmp_path / "b") + with TestClient(create_app(target)) as client: + made = client.post( + "/api/v1/sensors", + json={ + "name": "Old one", + "source_type": "http", + "source": "http://x", + "states": [{"name": "A"}, {"name": "B"}], + }, + ) + assert made.status_code == 201, made.text + # The same sensor (slug) as in the backup, looking for something else: only that goes. + same = client.post( + "/api/v1/sensors", + json={ + "name": "Driveway", + "kind": "objects", + "source_type": "http", + "source": "http://x", + "objects": {"classes": ["car", "dog"]}, + }, + ) + assert same.status_code == 201, same.text + (target.models_dir / "here-already.onnx").write_bytes(b"kept") + (target.sample_thumbs_dir / "1.jpg").write_bytes(b"stale thumbnail") + before = client.app.state.runtime + assert before.embedder is not None + resp = restore(client, data) + assert resp.status_code == 200, resp.text + result = resp.json() + assert result["sensors"] == 3 and result["people"] == 1 + assert [m["name"] for m in result["masked"]] == ["Garage door", "Water meter"] + assert removed["sensors"] == ["old_one"] # its entities go; the others keep theirs + assert removed["classes"] == ["driveway/dog"] + sensors = client.get("/api/v1/sensors").json() + assert sorted(s["name"] for s in sensors) == ["Driveway", "Garage door", "Water meter"] + assert client.get("/api/v1/status").status_code == 200 + assert client.get("/api/v1/people").json()["people"][0]["name"] == "Emma" + rt = client.app.state.runtime + assert set(rt._tasks) == {s["id"] for s in sensors} # the restored sensors run + # The old runtime let go of its models: two sets at once ran a small machine out of memory. + assert rt is not before and rt.embedder is not None + assert before.embedder is None and before.detector is None and not before.heads + assert not any(live.frames for live in before.live.values()) + assert files_listed(target) == { + "samples": 5, + "samples_missing": 0, + "faces": 1, + "faces_missing": 0, + "history": 3, + "history_missing": 0, + } + assert (target.models_dir / "here-already.onnx").exists() and (target.models_dir / "downloaded.onnx").exists() + assert not (target.sample_thumbs_dir / "1.jpg").exists() + assert (target.heads_dir / "1.joblib").exists() # restored (and retrained here: the test file is no model) + assert not backup.stage_dir(target).exists() + assert not list(target.data_dir.glob("*.restore-old")) and not list(target.media_dir.glob("*.restore-old")) + + +def test_restore_without_history_removes_the_history_here(tmp_path, removed): + source = make_settings(tmp_path / "a") + populate(source) + with TestClient(create_app(source)) as client: + data = client.get("/api/v1/backup/export?leave_out=history,faces").content + # Into the same app: its own history is gone with the rows that listed it. + assert restore(client, data).status_code == 200 + assert client.get("/api/v1/people").json()["people"] == [] + assert removed["people"] == ["emma"] + assert files_listed(source)["history"] == 0 + assert not any(source.history_dir.rglob("*.jpg")) + assert files_listed(source)["samples"] == 5 + + +def test_restore_refuses_what_it_cannot_restore(tmp_path): + settings = make_settings(tmp_path) + populate(settings) + good = make(tmp_path / "good.zip").read_bytes() + + def changed(**fields) -> bytes: + out = io.BytesIO() + with zipfile.ZipFile(io.BytesIO(good)) as src, zipfile.ZipFile(out, "w") as dst: + for item in src.infolist(): + body = src.read(item) + if item.filename == "backup.json": + body = json.dumps({**json.loads(body), **fields}).encode() + dst.writestr(item, body) + return out.getvalue() + + with TestClient(create_app(settings)) as client: + before = sorted(s["name"] for s in client.get("/api/v1/sensors").json()) + newer_db = restore(client, changed(schema=SCHEMA_VERSION + 1, app_version="9.9.9")) + assert newer_db.status_code == 400 and "newer VisionState (9.9.9" in newer_db.json()["detail"] + newer = restore(client, changed(format=BACKUP["format"] + 1)) + assert newer.status_code == 400 and "newer" in newer.json()["detail"] + assert restore(client, b"not a zip").status_code == 400 + bundle = io.BytesIO() + with zipfile.ZipFile(bundle, "w") as z: + z.writestr("manifest.json", "{}") # a sensor bundle, not a backup + assert "not a VisionState backup" in restore(client, bundle.getvalue()).json()["detail"] + assert sorted(s["name"] for s in client.get("/api/v1/sensors").json()) == before + assert not backup.stage_dir(settings).exists() + + +def test_restore_ignores_paths_outside_its_folders(tmp_path): + settings = make_settings(tmp_path) + data = make(tmp_path / "b.zip").read_bytes() + out = io.BytesIO() + with zipfile.ZipFile(io.BytesIO(data)) as src, zipfile.ZipFile(out, "w") as dst: + for item in src.infolist(): + dst.writestr(item, src.read(item)) + dst.writestr("files/samples_dir/../../../escaped.txt", "x") + dst.writestr("files/thumbs_dir/1.jpg", "x") # never in a backup + dst.writestr("files/somewhere/x.txt", "x") + dst.writestr("/etc/x", "x") + with TestClient(create_app(settings)) as client: + assert restore(client, out.getvalue()).status_code == 200 + assert not (tmp_path / "escaped.txt").exists() and not list(tmp_path.rglob("escaped.txt")) + assert not (settings.thumbs_dir / "1.jpg").exists() + + +def test_a_failed_start_puts_the_old_data_back(tmp_path, monkeypatch): + settings = make_settings(tmp_path) + data = make(tmp_path / "b.zip").read_bytes() + from visionstate.engine import Runtime + + original = Runtime.start + calls = {"n": 0} + + async def start(self): + calls["n"] += 1 + if calls["n"] == 2: # the first start after the swap + raise RuntimeError("broken") + await original(self) + + monkeypatch.setattr(Runtime, "start", start) + with TestClient(create_app(settings)) as client: + client.post( + "/api/v1/sensors", + json={ + "name": "Mine", + "source_type": "http", + "source": "http://x", + "states": [{"name": "A"}, {"name": "B"}], + }, + ) + resp = restore(client, data) + assert resp.status_code == 500 and "broken" in resp.json()["detail"] + assert [s["name"] for s in client.get("/api/v1/sensors").json()] == ["Mine"] + assert not list(settings.data_dir.glob("*.restore-old")) + + +def test_a_restore_cut_off_is_undone_at_start(tmp_path): + settings = make_settings(tmp_path) + populate(settings) + staged = backup.stage(settings, make(tmp_path / "b.zip")) + # Cut off halfway: the old database is aside, the new one not yet in place. + settings.db_path.rename(settings.db_path.with_name("visionstate.db.restore-old")) + settings.history_dir.rename(settings.history_dir.with_name("history.restore-old")) + assert staged.root.exists() + backup.clean_leftovers(settings) + assert settings.db_path.exists() and any(settings.history_dir.rglob("*.jpg")) + assert not backup.stage_dir(settings).exists() + # Cut off after the restored data started: what was kept aside goes. + aside = settings.history_dir.with_name("history.restore-old") + aside.mkdir() + (backup.stage_dir(settings)).mkdir() + (backup.stage_dir(settings) / backup.DONE).touch() + backup.clean_leftovers(settings) + assert not aside.exists() and any(settings.history_dir.rglob("*.jpg")) + + +def test_a_newer_database_keeps_its_version(tmp_path): + """An older version on a newer database (a rollback) must not lower its version: the next + update would run its data migrations again.""" + path = tmp_path / "x.db" + Database(path).init() + con = sqlite3.connect(path) + con.execute(f"PRAGMA user_version = {SCHEMA_VERSION + 3}") + con.commit() + con.close() + Database(path).init() + con = sqlite3.connect(path) + assert con.execute("PRAGMA user_version").fetchone()[0] == SCHEMA_VERSION + 3 + con.close() + + +# --- Every released backup format restores in this version ---------------------------------- + +KEPT = sorted(BACKUPS.glob("*.zip")) + + +def test_a_backup_of_this_version_is_kept(): + """When the database version or the backup format goes up, a backup made by it is added, so + every later version is tested against it: run `python -m tests.backup_fixture`.""" + made = {(m["format"], m["schema"]) for m in (json.loads(zipfile.ZipFile(p).read("backup.json")) for p in KEPT)} + assert (BACKUP["format"], SCHEMA_VERSION) in made, "run `python -m tests.backup_fixture` and commit the file" + + +@pytest.mark.parametrize("path", KEPT, ids=[p.name for p in KEPT]) +def test_every_kept_backup_restores(tmp_path, path, removed): + manifest = json.loads(zipfile.ZipFile(path).read("backup.json")) + settings = make_settings(tmp_path) + with TestClient(create_app(settings)) as client: + resp = restore(client, path.read_bytes()) + assert resp.status_code == 200, resp.text + sensors = client.get("/api/v1/sensors").json() + assert sorted(s["name"] for s in sensors) == sorted(s["name"] for s in manifest["sensors"]) + for sensor in sensors: + assert client.get(f"/api/v1/sensors/{sensor['id']}").status_code == 200 + assert client.get("/api/v1/history").status_code == 200 + listed = files_listed(settings) + assert listed["samples_missing"] == listed["faces_missing"] == listed["history_missing"] == 0 + assert listed["samples"] > 0 and listed["faces"] > 0 and listed["history"] > 0 + con = sqlite3.connect(settings.db_path) + try: + assert con.execute("PRAGMA user_version").fetchone()[0] == SCHEMA_VERSION + assert con.execute("SELECT count(*) FROM sensor").fetchone()[0] == len(manifest["sensors"]) + finally: + con.close() + with Database(settings.db_path).session() as s: + assert len(s.scalars(select(Sensor)).all()) == len(manifest["sensors"]) + + +def test_requests_wait_while_the_data_is_swapped(tmp_path): + """The UI keeps polling during a restore: its requests wait for the new runtime instead of failing.""" + import threading + import time + + with TestClient(create_app(make_settings(tmp_path))) as client: + ready = client.app.state.ready + assert client.portal is not None + client.portal.call(ready.clear) + answers = [] + asking = threading.Thread(target=lambda: answers.append(client.get("/api/v1/status").status_code)) + asking.start() + time.sleep(0.5) + assert answers == [] # still waiting + client.portal.call(ready.set) + asking.join(10) + assert answers == [200] diff --git a/visionstate/backend/visionstate/api/backup.py b/visionstate/backend/visionstate/api/backup.py new file mode 100644 index 0000000..f4965ac --- /dev/null +++ b/visionstate/backend/visionstate/api/backup.py @@ -0,0 +1,199 @@ +"""Backup of everything and restoring one (backup.py; the parts are settings.BACKUP_PARTS).""" + +from __future__ import annotations + +import asyncio +import logging +import shutil +import zipfile +from datetime import datetime + +from fastapi import APIRouter, HTTPException, Request +from fastapi.responses import StreamingResponse +from sqlalchemy import select + +from .. import backup +from ..db import Database, Sensor +from ..engine import Runtime +from ..engine.state import SensorConfig +from ..mqtt import SensorDescriptor +from ..redact import redact +from ..settings import BACKUP, BACKUP_PARTS +from .common import API_PREFIX, runtime + +log = logging.getLogger(__name__) +router = APIRouter(prefix=API_PREFIX, tags=["backup"]) + + +@router.get("/backup") +async def backup_info(request: Request) -> dict: + """Bytes on disk per part and the free space here (a restore needs about twice its size).""" + settings = runtime(request).settings + sizes = await asyncio.to_thread(backup.sizes, settings) + return {"parts": sizes, "free": shutil.disk_usage(settings.data_dir).free} + + +@router.get("/backup/export") +async def backup_export(request: Request, leave_out: str = "", credentials: bool = False) -> StreamingResponse: + """Everything as one ZIP, streamed while it is written; ``leave_out``: parts not wanted + (comma-separated), ``credentials``: with the camera passwords.""" + left_out = {part for part in leave_out.split(",") if part} + unknown = left_out - set(BACKUP_PARTS) + if unknown: + raise HTTPException(422, f"Unknown parts: {', '.join(sorted(unknown))}") + if left_out & backup.required_parts(): + raise HTTPException(422, "The sensors are always in a backup") + parts = set(BACKUP_PARTS) - left_out + settings = runtime(request).settings + filename = f"visionstate-backup-{datetime.now().strftime('%Y-%m-%d-%H%M')}.zip" + return StreamingResponse( + backup.stream(settings, parts, credentials), + media_type="application/zip", + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + + +@router.post("/backup/restore") +async def backup_restore(request: Request) -> dict: + """Replaces everything with a backup (the request body is the ZIP): staged and checked + first, then swapped in while the runtime is stopped; on any error the data from before stays.""" + app = request.app + if getattr(app.state, "restoring", False): + raise HTTPException(409, "A restore is already running") + app.state.restoring = True + try: + return await _restore(request) + finally: + app.state.restoring = False + app.state.ready.set() + + +async def _restore(request: Request) -> dict: + old: Runtime = runtime(request) + settings = old.settings + limit = BACKUP["max_restore_gb"] * 1_000_000_000 + length = int(request.headers.get("content-length") or 0) + if length > limit: + raise HTTPException(413, f"The backup is larger than {BACKUP['max_restore_gb']} GB") + # The upload, then what it holds, then the old data kept aside until the new runs. + need = 2 * length + BACKUP["free_margin_mb"] * 1_000_000 + if length and shutil.disk_usage(settings.data_dir).free < need: + raise HTTPException( + 507, f"Not enough free disk space: a restore of this backup needs about {need // 1_000_000} MB" + ) + folder = backup.stage_dir(settings) + await asyncio.to_thread(shutil.rmtree, folder, True) + upload = folder / "upload.zip" + try: + folder.mkdir(parents=True) + size = 0 + with upload.open("wb") as file: + async for chunk in request.stream(): + size += len(chunk) + if size > limit: + raise HTTPException(413, f"The backup is larger than {BACKUP['max_restore_gb']} GB") + await asyncio.to_thread(file.write, chunk) + try: + staged = await asyncio.to_thread(backup.stage, settings, upload) + except (ValueError, KeyError, OSError, zipfile.BadZipFile) as err: + raise HTTPException(400, f"Restore failed: {redact(str(err))}") from err + upload.unlink(missing_ok=True) + await _swap(request, old, staged) + finally: + await asyncio.to_thread(shutil.rmtree, folder, True) + manifest = staged.manifest + return { + "sensors": len(staged.slugs), + "people": len(staged.people), + "parts": manifest.get("parts", []), + "app_version": manifest.get("app_version"), + "exported_at": manifest.get("exported_at"), + "masked": staged.masked, + } + + +async def _swap(request: Request, old: Runtime, staged: backup.Staged) -> None: + """Stops the runtime, swaps the data and starts a runtime on it; puts the old data back and + starts on that when anything fails.""" + app = request.app + settings = old.settings + # The entities of sensors and people the backup does not have go away in Home Assistant; + # the others keep their unique IDs, and with them their entity IDs. + before = await asyncio.to_thread(_descriptors, old.db) + after = await asyncio.to_thread(_staged_descriptors, staged) + for slug, descriptor in before.items(): + if slug in after: + await old.mqtt.remove_stale(descriptor, after[slug]) + else: + await old.mqtt.remove_discovery(descriptor) + for key, _ in await asyncio.to_thread(old._people): + if key not in staged.people: + await old.mqtt.remove_person(key) + app.state.ready.clear() # other requests wait for the new runtime + await old.stop() + await _closed(old.db) + await asyncio.to_thread(old.release) # the new runtime loads the models again + try: + moved = await asyncio.to_thread(backup.swap, settings, staged) + except Exception as err: + log.exception("Restore: swapping the data failed; the data from before is kept") + await _start(app, settings) + raise HTTPException(500, f"Restore failed: {redact(str(err))}") from err + try: + await _start(app, settings) + except Exception as err: + log.exception("Restore: the restored data did not start; the data from before is put back") + await _stop(app) + await asyncio.to_thread(backup.undo, settings, moved) + await _start(app, settings) + raise HTTPException(500, f"Restore failed: {redact(str(err))}") from err + await asyncio.to_thread(backup.discard, settings, moved) + log.info("Restored a backup made by VisionState %s", staged.manifest.get("app_version")) + + +def _descriptors(db: Database) -> dict[str, SensorDescriptor]: + """Each sensor's entities, by slug.""" + with db.session() as s: + return {row.slug: SensorConfig.from_row(row).descriptor for row in s.scalars(select(Sensor))} + + +def _staged_descriptors(staged: backup.Staged) -> dict[str, SensorDescriptor]: + db = Database(staged.root / backup.DATABASE) + try: + return _descriptors(db) + finally: + db.engine.dispose() + backup.single_file(staged.root / backup.DATABASE) + + +async def _start(app, settings) -> None: # noqa: ANN001 + db = Database(settings.db_path) + try: + db.init() + rt = Runtime(settings, db) + app.state.runtime = rt + await rt.start() + except Exception: + db.engine.dispose() + raise + finally: + app.state.ready.set() + + +async def _stop(app) -> None: # noqa: ANN001 + rt: Runtime = app.state.runtime + app.state.ready.clear() + await rt.stop() + await _closed(rt.db) + await asyncio.to_thread(rt.release) + + +async def _closed(db: Database) -> None: + """Closes a stopped runtime's database and waits for the work still running in its threads + (a training, a clean-up) to let go of it.""" + in_use = db.close() + for _ in range(int(BACKUP["release_wait_s"] / 0.1)): + if in_use() == 0: + return + await asyncio.sleep(0.1) + log.warning("Restore: %d database connections still in use after %s s", in_use(), BACKUP["release_wait_s"]) diff --git a/visionstate/backend/visionstate/api/system.py b/visionstate/backend/visionstate/api/system.py index f43adbb..340a578 100644 --- a/visionstate/backend/visionstate/api/system.py +++ b/visionstate/backend/visionstate/api/system.py @@ -17,6 +17,8 @@ from ..db import Person, Prediction, Sensor from ..redact import redact from ..settings import ( + BACKUP, + BACKUP_PARTS, COUNTER_LIMIT, COUNTER_READERS, FACE_DEFAULTS, @@ -108,6 +110,11 @@ def ui_config() -> dict: "reminder_defaults": REMINDER_DEFAULTS, "reminder_limits": REMINDER_LIMITS, "sensor_kinds": SENSOR_KINDS, + "backup": { + "parts": [{"key": k, "optional": v["optional"]} for k, v in BACKUP_PARTS.items()], + "credentials": BACKUP["credentials"], + "max_restore_gb": BACKUP["max_restore_gb"], + }, "object_sensor_defaults": OBJECT_SENSOR_DEFAULTS, "object_defaults": OBJECT_DEFAULTS, "object_limits": OBJECT_LIMITS, diff --git a/visionstate/backend/visionstate/backup.py b/visionstate/backend/visionstate/backup.py new file mode 100644 index 0000000..b15edc8 --- /dev/null +++ b/visionstate/backend/visionstate/backup.py @@ -0,0 +1,454 @@ +"""A backup of everything (export) and restoring one, which replaces everything. + +What goes in is declared in ``settings.BACKUP_PARTS``: the database goes along whole (a part left +out has its rows deleted from the copy), each part's folders file by file. The archive: +``backup.json`` (``Manifest``), ``visionstate.db`` and ``files//``. + +A restore is staged first (``stage``: checked, extracted and the database migrated, all next to +the live data), then swapped in (``swap``) while the runtime is stopped (api/backup.py); the old +files are kept aside until the new runtime has started, and put back if anything fails. +""" + +from __future__ import annotations + +import json +import logging +import queue +import shutil +import sqlite3 +import tempfile +import threading +import time +import zipfile +from collections.abc import Iterator +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path, PurePosixPath +from typing import BinaryIO + +from .db import SCHEMA_VERSION, Database +from .redact import MASK, redact +from .settings import BACKUP, BACKUP_LEFT_OUT, BACKUP_PARTS, VERSION, Settings + +log = logging.getLogger(__name__) + +MANIFEST = "backup.json" +DATABASE = "visionstate.db" +FILES = "files/" +STAGE = ".restore" # in the data folder: the uploaded archive and what it holds +ASIDE = ".restore-old" # suffix of a folder kept aside during a swap +RENAME_TRIES = 25 # of 0.2 s each +DONE = "done" # in STAGE: the restored data started, what was kept aside can go +# Training images older versions kept in the media folder: copies already moved are left out. +SHADOWED = {"old_samples_dir": "samples_dir"} + + +def required_parts() -> set[str]: + return {name for name, part in BACKUP_PARTS.items() if not part["optional"]} + + +def folders(settings: Settings) -> dict[str, Path]: + """Every folder a backup knows, by its Settings name: those of the parts, then the left-out ones.""" + names = [name for part in BACKUP_PARTS.values() for name in part["folders"]] + return {name: getattr(settings, name) for name in (*names, *BACKUP_LEFT_OUT)} + + +def _files(settings: Settings, name: str) -> Iterator[tuple[Path, str]]: + """(file, path inside the folder) of one folder of a part, sorted.""" + root = getattr(settings, name) + if not root.is_dir(): + return + shadow = getattr(settings, SHADOWED[name]) if name in SHADOWED else None + if shadow is not None and root.resolve() == shadow.resolve(): + return # the media folder set to the app's own storage: the same files + for path in sorted(root.rglob("*")): + if not path.is_file() or path.is_symlink(): + continue + rel = path.relative_to(root).as_posix() + if shadow is not None and (shadow / rel).exists(): + continue + yield path, rel + + +def sizes(settings: Settings) -> dict[str, int]: + """Bytes on disk per part (the sensors part: with the database).""" + result = {} + for name, part in BACKUP_PARTS.items(): + total = sum(path.stat().st_size for folder in part["folders"] for path, _ in _files(settings, folder)) + if not part["optional"]: + total += sum(p.stat().st_size for p in _db_files(settings.db_path) if p.exists()) + result[name] = total + return result + + +def _db_files(db_path: Path) -> list[Path]: + return [db_path, db_path.with_name(db_path.name + "-wal"), db_path.with_name(db_path.name + "-shm")] + + +# --- Export --------------------------------------------------------------------------------- + + +def snapshot(db_path: Path, target: Path, parts: set[str], credentials: bool) -> dict: + """A consistent copy of the live database (SQLite's backup, safe while it is written to) + without the rows of the parts left out; camera credentials masked unless ``credentials``. + Returns what the manifest says about it.""" + source = sqlite3.connect(db_path) + copy = sqlite3.connect(target) + try: + source.backup(copy) + finally: + source.close() + try: + copy.execute("PRAGMA foreign_keys=OFF") + for name, part in BACKUP_PARTS.items(): + if name not in parts: + for table in part["tables"]: + copy.execute(f'DELETE FROM "{table}"') + masked = [] + for sensor_id, name, source_url in copy.execute("SELECT id, name, source FROM sensor").fetchall(): + if not credentials and redact(source_url) != source_url: + copy.execute("UPDATE sensor SET source = ? WHERE id = ?", (redact(source_url), sensor_id)) + masked.append(name) + sensors = [{"name": n, "kind": k} for n, k in copy.execute("SELECT name, kind FROM sensor ORDER BY id")] + people = copy.execute("SELECT count(*) FROM person").fetchone()[0] + schema = copy.execute("PRAGMA user_version").fetchone()[0] + copy.commit() + copy.execute("PRAGMA journal_mode=DELETE") # one file, no -wal next to it + copy.execute("VACUUM") + finally: + copy.close() + return {"schema": schema, "sensors": sensors, "people": people, "masked": masked} + + +def write(settings: Settings, out: BinaryIO, parts: set[str], credentials: bool) -> None: + """Writes a backup with ``parts`` (the required ones always) to ``out``, which need not be + seekable (a download is streamed).""" + parts = (parts | required_parts()) & set(BACKUP_PARTS) + with tempfile.TemporaryDirectory(prefix="visionstate-backup-") as tmp: + db_copy = Path(tmp) / DATABASE + about = snapshot(settings.db_path, db_copy, parts, credentials) + manifest = { + "format": BACKUP["format"], + "app_version": VERSION, + "schema": about["schema"], + "exported_at": datetime.now(UTC).isoformat(), + "parts": sorted(parts), + "credentials": credentials, + "sensors": about["sensors"], + "people": about["people"], + "masked": about["masked"], + } + with zipfile.ZipFile(out, "w", zipfile.ZIP_STORED, allowZip64=True) as archive: + archive.writestr(MANIFEST, json.dumps(manifest, indent=2, ensure_ascii=False), zipfile.ZIP_DEFLATED) + archive.write(db_copy, DATABASE, zipfile.ZIP_DEFLATED) + for name, part in BACKUP_PARTS.items(): + if name not in parts: + continue + for folder in part["folders"]: + for path, rel in _files(settings, folder): + # Images and models hardly shrink: stored as they are, so a Raspberry Pi is not held up. + archive.write(path, f"{FILES}{folder}/{rel}") + + +class _QueueWriter: + """A file-like ``write`` that hands pieces of ``BACKUP["chunk_kb"]`` to a queue; it gives up + (OSError) once the reader is gone.""" + + def __init__(self, pieces: queue.Queue, stopped: threading.Event): + self.pieces, self.stopped = pieces, stopped + self.buffer = bytearray() + + def write(self, data: bytes) -> int: + self.buffer += data + if len(self.buffer) >= BACKUP["chunk_kb"] * 1024: + self._put(bytes(self.buffer)) + self.buffer.clear() + return len(data) + + def flush(self) -> None: + if self.buffer: + self._put(bytes(self.buffer)) + self.buffer.clear() + + def _put(self, item: object) -> None: + while not self.stopped.is_set(): + try: + self.pieces.put(item, timeout=1) + return + except queue.Full: + continue + raise OSError("The download was stopped") + + +def stream(settings: Settings, parts: set[str], credentials: bool) -> Iterator[bytes]: + """The backup as it is written, piece by piece (no temporary file of its size): a thread + writes it, this yields the pieces. Stopping early (the download was cancelled) ends the thread.""" + pieces: queue.Queue = queue.Queue(maxsize=8) + stopped = threading.Event() + done = object() + + def produce() -> None: + writer = _QueueWriter(pieces, stopped) + try: + write(settings, writer, parts, credentials) # type: ignore[arg-type] # write() is all ZipFile needs + writer.flush() + writer._put(done) + except Exception as err: # noqa: BLE001 - handed to the reader, which fails the download + if not stopped.is_set(): + log.warning("Backup failed: %s", redact(str(err))) + try: + writer._put(err) + except OSError: + pass + + thread = threading.Thread(target=produce, name="backup", daemon=True) + thread.start() + try: + while True: + item = pieces.get() + if item is done: + return + if isinstance(item, BaseException): + raise item + yield item # type: ignore[misc] + finally: + stopped.set() + + +# --- Restore -------------------------------------------------------------------------------- + + +@dataclass +class Staged: + """A checked backup, extracted to ``root`` (``root/``, ``root/visionstate.db``).""" + + root: Path + manifest: dict + slugs: set[str] + people: set[str] + masked: list[dict] # sensors whose camera password was left out: {id, name} + + +def stage_dir(settings: Settings) -> Path: + return settings.data_dir / STAGE + + +def _member_target(name: str, known: dict[str, Path], root: Path) -> Path | None: + """Where a member of the archive goes in the staging folder; None for one a backup never has.""" + if name in (MANIFEST, DATABASE): + return root / name if name == DATABASE else None + if not name.startswith(FILES) or "\\" in name: + return None + path = PurePosixPath(name[len(FILES) :]) + if path.is_absolute() or len(path.parts) < 2 or any(p in ("", ".", "..") for p in path.parts): + return None + folder = path.parts[0] + if folder not in known or folder in BACKUP_LEFT_OUT: + return None + return root.joinpath(*path.parts) + + +def read_manifest(archive: zipfile.ZipFile) -> dict: + if MANIFEST not in archive.namelist(): + raise ValueError("This is not a VisionState backup (backup.json is missing)") + info = archive.getinfo(MANIFEST) + if info.file_size > 5_000_000: + raise ValueError("backup.json is too large") + manifest = json.loads(archive.read(info)) + if not isinstance(manifest, dict) or not isinstance(manifest.get("format"), int): + raise ValueError("This is not a VisionState backup (bad backup.json)") + made_by = manifest.get("app_version", "?") + if manifest["format"] > BACKUP["format"]: + raise ValueError(f"This backup was made by a newer VisionState ({made_by}): update this app first") + schema = manifest.get("schema") + if not isinstance(schema, int) or schema < 1: + raise ValueError("This is not a VisionState backup (no database version in backup.json)") + if schema > SCHEMA_VERSION: + raise ValueError( + f"This backup was made by a newer VisionState ({made_by}, database version {schema}; " + f"this app has {SCHEMA_VERSION}): update this app first" + ) + return manifest + + +def stage(settings: Settings, upload: Path) -> Staged: + """Checks the uploaded backup and extracts it next to the live data; migrates its database + on that copy, so nothing live is touched until ``swap``. Raises ValueError when it is not a + backup this version can restore.""" + root = stage_dir(settings) / "new" + shutil.rmtree(root, ignore_errors=True) + root.mkdir(parents=True) + known = folders(settings) + limit = BACKUP["max_restore_gb"] * 1_000_000_000 + with zipfile.ZipFile(upload) as archive: + manifest = read_manifest(archive) + members = [info for info in archive.infolist() if not info.is_dir()] + if sum(info.file_size for info in members) > limit: + raise ValueError(f"The backup is larger than {BACKUP['max_restore_gb']} GB unpacked") + if DATABASE not in archive.namelist(): + raise ValueError("The backup has no database") + for info in members: + target = _member_target(info.filename, known, root) + if target is None: + if info.filename != MANIFEST: + log.warning("Restore: left out %s (not something a backup holds)", info.filename[:200]) + continue + target.parent.mkdir(parents=True, exist_ok=True) + with archive.open(info) as src, target.open("wb") as dst: + shutil.copyfileobj(src, dst, 1024 * 1024) + db_path = root / DATABASE + try: + con = sqlite3.connect(db_path) + try: + if con.execute("PRAGMA quick_check").fetchone()[0] != "ok": + raise ValueError("The backup's database is damaged") + con.execute("SELECT count(*) FROM sensor").fetchone() + finally: + con.close() + db = Database(db_path) + try: + db.init() # migrate an older backup here, before anything live is replaced + finally: + db.engine.dispose() + except sqlite3.DatabaseError as err: + raise ValueError(f"The backup's database cannot be read: {err}") from err + single_file(db_path) + con = sqlite3.connect(db_path) + try: + slugs = {row[0] for row in con.execute("SELECT slug FROM sensor")} + people = {row[0] for row in con.execute("SELECT key FROM person")} + rows = con.execute("SELECT id, name, source FROM sensor ORDER BY name").fetchall() + masked = [{"id": i, "name": name} for i, name, source in rows if MASK in source] + finally: + con.close() + return Staged(root, manifest, slugs, people, masked) + + +def single_file(db_path: Path) -> None: + """Everything in the one database file (no -wal next to it): only that file is moved in.""" + con = sqlite3.connect(db_path) + try: + con.execute("PRAGMA journal_mode=DELETE") + finally: + con.close() + + +def _aside(path: Path) -> Path: + return path.with_name(path.name + ASIDE) + + +def _remove(path: Path) -> None: + if path.is_dir() and not path.is_symlink(): + shutil.rmtree(path, ignore_errors=True) + else: + path.unlink(missing_ok=True) + + +def _rename(path: Path, target: Path) -> None: + """A rename that waits a moment for a file still open (Windows locks it until a worker thread + of the stopped runtime lets go).""" + for attempt in range(RENAME_TRIES): + try: + path.rename(target) + return + except PermissionError: + if attempt == RENAME_TRIES - 1: + raise + time.sleep(0.2) + + +def _replaced(settings: Settings) -> list[Path]: + """What a restore moves aside: the database and every folder it empties (see BACKUP_PARTS).""" + paths = [*_db_files(settings.db_path)] + for part in BACKUP_PARTS.values(): + if part["restore"] == "replace": + paths += [getattr(settings, name) for name in part["folders"]] + paths += [getattr(settings, name) for name in BACKUP_LEFT_OUT] + # A folder inside another one is moved with the outer one. + unique = sorted(set(paths), key=lambda p: len(p.parts)) + return [p for p in unique if not any(q in p.parents for q in unique)] + + +def swap(settings: Settings, staged: Staged) -> list[tuple[Path, Path]]: + """Moves the live database and folders aside and the staged ones in. Returns what was moved + aside (for ``discard`` or ``undo``); on an error it puts everything back itself.""" + moved: list[tuple[Path, Path]] = [] + added: list[Path] = [] + try: + for path in _replaced(settings): + if path.exists(): + aside = _aside(path) + _remove(aside) + _rename(path, aside) + moved.append((path, aside)) + added.append(settings.db_path) + shutil.move(staged.root / DATABASE, settings.db_path) + for part in BACKUP_PARTS.values(): + for folder in part["folders"]: + source = staged.root / folder + if not source.is_dir(): + continue + target = getattr(settings, folder) + if part["restore"] == "add": + for path in sorted(source.rglob("*")): + if path.is_file(): + dest = target / path.relative_to(source) + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.move(path, dest) + added.append(dest) + else: + target.parent.mkdir(parents=True, exist_ok=True) + shutil.move(source, target) + settings.ensure_dirs() + except Exception: + undo(settings, moved, added) + raise + return moved + + +def undo(settings: Settings, moved: list[tuple[Path, Path]], added: list[Path] | None = None) -> None: + """Puts back what ``swap`` moved aside; the restored files are removed. Every step is tried, + so one file that cannot be removed does not keep the rest of the old data away.""" + if added is None: # the swap finished: the restored database is in place + added = [settings.db_path] + restored_db = settings.db_path in added + for path in [*added, *(_db_files(settings.db_path)[1:] if restored_db else [])]: + try: + path.unlink(missing_ok=True) + except OSError as err: + log.warning("Restore: could not remove %s: %s", path.name, err) + for path, aside in reversed(moved): + try: + _remove(path) + _rename(aside, path) + except OSError as err: + log.error("Restore: could not put %s back (it is in %s): %s", path, aside.name, err) + settings.ensure_dirs() + + +def discard(settings: Settings, moved: list[tuple[Path, Path]]) -> None: + """The restored data runs: removes the old files and the staging folder.""" + stage_dir(settings).mkdir(parents=True, exist_ok=True) + (stage_dir(settings) / DONE).touch() # from here on, a start-up finishes this instead of undoing + for _, aside in moved: + _remove(aside) + shutil.rmtree(stage_dir(settings), ignore_errors=True) + + +def clean_leftovers(settings: Settings) -> None: + """At start-up, after a restore that was cut off (the app stopped halfway): the data moved + aside is put back, unless the restored data had already started (then it is removed).""" + finished = (stage_dir(settings) / DONE).exists() + candidates = [*_db_files(settings.db_path), *folders(settings).values()] + asides = [(path, _aside(path)) for path in candidates if _aside(path).exists()] + if asides and not finished: + log.warning("A restore was cut off: the data from before it is put back") + for path in _db_files(settings.db_path): + path.unlink(missing_ok=True) + for path, aside in asides: + if finished: + _remove(aside) + else: + _remove(path) + aside.rename(path) + shutil.rmtree(stage_dir(settings), ignore_errors=True) diff --git a/visionstate/backend/visionstate/db.py b/visionstate/backend/visionstate/db.py index 7bfd937..f1c1822 100644 --- a/visionstate/backend/visionstate/db.py +++ b/visionstate/backend/visionstate/db.py @@ -27,6 +27,7 @@ ) from sqlalchemy.engine import Connection from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column, relationship, sessionmaker +from sqlalchemy.pool import QueuePool log = logging.getLogger(__name__) @@ -361,6 +362,16 @@ def _pragmas(dbapi_conn, _record): # noqa: ANN001 cur.close() self._factory = sessionmaker(self.engine, expire_on_commit=False) + self.closed = False + + def close(self) -> Callable[[], int]: + """No session after this (a backup is restored under it: a worker thread of the stopped + runtime must not write into the restored database). Returns how many connections are + still in use, as a function to wait on.""" + self.closed = True + pool = self.engine.pool + self.engine.dispose() + return pool.checkedout if isinstance(pool, QueuePool) else lambda: 0 def init(self) -> None: fresh = not inspect(self.engine).has_table("sensor") @@ -379,7 +390,9 @@ def init(self) -> None: for table in Base.metadata.sorted_tables: for index in table.indexes: index.create(conn, checkfirst=True) - conn.execute(text(f"PRAGMA user_version = {SCHEMA_VERSION}")) + # Never lowered: an older version on a newer database (a rollback, a backup restored) + # must not make the next update run that database's data migrations again. + conn.execute(text(f"PRAGMA user_version = {max(version, SCHEMA_VERSION)}")) @staticmethod def after_commit(session: Session, action: Callable[[], None]) -> None: @@ -397,6 +410,8 @@ def run(_session: Session) -> None: @contextmanager def session(self) -> Iterator[Session]: + if self.closed: + raise RuntimeError("The database was closed (a backup is being restored)") session = self._factory() try: yield session diff --git a/visionstate/backend/visionstate/engine/runtime.py b/visionstate/backend/visionstate/engine/runtime.py index f82091a..44e67ca 100644 --- a/visionstate/backend/visionstate/engine/runtime.py +++ b/visionstate/backend/visionstate/engine/runtime.py @@ -89,6 +89,17 @@ async def stop(self) -> None: await self.grabber.close() await self.ha.close() + def release(self) -> None: + """After ``stop``, when another runtime takes over in this process (a restored backup): + let go of the models, the sensors' heads and the frames kept in memory, and give the + memory back, so two sets of models are never loaded at once.""" + self.embedder = self.detector = self.reader = self.wheel_reader = None + self.face_model = self.face_index = None + self.heads.clear() + self.taught.clear() + self.live.clear() + backbones.release_memory() + async def _entity_registry_loop(self) -> None: """Keeps ha_entity_ids up to date: now and then, and shortly after discovery was published.""" last_error = "" diff --git a/visionstate/backend/visionstate/main.py b/visionstate/backend/visionstate/main.py index ac53aa3..539ab51 100644 --- a/visionstate/backend/visionstate/main.py +++ b/visionstate/backend/visionstate/main.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import logging from contextlib import asynccontextmanager @@ -9,10 +10,12 @@ from fastapi.responses import FileResponse, JSONResponse from fastapi.staticfiles import StaticFiles +from . import backup +from .api import backup as backup_api from .api import cameras, faces, history, imports, review, samples, sensors, system, teach from .db import Database from .engine import Runtime -from .settings import INGRESS_PROXY_IP, VERSION, Settings, load_settings +from .settings import BACKUP, INGRESS_PROXY_IP, VERSION, Settings, load_settings log = logging.getLogger("visionstate") @@ -23,17 +26,34 @@ def create_app(settings: Settings | None = None) -> FastAPI: @asynccontextmanager async def lifespan(app: FastAPI): + backup.clean_leftovers(settings) # a restore cut off by a stop db = Database(settings.db_path) db.init() app.state.runtime = Runtime(settings, db) await app.state.runtime.start() log.info("VisionState %s started", VERSION) yield + # A restored backup runs on a runtime (and database) of its own: stop the one there is now. await app.state.runtime.stop() - db.engine.dispose() # close SQLite connections cleanly on shutdown + app.state.runtime.db.engine.dispose() # close SQLite connections cleanly on shutdown app = FastAPI(title="VisionState", version=VERSION, lifespan=lifespan) + app.state.ready = asyncio.Event() # cleared while a restore swaps the data (api/backup.py) + app.state.ready.set() + + @app.middleware("http") + async def while_restoring(request: Request, call_next): + # A restore swaps the data under a stopped runtime for a few seconds: requests wait for + # the new runtime instead of failing. + ready: asyncio.Event = request.app.state.ready + if not ready.is_set() and request.url.path.startswith("/api/"): + try: + await asyncio.wait_for(ready.wait(), BACKUP["hold_requests_s"]) + except TimeoutError: + return JSONResponse({"detail": "Restoring a backup, try again in a moment"}, status_code=503) + return await call_next(request) + if settings.is_supervised: @app.middleware("http") @@ -49,6 +69,7 @@ async def ingress_only(request: Request, call_next): app.include_router(review.router) app.include_router(history.router) app.include_router(imports.router) + app.include_router(backup_api.router) app.include_router(sensors.router) app.include_router(samples.router) app.include_router(teach.router) diff --git a/visionstate/backend/visionstate/mqtt.py b/visionstate/backend/visionstate/mqtt.py index 8fd772c..490f06d 100644 --- a/visionstate/backend/visionstate/mqtt.py +++ b/visionstate/backend/visionstate/mqtt.py @@ -647,6 +647,19 @@ async def remove_discovery(self, sensor: SensorDescriptor) -> None: for key, *_ in sensor.objects: await self.remove_object_class(sensor.slug, key, discovery=False) + async def remove_stale(self, before: SensorDescriptor, after: SensorDescriptor) -> None: + """The same sensor (slug) with other entities: forget the ones it no longer has (a restored + backup); the others are left alone, so Home Assistant keeps what the user changed on them.""" + prefix = self.settings.discovery_prefix + kept = {topic for topic, _ in discovery_messages(prefix, after)} + for topic, _ in discovery_messages(prefix, before): + if topic not in kept: + await self.publish(topic, "", retain=True) + kept_keys = {key for key, *_ in after.objects} + for key, *_ in before.objects: + if key not in kept_keys: + await self.remove_object_class(before.slug, key) + async def remove_rate(self, slug: str) -> None: """Forget the rate entity of a reading sensor that is no longer a counter.""" uid = f"{APP_SLUG}_{slug}" diff --git a/visionstate/backend/visionstate/settings.py b/visionstate/backend/visionstate/settings.py index b1e5f9f..da41d3d 100644 --- a/visionstate/backend/visionstate/settings.py +++ b/visionstate/backend/visionstate/settings.py @@ -672,6 +672,54 @@ def merge_triggers(stored: dict | None) -> dict: # Remove camera credentials from exported sensor bundles (the importer re-enters them). EXPORT_STRIP_CREDENTIALS = True +# --- Backup (export and restore everything, backup.py) ------------------------ + +# What a backup holds, in parts the user can leave out at export (the ticks in Settings → Backup). +# Every table of db.py and every folder of Settings belongs to exactly one part, or is named in +# BACKUP_LEFT_OUT: tests/test_backup.py fails for a new table or folder that is not, so nothing +# added later is silently missing from a backup. The database goes along whole (new columns +# need nothing here); a left-out part's rows are deleted from the copy. +# "restore": "replace" — a restore empties the part's folders here, also when the backup does not +# hold the part (its rows are gone); "add" — the backup's files are added, nothing here is removed. +BACKUP_PARTS: dict[str, dict] = { + "sensors": { + "optional": False, + "tables": ("sensor", "state", "model_info", "reading_watch", "setting"), + "folders": ("heads_dir",), + "restore": "replace", + }, + "samples": { + "optional": True, + "tables": ("sample", "sample_label", "embedding"), + "folders": ("samples_dir", "old_samples_dir"), + "restore": "replace", + }, + "faces": { + "optional": True, + "tables": ("person", "face", "face_embedding"), + "folders": ("faces_dir",), + "restore": "replace", + }, + "history": { + "optional": True, + "tables": ("prediction", "reading_stat"), + "folders": ("history_dir",), + "restore": "replace", + }, + "models": {"optional": True, "tables": (), "folders": ("models_dir",), "restore": "add"}, +} +# Folders never in a backup: made again when asked for. A restore empties them (they are keyed by row IDs). +BACKUP_LEFT_OUT = ("sample_thumbs_dir", "thumbs_dir") +BACKUP = { + "format": 1, # the archive's layout (backup.json "format"); a restore refuses a newer one + "credentials": False, # camera passwords in the backup: off unless the user ticks it + "max_restore_gb": 50, # largest backup a restore takes + "free_margin_mb": 200, # free disk space kept on top of what a restore needs + "chunk_kb": 1024, # streamed download: size of each piece + "hold_requests_s": 120, # requests made while a restore swaps the data wait this long for it + "release_wait_s": 60, # a restore waits this long for the stopped runtime's work to let go of the database +} + SUPERVISOR_URL = "http://supervisor" INGRESS_PROXY_IP = "172.30.32.2" diff --git a/visionstate/config.yaml b/visionstate/config.yaml index 21fcf1a..3c1cb45 100644 --- a/visionstate/config.yaml +++ b/visionstate/config.yaml @@ -1,5 +1,5 @@ name: VisionState (beta) -version: "0.9.2b4" +version: "0.10.0b1" slug: visionstate description: Teach a local AI to recognise states (open, closed, …), find objects (people, cars, pets) or read numbers in your camera images url: https://github.com/oleost/VisionState @@ -12,6 +12,9 @@ arch: - aarch64 ingress: true ingress_port: 8099 +# Uploads (videos, sensor bundles, backups) are streamed to the app: without it the Supervisor reads +# a request into memory and refuses one over 16 MB. +ingress_stream: true panel_icon: mdi:eye-check-outline panel_title: VisionState β homeassistant_api: true diff --git a/visionstate/frontend/e2e/ui.spec.ts b/visionstate/frontend/e2e/ui.spec.ts index 23f2945..38aaaee 100644 --- a/visionstate/frontend/e2e/ui.spec.ts +++ b/visionstate/frontend/e2e/ui.spec.ts @@ -1180,6 +1180,41 @@ test('the wheel reader shows in Settings and is offered to counters read as text } }); +test('a backup of everything is downloaded and restored', async ({ page, request }, info) => { + test.setTimeout(180_000); // the restore stops and starts every sensor + const errors = watchErrors(page); + await page.goto('#/settings'); + const card = page.getByTestId('backup'); + await card.scrollIntoViewIfNeeded(); + // Everything ticked, the sensors always; camera passwords only when ticked. + await expect(card.getByRole('checkbox', { name: /Sensors and settings/ })).toBeChecked(); + await expect(card.getByRole('checkbox', { name: /Sensors and settings/ })).toBeDisabled(); + await expect(card.getByRole('checkbox', { name: /History/ })).toBeChecked(); + await expect(card.getByRole('checkbox', { name: /Camera passwords/ })).not.toBeChecked(); + const download = card.getByRole('link', { name: /Download backup/ }); + await press(card.getByRole('checkbox', { name: /Downloaded AI models/ }), info); + await expect(download).toHaveAttribute('href', /leave_out=models/); + await expectNoHorizontalOverflow(page); + await expectNoClipping(page, '[data-testid=backup] .btn, [data-testid=backup] .check'); + await page.screenshot({ path: path.join('test-results', 'pages', info.project.name, 'settings-backup.png'), fullPage: true }); + test.skip(info.project.name !== 'desktop', 'the restore itself once (it restarts every sensor)'); + + const before = (await (await request.get('api/v1/sensors')).json()).map((s: { name: string }) => s.name).sort(); + const [file] = await Promise.all([page.waitForEvent('download'), download.click()]); + expect(file.suggestedFilename()).toMatch(/^visionstate-backup-.*\.zip$/); + const saved = info.outputPath('backup.zip'); + await file.saveAs(saved); + await card.locator('#backup-restore').setInputFiles(saved); + await expect(card.getByText(/Replace everything here with backup\.zip/)).toBeVisible(); + const replace = card.getByRole('button', { name: 'Replace everything' }); + await press(replace, info); + await press(card.getByRole('button', { name: 'Click again to replace everything' }), info); + await expect(card.getByText(new RegExp(`Restored ${before.length} sensors`))).toBeVisible({ timeout: 120_000 }); + const after = (await (await request.get('api/v1/sensors')).json()).map((s: { name: string }) => s.name).sort(); + expect(after).toEqual(before); + errors.expectNone(); +}); + test('the AI models can be compared, and the teaching and face limits set for all object sensors', async ({ page, request }, info) => { const errors = watchErrors(page); try { diff --git a/visionstate/frontend/src/App.svelte b/visionstate/frontend/src/App.svelte index b61e49f..ee3a1cb 100644 --- a/visionstate/frontend/src/App.svelte +++ b/visionstate/frontend/src/App.svelte @@ -43,7 +43,7 @@ {:else if section === 'people'} {:else if section === 'settings'} - + {#key app.restored}{/key} {:else} {/if} diff --git a/visionstate/frontend/src/lib/api.ts b/visionstate/frontend/src/lib/api.ts index 23bc764..ffeb717 100644 --- a/visionstate/frontend/src/lib/api.ts +++ b/visionstate/frontend/src/lib/api.ts @@ -1,6 +1,9 @@ // Typed client for the backend. Every URL the UI uses is built here. import type { AppConfig, + BackupInfo, + BackupPart, + RestoreResult, ObjectSettings, FaceLimitsInfo, FaceOverrides, @@ -304,6 +307,31 @@ export const api = { deleteModel: (kind: ModelKind, id: string) => request(`models/${kind}/${encodeURIComponent(id)}`, send('DELETE')), + backup: () => request('backup'), + backupUrl: (leaveOut: BackupPart[], credentials: boolean) => + `${BASE}backup/export?${qs({ leave_out: leaveOut.join(','), credentials: credentials || undefined })}`, + /** Replaces everything with a backup; XMLHttpRequest for the upload's progress (0–1), which fetch does not give. */ + restoreBackup(file: File, progress: (done: number) => void): Promise { + return new Promise((resolve, reject) => { + const xhr = new XMLHttpRequest(); + xhr.open('POST', `${BASE}backup/restore`); + xhr.setRequestHeader('Content-Type', 'application/zip'); + xhr.upload.onprogress = (e) => e.lengthComputable && progress(e.loaded / e.total); + xhr.onload = () => { + let body: { detail?: unknown } & Partial = {}; + try { + body = JSON.parse(xhr.responseText); + } catch { + /* not JSON */ + } + if (xhr.status >= 200 && xhr.status < 300) resolve(body as RestoreResult); + else reject(new ApiError(xhr.status, typeof body.detail === 'string' ? body.detail : xhr.statusText || 'Restore failed')); + }; + xhr.onerror = () => reject(new ApiError(0, 'The upload was interrupted')); + xhr.send(file); + }); + }, + importBundle(file: File) { const form = new FormData(); form.append('file', file, file.name); diff --git a/visionstate/frontend/src/lib/app.svelte.ts b/visionstate/frontend/src/lib/app.svelte.ts index 96e93a0..ba5d95a 100644 --- a/visionstate/frontend/src/lib/app.svelte.ts +++ b/visionstate/frontend/src/lib/app.svelte.ts @@ -1,13 +1,20 @@ // Global app state: server config, status and toasts. import { api } from './api'; import { plural } from './format'; -import type { AppConfig, ImportResult, Sensor, StateDef, Status } from './types'; +import type { AppConfig, ImportResult, RestoreResult, Sensor, StateDef, Status } from './types'; import { TOAST_MAX, TOAST_MS } from './ui'; -export const app = $state<{ config: AppConfig | null; status: Status | null; error: string }>({ +export const app = $state<{ + config: AppConfig | null; + status: Status | null; + error: string; + /** The last backup restored here: Settings is made again with what came back, and shows this. */ + restored: RestoreResult | null; +}>({ config: null, status: null, error: '', + restored: null, }); export async function loadConfig(): Promise { diff --git a/visionstate/frontend/src/lib/components/BackupCard.svelte b/visionstate/frontend/src/lib/components/BackupCard.svelte new file mode 100644 index 0000000..9625991 --- /dev/null +++ b/visionstate/frontend/src/lib/components/BackupCard.svelte @@ -0,0 +1,170 @@ + + +
+

Backup

+

+ Everything in one file: to move VisionState to another Home Assistant, or between the stable and the beta app. + Home Assistant's own backups hold the app too, but not its history. +

+ {#if config} +
+ {#each config.parts as part (part.key)} + + {/each} + +
+ + Download backup{#if info} · about {size(total)}{/if} + + +

Restore

+

+ Replaces everything here — sensors, people, training images and history — with a backup. Sensors that are in + the backup keep their entities in Home Assistant; the others are removed. +

+ + {#if !file} + + {:else} +
+ Replace everything here with {file.name} ({size(file.size)})? + + Download a backup of this app first if you may want it back. Stop the other VisionState app if it still runs + (they share their entities in Home Assistant). + + {#if info && file.size * 2 > info.free} + Free space here is {size(info.free)}: a restore needs about twice the backup's size. + {/if} +
+ {#if progress === null} + Replace everything + + {:else} + + {progress < 1 ? `Uploading… ${Math.round(progress * 100)} %` : 'Restoring… this takes a moment'} + + {/if} +
+
+ {/if} + {#if result} +
+ Restored {result.sensors} sensor{result.sensors === 1 ? '' : 's'} and {result.people} {result.people === 1 ? 'person' : 'people'} + {#if result.app_version}From VisionState {result.app_version}{/if} + {#if result.masked.length} + Enter the camera password again for: + + {#each result.masked as sensor (sensor.id)} + + {/each} + + {/if} +
+ {/if} + {:else} +

Loading…

+ {/if} +
+ + diff --git a/visionstate/frontend/src/lib/types.ts b/visionstate/frontend/src/lib/types.ts index 8c49658..421ca97 100644 --- a/visionstate/frontend/src/lib/types.ts +++ b/visionstate/frontend/src/lib/types.ts @@ -537,6 +537,8 @@ export interface AppConfig { reminder_defaults: ReminderRules; reminder_limits: Record<'after_days' | 'min_items' | 'repeat_days', [number, number]>; sensor_kinds: SensorKind[]; + /** Settings → Backup: the parts (the ones not optional are always in it), camera passwords ticked by default or not. */ + backup: { parts: { key: BackupPart; optional: boolean }[]; credentials: boolean; max_restore_gb: number }; object_sensor_defaults: { interval_s: number; threshold: number; debounce: number }; object_defaults: ObjectSettings; object_limits: Record<'min_size' | 'clear_after_s', [number, number]>; @@ -728,6 +730,24 @@ export interface BackboneInfo { export type ModelKind = 'backbones' | 'detectors' | 'face_models' | 'readers'; /** GET /storage: disk use and the history limits (whichever is reached first applies). */ +export type BackupPart = 'sensors' | 'samples' | 'faces' | 'history' | 'models'; + +/** GET /backup: bytes on disk per part, and the free space. */ +export interface BackupInfo { + parts: Record; + free: number; +} + +/** POST /backup/restore: what came back; `masked`: sensors whose camera password must be entered again. */ +export interface RestoreResult { + sensors: number; + people: number; + parts: BackupPart[]; + app_version: string | null; + exported_at: string | null; + masked: { id: number; name: string }[]; +} + export interface StorageInfo { history_bytes: number; history_frames: number; diff --git a/visionstate/frontend/src/lib/ui.ts b/visionstate/frontend/src/lib/ui.ts index f8798bd..3849766 100644 --- a/visionstate/frontend/src/lib/ui.ts +++ b/visionstate/frontend/src/lib/ui.ts @@ -1,5 +1,5 @@ // UI constants: polling intervals, labels and tones. Change behaviour of the UI here. -import type { CounterReader, ReadingDisplay, ReadingMode, ReadingType, SensorKind, SensorStatus } from './types'; +import type { BackupPart, CounterReader, ReadingDisplay, ReadingMode, ReadingType, SensorKind, SensorStatus } from './types'; /** Refresh intervals in milliseconds. */ export const POLL = { @@ -21,6 +21,15 @@ export const LIGHT_HOLD_STORAGE_KEY = 'visionstate.lightWhileViewing'; /** At most this many toasts on screen; older ones make room (labelling fast stacked them over the page). */ export const TOAST_MAX = 2; +/** Settings → Backup: what each part of a backup holds. */ +export const BACKUP_PARTS: Record = { + sensors: { label: 'Sensors and settings', help: 'Every sensor, its trained model and the app’s settings (always included)' }, + samples: { label: 'Training images', help: 'Labelled images and taught boxes' }, + faces: { label: 'People and their faces', help: 'Everyone taught by face, with their face images' }, + history: { label: 'History', help: 'Stored frames, readings and the review queue' }, + models: { label: 'Downloaded AI models', help: 'Downloaded again when needed if left out' }, +}; + export type Tone = 'ok' | 'warn' | 'danger' | 'muted' | 'info'; /** CSS colour for each tone (see tokens.css). */ diff --git a/visionstate/frontend/src/pages/Settings.svelte b/visionstate/frontend/src/pages/Settings.svelte index 7ca8422..a238470 100644 --- a/visionstate/frontend/src/pages/Settings.svelte +++ b/visionstate/frontend/src/pages/Settings.svelte @@ -1,6 +1,7 @@ -
+
{#each detections as d, i (i)} {@const color = d.filtered ? 'var(--c-unknown)' : objectColor(classes, d.label ?? d.key)} {@const left = `${d.box[0] * 100}%`} From fd3a37c0ab660457b73eaee552e5af1c726d34a4 Mon Sep 17 00:00:00 2001 From: oleost <5670710+oleost@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:30:23 +0200 Subject: [PATCH 07/18] Backup: extract each folder on the disk it goes to The history was extracted next to the database and moved to the media folder during the swap, while the app is stopped. On Home Assistant OS both are on one disk and the move is a rename, but the media folder may be network storage: then gigabytes were copied with the app stopped and requests held, long enough for Home Assistant's watchdog to restart the app mid-swap, and the next start put the old data back. Each folder is now staged on its own disk (/.restore for the history, backup._stage_root), so the swap only renames; both staging folders are cleared afterwards and at start-up. A test checks that no file moves between the two disks during the swap. Co-Authored-By: Claude Opus 5.5 --- docs/SCOPE.md | 4 ++- visionstate/backend/tests/test_backup.py | 29 ++++++++++++++- visionstate/backend/visionstate/api/backup.py | 4 +-- visionstate/backend/visionstate/backup.py | 36 +++++++++++++------ 4 files changed, 59 insertions(+), 14 deletions(-) diff --git a/docs/SCOPE.md b/docs/SCOPE.md index cf5be5f..2213c6f 100644 --- a/docs/SCOPE.md +++ b/docs/SCOPE.md @@ -725,7 +725,9 @@ installations is much code for a rare need. streamed (`ingress_stream: true` in `config.yaml`): without it the Supervisor reads a request into memory and refuses one over 16 MB, which had also kept videos and sensor bundles under that. The database files are moved aside first, so a restore cut off before that leaves the live - database alone. The staging folder is left out of Home Assistant's backups (`backup_exclude`). + database alone. Each folder is extracted on the disk it goes to (`/.restore` for the + history, `backup._stage_root`): the media folder may be network storage, and copying the history + there while the app is stopped could outlast Home Assistant's watchdog; the swap only renames. The staging folder is left out of Home Assistant's backups (`backup_exclude`). - **A backup is not trusted more than a sensor bundle**: every sensor in it is validated like a bundle import (`api.imports.validated_sensor`, shared by both), which also catches a setting of a newer version that needed no new database version (a new reading mode, a new display); its diff --git a/visionstate/backend/tests/test_backup.py b/visionstate/backend/tests/test_backup.py index ee3501a..5d3eef0 100644 --- a/visionstate/backend/tests/test_backup.py +++ b/visionstate/backend/tests/test_backup.py @@ -262,7 +262,7 @@ def test_restore_replaces_everything(tmp_path, removed): assert not (target.models_dir / "downloaded.onnx").exists() # not a model of the registries assert not (target.sample_thumbs_dir / "1.jpg").exists() assert (target.heads_dir / "1.joblib").exists() # restored (and retrained here: the test file is no model) - assert not backup.stage_dir(target).exists() + assert not backup.stage_dir(target).exists() and not (target.media_dir / backup.STAGE).exists() assert not list(target.data_dir.glob("*.restore-old")) and not list(target.media_dir.glob("*.restore-old")) @@ -453,6 +453,33 @@ def test_a_restore_cut_off_is_undone_at_start(tmp_path): assert not aside.exists() and any(settings.history_dir.rglob("*.jpg")) +def test_each_folder_is_staged_on_the_disk_it_goes_to(tmp_path): + """The media folder may be network storage: the history is extracted there, so the swap (with + the app stopped) only renames instead of copying it.""" + settings = make_settings(tmp_path) + populate(settings) + staged = backup.stage(settings, make(tmp_path / "b.zip")) + media_stage = settings.media_dir / backup.STAGE / "new" + assert any((media_stage / "history_dir").rglob("*.jpg")) + assert not (staged.root / "history_dir").exists() + assert any((staged.root / "samples_dir").rglob("*.jpg")) and (staged.root / backup.DATABASE).exists() + moves = [] + real_move = backup.shutil.move + + def move(src, dst): + moves.append((Path(src), Path(dst))) + return real_move(src, dst) + + backup.shutil.move = move + try: + backup.swap(settings, staged) + finally: + backup.shutil.move = real_move + for src, dst in moves: # never from one disk's folder to the other's + assert (settings.media_dir in src.parents) == (settings.media_dir in dst.parents), (src, dst) + assert any(settings.history_dir.rglob("*.jpg")) + + def test_a_restore_cut_off_before_the_database_moved_keeps_it(tmp_path): """The database is moved aside first; cut off before that, the live database is the one from before and must be left alone (an early version deleted it at the next start).""" diff --git a/visionstate/backend/visionstate/api/backup.py b/visionstate/backend/visionstate/api/backup.py index a936d36..f3cfe63 100644 --- a/visionstate/backend/visionstate/api/backup.py +++ b/visionstate/backend/visionstate/api/backup.py @@ -83,7 +83,7 @@ async def _restore(request: Request) -> dict: 507, f"Not enough free disk space: a restore of this backup needs about {need // 1_000_000} MB" ) folder = backup.stage_dir(settings) - await asyncio.to_thread(shutil.rmtree, folder, True) + await asyncio.to_thread(backup.clear_stage, settings) upload = folder / "upload.zip" try: folder.mkdir(parents=True) @@ -102,7 +102,7 @@ async def _restore(request: Request) -> dict: upload.unlink(missing_ok=True) await _swap(request, old, staged, after) finally: - await asyncio.to_thread(shutil.rmtree, folder, True) + await asyncio.to_thread(backup.clear_stage, settings) manifest = staged.manifest return { "sensors": len(staged.slugs), diff --git a/visionstate/backend/visionstate/backup.py b/visionstate/backend/visionstate/backup.py index 6d56569..38c49ee 100644 --- a/visionstate/backend/visionstate/backup.py +++ b/visionstate/backend/visionstate/backup.py @@ -221,7 +221,7 @@ def produce() -> None: @dataclass class Staged: - """A checked backup, extracted to ``root`` (``root/``, ``root/visionstate.db``).""" + """A checked backup: its database in ``root``, each folder next to where it goes (``_stage_root``).""" root: Path manifest: dict @@ -234,8 +234,23 @@ def stage_dir(settings: Settings) -> Path: return settings.data_dir / STAGE -def _member_target(name: str, known: dict[str, Path], root: Path) -> Path | None: - """Where a member of the archive goes in the staging folder; None for one a backup never has.""" +def _stage_root(settings: Settings, folder: str) -> Path: + """Where a folder of the backup is extracted: on the same disk as the folder it replaces, so + the swap only renames — the media folder may be network storage, and copying the history + there would hold the stopped app for minutes (long enough for Home Assistant's watchdog).""" + target = getattr(settings, folder) + base = settings.media_dir if settings.media_dir in target.parents else settings.data_dir + return base / STAGE / "new" + + +def clear_stage(settings: Settings) -> None: + """Removes the staging folders (the data folder's and the media folder's).""" + for base in (settings.data_dir, settings.media_dir): + shutil.rmtree(base / STAGE, ignore_errors=True) + + +def _member_target(name: str, known: dict[str, Path], root: Path, settings: Settings) -> Path | None: + """Where a member of the archive goes in a staging folder; None for one a backup never has.""" if name in (MANIFEST, DATABASE): return root / name if name == DATABASE else None if not name.startswith(FILES) or "\\" in name: @@ -246,7 +261,7 @@ def _member_target(name: str, known: dict[str, Path], root: Path) -> Path | None folder = path.parts[0] if folder not in known or folder in BACKUP_LEFT_OUT: return None - return root.joinpath(*path.parts) + return _stage_root(settings, folder).joinpath(*path.parts) def read_manifest(archive: zipfile.ZipFile) -> dict: @@ -279,7 +294,8 @@ def stage(settings: Settings, upload: Path) -> Staged: on that copy, so nothing live is touched until ``swap``. Raises ValueError when it is not a backup this version can restore.""" root = stage_dir(settings) / "new" - shutil.rmtree(root, ignore_errors=True) + for base in (settings.data_dir, settings.media_dir): + shutil.rmtree(base / STAGE / "new", ignore_errors=True) root.mkdir(parents=True) known = folders(settings) limit = BACKUP["max_restore_gb"] * 1_000_000_000 @@ -291,7 +307,7 @@ def stage(settings: Settings, upload: Path) -> Staged: if DATABASE not in archive.namelist(): raise ValueError("The backup has no database") for info in members: - target = _member_target(info.filename, known, root) + target = _member_target(info.filename, known, root, settings) if target is None: if info.filename != MANIFEST: log.warning("Restore: left out %s (not something a backup holds)", info.filename[:200]) @@ -299,7 +315,7 @@ def stage(settings: Settings, upload: Path) -> Staged: target.parent.mkdir(parents=True, exist_ok=True) with archive.open(info) as src, target.open("wb") as dst: shutil.copyfileobj(src, dst, 1024 * 1024) - _keep_known_models(root / "models_dir") + _keep_known_models(_stage_root(settings, "models_dir") / "models_dir") db_path = root / DATABASE try: con = sqlite3.connect(db_path) @@ -411,7 +427,7 @@ def swap(settings: Settings, staged: Staged) -> list[tuple[Path, Path]]: shutil.move(staged.root / DATABASE, settings.db_path) for part in BACKUP_PARTS.values(): for folder in part["folders"]: - source = staged.root / folder + source = _stage_root(settings, folder) / folder if not source.is_dir(): continue target = getattr(settings, folder) @@ -458,7 +474,7 @@ def discard(settings: Settings, moved: list[tuple[Path, Path]]) -> None: (stage_dir(settings) / DONE).touch() # from here on, a start-up finishes this instead of undoing for _, aside in moved: _remove(aside) - shutil.rmtree(stage_dir(settings), ignore_errors=True) + clear_stage(settings) def clean_leftovers(settings: Settings) -> None: @@ -480,4 +496,4 @@ def clean_leftovers(settings: Settings) -> None: else: _remove(path) aside.rename(path) - shutil.rmtree(stage_dir(settings), ignore_errors=True) + clear_stage(settings) From e640ce3e542841415fabde05f2a1b6e24aaa2ac7 Mon Sep 17 00:00:00 2001 From: oleost <5670710+oleost@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:46:56 +0200 Subject: [PATCH 08/18] 0.10.0b3: restoring with the media folder on network storage; a dashboard ResizeObserver error Co-Authored-By: Claude Opus 5.5 --- docs/SCOPE.md | 2 +- visionstate/CHANGELOG.md | 10 ++++++++++ visionstate/config.yaml | 2 +- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/SCOPE.md b/docs/SCOPE.md index 2213c6f..fdee14d 100644 --- a/docs/SCOPE.md +++ b/docs/SCOPE.md @@ -772,7 +772,7 @@ is part of backups and needs no restart. | **Recognising faces** ✅ | Faces in person boxes (YuNet), people taught by face and shared by every camera (SFace, or AuraFace for a fast PC), a People page, an entity per person saying where they are, unknown person, review questions, export opt-in — phases 1–3 of [`FACE_RECOGNITION_PLAN.md`](FACE_RECOGNITION_PLAN.md), and teaching a face from photos | 0.9.0 (betas 0.8.1b3–b5) | | **Counter limits** ✅ | A counter's limit grows with the time it was away (learned, a fixed rate or per reading), Review moves a counter on, readings held when the camera's picture changes size, training images in the app's storage (backups hold them) | 0.9.1 (betas 0.9.1b1–b6) | | **Reviewing readings** ✅ | A counter's value sent at its first reading, decimals for its rate, a rejected reading that repeats waits in Review once, Review shows a reading's region large and one sensor at a time | 0.9.2 (betas 0.9.2b1–b4) | -| **Backup of everything** | One file with everything (parts to leave out, camera passwords only when ticked), a restore that replaces it and keeps the entities, backups of every database version kept in the tests; uploads over 16 MB through Ingress | betas 0.10.0b1–b2 | +| **Backup of everything** | One file with everything (parts to leave out, camera passwords only when ticked), a restore that replaces it and keeps the entities, backups of every database version kept in the tests; uploads over 16 MB through Ingress | betas 0.10.0b1–b3 | | **Better counter readings** | Benchmark, the last value as an expectation, calibrated confidence, extra decimal, self-calibrating regions, wheels-v2, learning per sensor — status per phase in [`WHEEL_READER_PLAN.md`](WHEEL_READER_PLAN.md) | planned | **Open ideas** (not scheduled): more on the History page (export of a filtered selection, a diff --git a/visionstate/CHANGELOG.md b/visionstate/CHANGELOG.md index cdbc203..0bc241e 100644 --- a/visionstate/CHANGELOG.md +++ b/visionstate/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## 0.10.0b3 + +### Fixed + +- **Restoring a backup with the media folder on network storage**: the history is now unpacked + on the disk it goes to, so the app is stopped only for a moment while the backup is put in + place — copying it over the network then could take long enough for Home Assistant to restart + the app, which put the old data back. +- A rare "ResizeObserver loop" error on the dashboard in Safari and the Home Assistant app on iPhone. + ## 0.10.0b2 ### Fixed diff --git a/visionstate/config.yaml b/visionstate/config.yaml index d99bc14..c44f38f 100644 --- a/visionstate/config.yaml +++ b/visionstate/config.yaml @@ -1,5 +1,5 @@ name: VisionState (beta) -version: "0.10.0b2" +version: "0.10.0b3" slug: visionstate description: Teach a local AI to recognise states (open, closed, …), find objects (people, cars, pets) or read numbers in your camera images url: https://github.com/oleost/VisionState From 9e5e4f54b309adbed159be244d3ec747f72efe61 Mon Sep 17 00:00:00 2001 From: oleost <5670710+oleost@users.noreply.github.com> Date: Tue, 20 Oct 2026 11:19:55 +0200 Subject: [PATCH 09/18] README: a layout that reads well on phones The three kinds of sensor are short sections instead of a three-column table, each screenshot sits with its section at full width, the Home Assistant screenshots wrap from side by side to stacked, and the code blocks fit a phone without scrolling sideways. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 2 +- README.md | 77 ++++++++++++++++++------------ visionstate/frontend/e2e/promo.mjs | 2 +- 3 files changed, 49 insertions(+), 32 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 7475d3b..504e9a6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -155,7 +155,7 @@ Many users run Home Assistant on phones and tablets. `--app ` (photos from GitHub). Running it twice changes nothing. - **Docs screenshots** (`docs/promo/`): seed the demo sensors on a local backend with a local MQTT broker (so the header shows *MQTT connected*), then `node e2e/promo.mjs ` in - `visionstate/frontend` (1440 px wide, device scale 1.2, one row of the README the same height). + `visionstate/frontend` (1440 px wide, device scale 1.2). The Home Assistant screenshots (`ha-*.png`) are taken by hand on a real Home Assistant. Show the maintainer before publishing. diff --git a/README.md b/README.md index 7e74a81..e08b9f1 100644 --- a/README.md +++ b/README.md @@ -31,26 +31,33 @@ ## Three kinds of sensor -| | What you get | How | -|---|---|---| -| 🚪 **States** | *Garage door open, closed or halfway*, gate shut, lights on in the shed | Click a few examples of each state; it learns in about a second | -| 🧍 **Objects & people** | *Emma is home*, *our car is in the driveway*, a cat on the lawn, an unknown person at the door | Pick from 80 objects — no training. Name people by their face once; every camera knows them | -| 🔢 **Readings** | The water, gas or power meter — a cheap ESP32-CAM above it is enough —, the minutes left on the washer | Draw a box around the number. Counters only go up and land in the Energy dashboard | +### 🚪 States + +*Garage door open, closed or halfway*, the gate shut, the lights on in the shed. + +Click a few examples of each state; it learns in about a second. + +### 🧍 Objects & people + +*Emma is home*, *our car is in the driveway*, a cat on the lawn, an unknown person at the door. + +Pick from 80 objects — no training. Name people by their face once; every camera knows them.

- An object sensor recognising Emma by her face and telling our car from any car + An object sensor recognising Emma by her face and telling our car from any car
+ Emma recognised by her face, and our own car told apart from any other car.

- - - - - - - - - -
A reading sensor reading the wheels of a real water meterThe People page with the faces taught for Emma
Real meters — the rolling wheels of a water meter, read right even while a wheel is half way between two digits.People — a few photos from your phone are enough to start; faces never leave your Home Assistant.
+

+ The People page with the faces taught for Emma
+ A few photos from your phone are enough to start; faces never leave your Home Assistant. +

+ +### 🔢 Readings + +*The water, gas or power meter*, the minutes left on the washer. + +Draw a box around the number. Counters only go up and land in the Energy dashboard. ## Read a water, gas or power meter with an ESP32-CAM @@ -61,6 +68,11 @@ before it is published (a counter never goes down or jumps), and the total lands dashboard — with a rate (m³/h, kW) if you want one. For a dark meter cabinet it switches a light on for each reading; a battery camera can trigger a check when it has a new picture. +

+ A reading sensor reading the wheels of a real water meter
+ The rolling wheels of a real water meter, read right even while a wheel is half way between two digits. +

+ ## Highlights - 🧠 **100 % local on any CPU** — Intel, AMD, Raspberry Pi 4/5. No cloud, no GPU, no subscription, no telemetry. @@ -72,28 +84,27 @@ on for each reading; a battery camera can trigger a check when it has a new pict ## In Home Assistant - - - - - - - - - -
A garage door state sensor as a Home Assistant deviceThe driveway object sensor as a Home Assistant device, with Emma and our car
State sensor — the state with its confidence, the last frame, a button to check now and a switch to pause it.Object sensor — an on/off sensor and a count for every object, person and own label.
+

+ A garage door state sensor as a Home Assistant device + The driveway object sensor as a Home Assistant device, with Emma and our car +

+ +- **State sensor** — the state with its confidence, the last frame, a button to check now and a switch to pause it. +- **Object sensor** — an on/off sensor and a count for every object, person and own label. + +Use them like any other sensor — for example, tell me when the garage door has been open for +10 minutes: ```yaml -# Example: tell me when the garage door has been open for 10 minutes triggers: - trigger: state entity_id: sensor.garage_door to: open for: "00:10:00" actions: - - action: notify.mobile_app_phone + - action: notify.notify data: - message: The garage door has been open for 10 minutes. + message: Garage door open! ``` ## Install @@ -117,7 +128,13 @@ The user guide — every entity, triggers, faces, readings, review — is on the ## How it works ``` -camera ─► your region ─► pretrained model (ONNX, CPU) ─► small per-sensor model ─► debounce ─► MQTT ─► Home Assistant +camera + ▼ your region + ▼ pretrained model (ONNX, CPU) + ▼ small per-sensor model + ▼ debounce + ▼ MQTT +Home Assistant ``` - **States:** [DINOv2](https://github.com/facebookresearch/dinov2) turns the region into a feature diff --git a/visionstate/frontend/e2e/promo.mjs b/visionstate/frontend/e2e/promo.mjs index a8d3827..a8a7ff6 100644 --- a/visionstate/frontend/e2e/promo.mjs +++ b/visionstate/frontend/e2e/promo.mjs @@ -1,6 +1,6 @@ // The README screenshots (docs/promo/), from an app seeded with scripts/seed_demo.py. // node e2e/promo.mjs [app URL, default http://127.0.0.1:8099] [output folder, default ../../docs/promo] -// 1440 px wide at device scale 1.2; images in one row of the README get the same height. +// 1440 px wide at device scale 1.2. import { chromium } from '@playwright/test'; import path from 'node:path'; From 96d02f3265a883693e0b6217f449d17013127177 Mon Sep 17 00:00:00 2001 From: oleost <5670710+oleost@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:17:35 +0200 Subject: [PATCH 10/18] Memory: no ONNX Runtime arena, give freed memory back after every check ONNX Runtime's CPU arena kept the largest working memory any image had needed for the rest of the run (D-FINE S about 150 MB on Home Assistant OS), and glibc kept memory freed after a model run as well. Sessions now run without the arena and each check ends with malloc_trim (about 1 ms). With the default models the app is about 190 MB smaller (a test installation with nine sensors and faces: 943 -> 677 MB on average) for about 6 % more time per check. The detectors' footprints are measured again this way. Co-Authored-By: Claude Opus 5.5 --- docs/SCOPE.md | 6 ++++++ visionstate/backend/visionstate/backbones.py | 13 +++++++++++++ visionstate/backend/visionstate/detectors.json | 8 ++++---- visionstate/backend/visionstate/engine/checks.py | 3 ++- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/docs/SCOPE.md b/docs/SCOPE.md index fdee14d..0d0d49c 100644 --- a/docs/SCOPE.md +++ b/docs/SCOPE.md @@ -89,6 +89,12 @@ so retraining only runs the backbone on new or changed samples. (`backbones.Footprint`: extra memory, time per check on 4 threads of a desktop CPU, an accuracy rank within its kind), measured when the model was added. Texts and footprint may be updated in a released entry; its file and inputs never change. +- Memory before speed, since many run Home Assistant on small machines: ONNX Runtime runs without its + memory arena (`backbones.cpu_session`), which kept the largest working memory any image had needed + for the rest of the run (D-FINE S about 150 MB on Home Assistant OS), and every check ends with + `backbones.trim_memory`, as glibc kept the freed memory as well. With the default models the app + is about 190 MB smaller (a test installation with nine sensors and faces: 943 → 677 MB) for + about 6 % more time per check. The footprints of the detectors were measured again this way. - DINOv2 base, evaluated (2026-10-11) with the crops of the 14 objects in the CC0 test photos, each also changed in light, framing, blur, JPEG quality and night colours: the gap between the same object changed (5th percentile) and other objects of its kind (90th percentile) was 0.12 for base diff --git a/visionstate/backend/visionstate/backbones.py b/visionstate/backend/visionstate/backbones.py index 6d54891..1272cd5 100644 --- a/visionstate/backend/visionstate/backbones.py +++ b/visionstate/backend/visionstate/backbones.py @@ -161,10 +161,15 @@ def cpu_session(model_path: Path): Only the CPU provider is passed, so nothing else ONNX Runtime ships with (such as its Azure provider) is ever used. Only errors are logged: SFace's file lists its weights as graph inputs too, and ONNX Runtime warned about every one of them (174 lines) each time it loaded. + + No memory arena: with it, each session keeps the largest working memory any image ever + needed (D-FINE S alone held about 265 MB after a few detections) for the rest of the run; without + it that memory is freed after every run, for about 6 % more time per check. """ import onnxruntime as ort options = ort.SessionOptions() + options.enable_cpu_mem_arena = False options.log_severity_level = 3 options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL return ort.InferenceSession(str(model_path), sess_options=options, providers=["CPUExecutionProvider"]) @@ -177,6 +182,14 @@ def release_memory() -> None: switching models left the app hundreds of MB larger every time (until a restart). ``malloc_trim`` returns it; on other C libraries (Windows, macOS) there is nothing to do.""" gc.collect() + trim_memory() + + +def trim_memory() -> None: + """Give memory freed after a model run back to the system (``malloc_trim``, about 1 ms). + + Without it glibc keeps what the largest image needed: on Home Assistant OS the app stayed about + 190 MB larger with the default models. Only glibc has it; elsewhere this does nothing.""" try: ctypes.CDLL("libc.so.6").malloc_trim(0) except OSError, AttributeError: diff --git a/visionstate/backend/visionstate/detectors.json b/visionstate/backend/visionstate/detectors.json index 86f7577..d9a84b1 100644 --- a/visionstate/backend/visionstate/detectors.json +++ b/visionstate/backend/visionstate/detectors.json @@ -508,7 +508,7 @@ "license": "Apache-2.0", "source": "https://huggingface.co/ustc-community/dfine-small-coco", "bundled": true, - "ram_mb": 230, + "ram_mb": 85, "time_ms": 100, "accuracy": 3 }, @@ -523,7 +523,7 @@ "license": "Apache-2.0", "source": "https://huggingface.co/ustc-community/dfine-nano-coco", "bundled": false, - "ram_mb": 100, + "ram_mb": 60, "time_ms": 40, "accuracy": 2 }, @@ -538,7 +538,7 @@ "license": "Apache-2.0", "source": "https://huggingface.co/ustc-community/dfine-medium-coco", "bundled": false, - "ram_mb": 300, + "ram_mb": 125, "time_ms": 290, "accuracy": 4 }, @@ -553,7 +553,7 @@ "license": "Apache-2.0", "source": "https://huggingface.co/ustc-community/dfine-large-coco", "bundled": false, - "ram_mb": 350, + "ram_mb": 170, "time_ms": 420, "accuracy": 5 } diff --git a/visionstate/backend/visionstate/engine/checks.py b/visionstate/backend/visionstate/engine/checks.py index b904c85..16cbb77 100644 --- a/visionstate/backend/visionstate/engine/checks.py +++ b/visionstate/backend/visionstate/engine/checks.py @@ -11,7 +11,7 @@ from PIL import Image -from .. import imaging +from .. import backbones, imaging from ..db import Prediction from ..mqtt import topics from ..redact import redact @@ -66,6 +66,7 @@ async def _sensor_step(self, sensor_id: int) -> float | None | bool: try: if forced or kind == "full": await self.run_once(cfg) + await asyncio.to_thread(backbones.trim_memory) else: await self.probe(cfg) live.failure = "" From 9000901247847921eaad0f97fef377e97c5ff0c6 Mon Sep 17 00:00:00 2001 From: oleost <5670710+oleost@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:50:07 +0200 Subject: [PATCH 11/18] Classifier: logistic regression in NumPy instead of scikit-learn Loading scikit-learn and SciPy cost about 90 MB of memory on Home Assistant OS for a few matrix products; the dev installation with nine sensors went from 677 to 537 MB on average. classifier.fit minimises the same objective as LogisticRegression(class_weight="balanced") with lbfgs (L2 on the weights, not the intercept; one vector for two states, softmax for more) with L-BFGS. The problem is convex: on real DINOv2 embeddings the heads matched a fully converged scikit-learn to within 0.05 percentage points, closer than scikit-learn's own default stop. Heads are stored as .npz (no pickle) with a format number. The old pickles (.joblib) are retrained once at startup from the stored embeddings and left in place for an older version; a state sensor whose first model is training publishes nothing instead of "unknown". Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 4 +- docs/SCOPE.md | 14 +- scripts/cpu_probe.py | 24 +-- visionstate/backend/requirements.txt | 2 - visionstate/backend/tests/backup_fixture.py | 2 +- visionstate/backend/tests/test_backup.py | 4 +- visionstate/backend/tests/test_logic.py | 33 ++- visionstate/backend/tests/test_quality.py | 26 ++- visionstate/backend/visionstate/classifier.py | 198 +++++++++++++++--- .../backend/visionstate/engine/checks.py | 4 + .../backend/visionstate/engine/runtime.py | 5 +- .../backend/visionstate/engine/training.py | 11 +- visionstate/backend/visionstate/settings.py | 3 +- 13 files changed, 250 insertions(+), 80 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index e180afb..01d97de 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -21,7 +21,7 @@ docs/ SCOPE.md, ARCHITECTURE.md, RELEASING.md, WHEEL_READER_PL ``` **Stack:** Python 3.14, FastAPI, uvicorn, SQLAlchemy (SQLite), asyncio; ONNX Runtime (CPU only), -NumPy, Pillow, PyAV (bundled FFmpeg), scikit-learn; aiomqtt, websockets, httpx. Frontend: Svelte 5 + +NumPy, Pillow, PyAV (bundled FFmpeg); aiomqtt, websockets, httpx. Frontend: Svelte 5 + Vite + TypeScript, plain CSS with design tokens, a hash router (Ingress-safe), bundled fonts. Docker image `python:3.14-slim`, built by GitHub Actions and published on GHCR. Quality: pytest (with a fake camera, Home Assistant and MQTT), ruff, pyright, svelte-check, Playwright, an image smoke test. @@ -128,7 +128,7 @@ error is kept in `LiveState.error` and shown in the UI. the media folder (see `storage.Storage.face_path`). - **Files**: `/samples//` (training images; for object sensors the crops of taught boxes — next to the database that lists them, so a backup holds both), `/thumbs/sample/`, - `/heads/` (trained heads), `/models/` (downloaded models; models and thumbnails are + `/heads/` (trained heads, `.npz`), `/models/` (downloaded models; models and thumbnails are left out of backups); `/history//` and `/thumbs/history/`. Versions before the move kept training images in `/samples/`: they are read there until `engine.HistoryMixin.move_samples` has copied them (`settings.SAMPLES_MOVE`). A file is removed diff --git a/docs/SCOPE.md b/docs/SCOPE.md index 0d0d49c..bbb6276 100644 --- a/docs/SCOPE.md +++ b/docs/SCOPE.md @@ -70,7 +70,7 @@ of a meter. Neither needs training. 1. Grab frame → crop ROI → letterbox to a square → normalise. 2. Backbone (ONNX Runtime) → L2-normalised embedding (CLS token + mean of patch tokens). -3. Per-sensor head: scikit-learn logistic regression (`C=30`, class-balanced) → probability per state. +3. Per-sensor head: logistic regression (`C=30`, class-balanced) → probability per state. 4. Post-processing → published state (§7). Why: training takes about a second even on a Raspberry Pi, works with few samples, and every @@ -109,8 +109,16 @@ so retraining only runs the backbone on new or changed samples. Azure provider, are never offered or used). GPU/Coral were assessed and deliberately not pursued (little gain for this workload; Coral cannot run the transformer model and is often in use by other software). -- Heads trained by another scikit-learn version, or for another backbone, are retrained - automatically at startup. +- Heads of another format (`classifier.HEAD_FORMAT`), or for another backbone, are retrained + automatically at startup, from the stored embeddings. Until a sensor has a head again its checks + publish nothing (no "unknown" after an update). +- The head is logistic regression in NumPy (`classifier.fit`, L-BFGS): the objective of + scikit-learn's `LogisticRegression` (lbfgs, L2 on the weights, balanced class weights), which + trained the heads until 0.10 — importing scikit-learn and SciPy cost about 90 MB of memory on Home + Assistant OS. The problem is convex: on real embeddings the heads matched a fully converged + scikit-learn to within 0.05 percentage points (scikit-learn's own default stopping point is + less exact). Its old heads (pickles, `.joblib`) are retrained once and left in place, so an + older version the app is taken back to still finds them. - **Quality check:** after each training, cross-validated predictions give the accuracy, the confusion matrix and a list of *possibly mislabelled* samples (label ≠ out-of-fold prediction). - Not implemented: training-time augmentation (not needed so far), fine-tuning. diff --git a/scripts/cpu_probe.py b/scripts/cpu_probe.py index 04790c3..3800865 100644 --- a/scripts/cpu_probe.py +++ b/scripts/cpu_probe.py @@ -64,25 +64,9 @@ def _pillow(): return PIL.__version__ -@step("scipy") -def _scipy(): +@step("classifier (train and cross-validate a head)") +def _classifier(): import numpy as np - import scipy - from scipy import linalg, optimize, sparse, special - - a = np.random.default_rng(1).normal(size=(120, 120)) - linalg.svd(a) - linalg.solve(a @ a.T + np.eye(120), np.ones(120)) - optimize.minimize(lambda x: ((x - 3) ** 2).sum(), np.zeros(8), method="L-BFGS-B") - sparse.random(200, 200, density=0.05, random_state=1).tocsr() @ np.ones(200) - special.expit(a) - return scipy.__version__ - - -@step("scikit-learn (train and cross-validate a head)") -def _sklearn(): - import numpy as np - import sklearn from visionstate import classifier @@ -91,7 +75,7 @@ def _sklearn(): labels = ["open"] * 30 + ["closed"] * 30 result = classifier.train(vectors, labels, "probe", 1, ["open", "closed"]) result.head.predict(vectors[:4], ["open", "closed"]) - return sklearn.__version__ + return f"accuracy {result.accuracy:.2f}" @step("onnxruntime: state backbone") @@ -216,7 +200,7 @@ async def close(self): def _others(): import importlib - names = ["httptools", "websockets", "yaml", "watchfiles", "google.protobuf", "pydantic_core", "sqlalchemy", "joblib", "aiomqtt"] + names = ["httptools", "websockets", "yaml", "watchfiles", "google.protobuf", "pydantic_core", "sqlalchemy", "aiomqtt"] missing = [] for name in names: try: diff --git a/visionstate/backend/requirements.txt b/visionstate/backend/requirements.txt index 0e49f75..7ac18bc 100644 --- a/visionstate/backend/requirements.txt +++ b/visionstate/backend/requirements.txt @@ -7,7 +7,5 @@ aiomqtt==2.5.1 onnxruntime==1.30.0 numpy==2.3.5 pillow==12.3.0 -scikit-learn==1.9.1 -joblib==1.6.0 av==19.0.0 websockets==17.1 diff --git a/visionstate/backend/tests/backup_fixture.py b/visionstate/backend/tests/backup_fixture.py index cc7d525..e5ee5fd 100644 --- a/visionstate/backend/tests/backup_fixture.py +++ b/visionstate/backend/tests/backup_fixture.py @@ -119,7 +119,7 @@ def populate(settings) -> dict: # noqa: ANN001 s.add(ModelInfo(sensor_id=door.id, backbone="test", n_samples=4)) ids = {"door": door.id, "drive": drive.id, "meter": meter.id} db.set_setting("storage", {"history_days": 9}) - (settings.heads_dir / f"{ids['door']}.joblib").write_bytes(b"head") + (settings.heads_dir / f"{ids['door']}.npz").write_bytes(b"head") (settings.models_dir / "downloaded.onnx").write_bytes(b"model") finally: db.engine.dispose() diff --git a/visionstate/backend/tests/test_backup.py b/visionstate/backend/tests/test_backup.py index 5d3eef0..343c750 100644 --- a/visionstate/backend/tests/test_backup.py +++ b/visionstate/backend/tests/test_backup.py @@ -261,7 +261,7 @@ def test_restore_replaces_everything(tmp_path, removed): assert (target.models_dir / "here-already.onnx").exists() # models are added, never removed assert not (target.models_dir / "downloaded.onnx").exists() # not a model of the registries assert not (target.sample_thumbs_dir / "1.jpg").exists() - assert (target.heads_dir / "1.joblib").exists() # restored (and retrained here: the test file is no model) + assert (target.heads_dir / "1.npz").exists() # restored (and retrained here: the test file is no model) assert not backup.stage_dir(target).exists() and not (target.media_dir / backup.STAGE).exists() assert not list(target.data_dir.glob("*.restore-old")) and not list(target.media_dir.glob("*.restore-old")) @@ -491,7 +491,7 @@ def test_a_restore_cut_off_before_the_database_moved_keeps_it(tmp_path): size = settings.db_path.stat().st_size backup.clean_leftovers(settings) assert settings.db_path.exists() and settings.db_path.stat().st_size == size - assert (settings.heads_dir / "1.joblib").exists() + assert (settings.heads_dir / "1.npz").exists() def test_a_newer_database_keeps_its_version(tmp_path): diff --git a/visionstate/backend/tests/test_logic.py b/visionstate/backend/tests/test_logic.py index 396d17e..fcd7eef 100644 --- a/visionstate/backend/tests/test_logic.py +++ b/visionstate/backend/tests/test_logic.py @@ -1,11 +1,12 @@ import numpy as np +import pytest from PIL import Image from visionstate import classifier, imaging from visionstate.api.sensors import quality_tips from visionstate.engine import Debouncer, review_reason from visionstate.mqtt import SensorDescriptor, discovery_messages, topics -from visionstate.settings import REVIEW_DEFAULTS, UNKNOWN_STATE, merge_review +from visionstate.settings import CLASSIFIER, REVIEW_DEFAULTS, UNKNOWN_STATE, merge_review def test_debouncer_publishes_first_value_immediately_then_waits(): @@ -91,6 +92,36 @@ def test_classifier_trains_and_predicts_with_missing_states(): assert max(probs, key=probs.get) == "open" +@pytest.mark.parametrize("states", [2, 4]) +def test_classifier_reaches_the_optimum(tmp_path, states): + """The head is the optimum of scikit-learn's objective (balanced weights, L2 on the weights + only): its gradient vanishes there. Two states have one weight vector, more a softmax.""" + rng = np.random.default_rng(states) + centres = rng.normal(0, 1, (states, 16)) + labels = np.repeat([f"s{k}" for k in range(states)], [12 + 7 * k for k in range(states)]) # unbalanced + target = np.searchsorted(sorted(set(labels)), labels) + vectors = centres[target] + rng.normal(0, 1.2, (len(labels), 16)) + vectors /= np.linalg.norm(vectors, axis=1, keepdims=True) + keys, coef, intercept = classifier.fit(vectors, labels) + assert keys == sorted(set(labels)) and coef.shape == (1 if states == 2 else states, 16) + n = len(labels) + weight = (n / (states * np.bincount(target)))[target] / n + _, grad = classifier._objective( + np.concatenate([coef.ravel(), intercept]), vectors, target, weight, coef.shape[0], 1 / (CLASSIFIER["C"] * n) + ) + assert np.abs(grad).max() <= CLASSIFIER["tol"] + head = classifier.Head("test", keys, coef, intercept, version=3) + probs = head.probabilities(vectors) + assert np.allclose(probs.sum(axis=1), 1) and (probs.argmax(axis=1) == target).mean() > 0.6 + path = classifier.head_path(tmp_path, 1) + classifier.save(head, path) + loaded = classifier.load(path) + assert loaded is not None and loaded.keys == keys and loaded.version == 3 and loaded.backbone == "test" + assert np.array_equal(loaded.probabilities(vectors), probs) and classifier.is_current(loaded, "test") + path.write_bytes(b"not a head") + assert classifier.load(path) is None + + def test_classifier_needs_two_states(): result = classifier.train(np.ones((3, 4)), ["open"] * 3, "test", 1, ["open", "closed"]) assert result.head is None diff --git a/visionstate/backend/tests/test_quality.py b/visionstate/backend/tests/test_quality.py index ceaeaf3..4718d25 100644 --- a/visionstate/backend/tests/test_quality.py +++ b/visionstate/backend/tests/test_quality.py @@ -1,7 +1,9 @@ """Possibly mislabelled samples and the app-wide review queue entity.""" +import pytest from fastapi.testclient import TestClient +from visionstate import classifier from visionstate.main import create_app from visionstate.mqtt import REVIEW_TOPICS, hub_discovery_messages from visionstate.settings import Settings @@ -114,10 +116,10 @@ def test_relabel_and_delete_retrain_without_errors(tmp_path, model_dir): @requires_model -def test_outdated_heads_are_retrained_on_startup(tmp_path, model_dir): - """A head pickled by another scikit-learn version is retrained automatically at startup.""" - import joblib - +@pytest.mark.parametrize("outdated", ["legacy", "format"]) +def test_outdated_heads_are_retrained_on_startup(tmp_path, model_dir, outdated): + """A head from before HEAD_FORMAT (a scikit-learn pickle, .joblib) or of another format is + retrained automatically at startup; the pickle stays for an older version.""" settings = Settings( data_dir=tmp_path / "data", media_dir=tmp_path / "media", frontend_dir=tmp_path, bundled_models_dir=model_dir ) @@ -133,13 +135,19 @@ def test_outdated_heads_are_retrained_on_startup(tmp_path, model_dir): assert wait_for(lambda: client.get(f"/api/v1/sensors/{sid}").json()["model"] is not None) version = client.get(f"/api/v1/sensors/{sid}").json()["model"]["version"] - # Pretend the stored head came from an older scikit-learn. - path = settings.heads_dir / f"{sid}.joblib" - head = joblib.load(path) - head.sklearn_version = "0.0-old" - joblib.dump(head, path) + path = classifier.head_path(settings.heads_dir, sid) + if outdated == "legacy": + path.rename(classifier.legacy_path(settings.heads_dir, sid)) # no .npz: what 0.10.0b3 left + else: + head = classifier.load(path) + assert head is not None + head.format = 0 + classifier.save(head, path) with TestClient(create_app(settings)) as client: client.app.state.runtime.grabber = camera assert wait_for(lambda: client.get(f"/api/v1/sensors/{sid}").json()["model"]["version"] > version) assert client.get(f"/api/v1/sensors/{sid}").json()["status"] == "ok" + head = classifier.load(path) + assert head is not None and head.format == classifier.HEAD_FORMAT + assert classifier.legacy_path(settings.heads_dir, sid).exists() == (outdated == "legacy") diff --git a/visionstate/backend/visionstate/classifier.py b/visionstate/backend/visionstate/classifier.py index e91d46f..4301d7e 100644 --- a/visionstate/backend/visionstate/classifier.py +++ b/visionstate/backend/visionstate/classifier.py @@ -1,37 +1,55 @@ -"""Per-sensor classifier head trained on backbone embeddings.""" +"""Per-sensor classifier head trained on backbone embeddings: logistic regression in NumPy. + +The model of scikit-learn's ``LogisticRegression(C=…, class_weight="balanced")`` with its default +lbfgs solver: an L2 penalty on the weights (not on the intercept), one weight vector for two states +and a softmax over all states for more. The problem is convex, so it has one optimum and any solver +that reaches it gives the same head. scikit-learn (with SciPy) did this until it was replaced here: +loading it cost about 90 MB of memory on Home Assistant OS for a few matrix products. +""" from __future__ import annotations import time -import warnings from dataclasses import dataclass, field from pathlib import Path -import joblib import numpy as np -import sklearn -from sklearn.linear_model import LogisticRegression -from sklearn.model_selection import StratifiedKFold, cross_val_predict from .settings import CLASSIFIER, QUALITY -# Heads are pickled scikit-learn objects: after a scikit-learn upgrade they are retrained. -SKLEARN_VERSION = sklearn.__version__ +# Kept in each head file; a head of another format is retrained (from the stored embeddings). +HEAD_FORMAT = 1 +# Heads before HEAD_FORMAT were scikit-learn pickles (.joblib): retrained from the stored +# embeddings. The old file is left (a few kB) for an older version the app may be taken back to; +# deleting the sensor removes it. +LEGACY_SUFFIX = ".joblib" + + +def head_path(heads_dir: Path, sensor_id: int) -> Path: + return heads_dir / f"{sensor_id}.npz" + + +def legacy_path(heads_dir: Path, sensor_id: int) -> Path: + return heads_dir / f"{sensor_id}{LEGACY_SUFFIX}" @dataclass class Head: backbone: str keys: list[str] # class order of the model - model: LogisticRegression + coef: np.ndarray # (1, dim) for two classes, (classes, dim) for more + intercept: np.ndarray version: int = 0 - sklearn_version: str = "" # scikit-learn version that trained this head ("" = unknown/older) + format: int = HEAD_FORMAT + + def probabilities(self, vectors: np.ndarray) -> np.ndarray: + """One row per vector, one column per class (in ``keys`` order).""" + return _probabilities(np.asarray(vectors, dtype=np.float64), self.coef, self.intercept) def predict(self, vectors: np.ndarray, all_keys: list[str]) -> list[dict[str, float]]: """Probabilities per state key; states without training data get 0.""" - probs = self.model.predict_proba(vectors) out = [] - for row in probs: + for row in self.probabilities(vectors): result = {key: 0.0 for key in all_keys} for key, p in zip(self.keys, row, strict=False): if key in result: @@ -40,6 +58,102 @@ def predict(self, vectors: np.ndarray, all_keys: list[str]) -> list[dict[str, fl return out +def _probabilities(x: np.ndarray, coef: np.ndarray, intercept: np.ndarray) -> np.ndarray: + z = x @ coef.T + intercept + if coef.shape[0] == 1: # two classes: the second one is the positive one + p = 0.5 * (1.0 + np.tanh(0.5 * z[:, 0])) # the logistic function, without overflow + return np.stack([1.0 - p, p], axis=1) + z -= z.max(axis=1, keepdims=True) + e = np.exp(z) + return e / e.sum(axis=1, keepdims=True) + + +def _objective(theta: np.ndarray, x: np.ndarray, target: np.ndarray, weight: np.ndarray, rows: int, alpha: float): + """Weighted mean log loss plus ``alpha / 2 · |coef|²``, and its gradient (scikit-learn's scaling).""" + dim = x.shape[1] + coef = theta[: rows * dim].reshape(rows, dim) + intercept = theta[rows * dim :] + z = x @ coef.T + intercept + if rows == 1: + z = z[:, 0] + loss = np.logaddexp(0.0, z) - target * z # target 0/1 + diff = (0.5 * (1.0 + np.tanh(0.5 * z)) - target)[:, None] + else: + top = z.max(axis=1, keepdims=True) + lse = top[:, 0] + np.log(np.exp(z - top).sum(axis=1)) + loss = lse - z[np.arange(len(z)), target] + diff = np.exp(z - lse[:, None]) + diff[np.arange(len(z)), target] -= 1.0 + value = float(weight @ loss) + 0.5 * alpha * float((coef * coef).sum()) + g = diff * weight[:, None] + grad_coef = g.T @ x + alpha * coef + return value, np.concatenate([grad_coef.ravel(), g.sum(axis=0)]) + + +def _lbfgs(fun, theta: np.ndarray, max_iter: int, tol: float, memory: int = 10) -> np.ndarray: + """Minimises a smooth convex function: L-BFGS with a backtracking (Armijo) line search.""" + value, grad = fun(theta) + steps: list[tuple[np.ndarray, np.ndarray, float]] = [] + for _ in range(max_iter): + if np.abs(grad).max() <= tol: + break + q = grad.copy() + alphas = [] + for s, y, rho in reversed(steps): + a = rho * float(s @ q) + alphas.append(a) + q -= a * y + if steps: + s, y, _ = steps[-1] + q *= float(s @ y) / float(y @ y) + else: + q /= max(1.0, float(np.linalg.norm(grad))) + for (s, y, rho), a in zip(steps, reversed(alphas), strict=True): + q += s * (a - rho * float(y @ q)) + direction = -q + slope = float(grad @ direction) + if slope >= 0: # not a descent direction (rounding): start over from the gradient + steps.clear() + direction, slope = -grad, -float(grad @ grad) + step = 1.0 + while True: + new_value, new_grad = fun(theta + step * direction) + if new_value <= value + 1e-4 * step * slope: + break + step *= 0.5 + if step < 1e-12: # no lower value along it: as close to the optimum as rounding allows + return theta + s = step * direction + y = new_grad - grad + sy = float(s @ y) + if sy > 1e-12: + steps.append((s, y, 1.0 / sy)) + if len(steps) > memory: + steps.pop(0) + theta, value, grad = theta + s, new_value, new_grad + return theta + + +def fit(vectors: np.ndarray, labels: np.ndarray) -> tuple[list[str], np.ndarray, np.ndarray]: + """Trains on vectors with their labels; returns the classes (sorted), coef and intercept.""" + x = np.asarray(vectors, dtype=np.float64) + classes, target = np.unique(labels, return_inverse=True) + n, dim = x.shape + count = np.bincount(target) + # class_weight="balanced": every class weighs as much in total; the mean over all samples. + weight = (n / (len(classes) * count))[target] / n + rows = 1 if len(classes) == 2 else len(classes) + alpha = 1.0 / (CLASSIFIER["C"] * n) + theta = _lbfgs( + lambda t: _objective(t, x, target, weight, rows, alpha), + np.zeros(rows * dim + rows), + CLASSIFIER["max_iter"], + CLASSIFIER["tol"], + ) + coef = theta[: rows * dim].reshape(rows, dim) + return [str(c) for c in classes], coef, theta[rows * dim :].copy() + + @dataclass class TrainResult: head: Head | None @@ -51,8 +165,14 @@ class TrainResult: suspects: list[dict] = field(default_factory=list) # {index, label, predicted, confidence} -def make_model() -> LogisticRegression: - return LogisticRegression(max_iter=CLASSIFIER["max_iter"], C=CLASSIFIER["C"], class_weight="balanced") +def _folds(target: np.ndarray, folds: int) -> np.ndarray: + """A fold for each sample, every class spread evenly over the folds (shuffled, always the same).""" + rng = np.random.default_rng(0) + fold = np.empty(len(target), dtype=int) + for c in np.unique(target): + members = rng.permutation(np.flatnonzero(target == c)) + fold[members] = (np.arange(len(members)) + int(rng.integers(folds))) % folds + return fold def train(vectors: np.ndarray, labels: list[str], backbone: str, version: int, state_keys: list[str]) -> TrainResult: @@ -70,25 +190,23 @@ def train(vectors: np.ndarray, labels: list[str], backbone: str, version: int, s return TrainResult(head=None, n_samples=len(labels), counts=counts) y = np.array(labels) - model = make_model().fit(vectors, y) - head = Head( - backbone=backbone, - keys=[str(c) for c in model.classes_], - model=model, - version=version, - sklearn_version=SKLEARN_VERSION, - ) + keys, coef, intercept = fit(vectors, y) + head = Head(backbone=backbone, keys=keys, coef=coef, intercept=intercept, version=version) accuracy = confusion = None suspects: list[dict] = [] min_count = min(counts[c] for c in classes) folds = min(QUALITY["cv_folds"], min_count) if folds >= 2: - cv = StratifiedKFold(n_splits=folds, shuffle=True, random_state=0) # Each sample is predicted by a model that never saw it, so disagreements point at # samples that are either labelled wrong or genuinely ambiguous. - proba = np.asarray(cross_val_predict(make_model(), vectors, y, cv=cv, method="predict_proba")) - order = np.array(sorted(classes)) + fold = _folds(np.searchsorted(classes, y), folds) + proba = np.zeros((len(y), len(classes))) + for k in range(folds): + held = fold == k + _, c, b = fit(vectors[~held], y[~held]) # every class is in each training part + proba[held] = _probabilities(np.asarray(vectors[held], dtype=np.float64), c, b) + order = np.array(classes) predicted = order[proba.argmax(axis=1)] accuracy = float((predicted == y).mean()) suspects = sorted( @@ -118,7 +236,16 @@ def train(vectors: np.ndarray, labels: list[str], backbone: str, version: int, s def save(head: Head, path: Path) -> None: tmp = path.with_suffix(".tmp") - joblib.dump(head, tmp) + with tmp.open("wb") as file: + np.savez( + file, + format=np.array(head.format), + backbone=np.array(head.backbone), + version=np.array(head.version), + keys=np.array(head.keys, dtype=str), + coef=head.coef, + intercept=head.intercept, + ) tmp.replace(path) @@ -126,14 +253,19 @@ def load(path: Path) -> Head | None: if not path.exists(): return None try: - with warnings.catch_warnings(): - # Pickles from another scikit-learn version warn; is_current() decides to retrain instead. - warnings.simplefilter("ignore") - return joblib.load(path) - except Exception: # noqa: BLE001 - a corrupt or incompatible head is simply retrained + with np.load(path, allow_pickle=False) as data: + return Head( + backbone=str(data["backbone"]), + keys=[str(k) for k in data["keys"]], + coef=data["coef"], + intercept=data["intercept"], + version=int(data["version"]), + format=int(data["format"]), + ) + except Exception: # noqa: BLE001 - a corrupt or unknown head is simply retrained return None def is_current(head: Head, backbone: str | None) -> bool: - """True when the head was trained by this scikit-learn version and for the active backbone.""" - return getattr(head, "sklearn_version", "") == SKLEARN_VERSION and (backbone is None or head.backbone == backbone) + """True when the head has this version's format and was trained for the active backbone.""" + return head.format == HEAD_FORMAT and (backbone is None or head.backbone == backbone) diff --git a/visionstate/backend/visionstate/engine/checks.py b/visionstate/backend/visionstate/engine/checks.py index 16cbb77..0237f6f 100644 --- a/visionstate/backend/visionstate/engine/checks.py +++ b/visionstate/backend/visionstate/engine/checks.py @@ -266,6 +266,10 @@ async def _run_states(self, cfg: SensorConfig, image: Image.Image) -> None: """One check of a state sensor: classify the region, debounce, publish, keep history.""" live = self.live_state(cfg.id) t = topics(cfg.slug) + if cfg.id not in self.heads and cfg.id in self.training: + # Its first model is being trained, or a head of an older format retrained after an + # update: "unknown" until then would be wrong. The training checks again when done. + return probs = await self.classify_image(cfg, image) if probs: top, confidence = max(probs.items(), key=lambda kv: kv[1]) diff --git a/visionstate/backend/visionstate/engine/runtime.py b/visionstate/backend/visionstate/engine/runtime.py index 44e67ca..8c50635 100644 --- a/visionstate/backend/visionstate/engine/runtime.py +++ b/visionstate/backend/visionstate/engine/runtime.py @@ -8,7 +8,7 @@ from sqlalchemy import select -from .. import backbones, detectors, faces, readers +from .. import backbones, classifier, detectors, faces, readers from ..db import Database, Sensor from ..ha_events import HaEventListener, fetch_entity_ids from ..mqtt import MqttBridge, SensorDescriptor @@ -160,7 +160,8 @@ async def sensor_deleted(self, sensor_id: int, descriptor: SensorDescriptor) -> self.taught.pop(sensor_id, None) self._person_last = {k: v for k, v in self._person_last.items() if v[1] != sensor_id} self._published_classes.pop(sensor_id, None) - (self.settings.heads_dir / f"{sensor_id}.joblib").unlink(missing_ok=True) + classifier.head_path(self.settings.heads_dir, sensor_id).unlink(missing_ok=True) + classifier.legacy_path(self.settings.heads_dir, sensor_id).unlink(missing_ok=True) await self.mqtt.remove_discovery(descriptor) await asyncio.to_thread(self.storage.delete_sensor, sensor_id) await self.publish_review_count() diff --git a/visionstate/backend/visionstate/engine/training.py b/visionstate/backend/visionstate/engine/training.py index 7990317..9abe879 100644 --- a/visionstate/backend/visionstate/engine/training.py +++ b/visionstate/backend/visionstate/engine/training.py @@ -21,13 +21,16 @@ class TrainingMixin(RuntimeBase): def heads_load(self, sensor_id: int) -> bool: - """Loads a sensor's head. Returns True when it must be retrained (missing, unreadable or outdated).""" - path = self.settings.heads_dir / f"{sensor_id}.joblib" + """Loads a sensor's head. Returns True when it must be retrained (unreadable, outdated or + from before HEAD_FORMAT); False also for a sensor that has none (untrained).""" + path = classifier.head_path(self.settings.heads_dir, sensor_id) head = classifier.load(path) if head is not None: self.heads[sensor_id] = head # keep using it until the retrain finishes backbone = self.embedder.spec.id if self.embedder else None - return path.exists() and (head is None or not classifier.is_current(head, backbone)) + if not path.exists(): + return classifier.legacy_path(self.settings.heads_dir, sensor_id).exists() + return head is None or not classifier.is_current(head, backbone) def schedule_retrain(self, sensor_id: int, delay: float | None = None) -> None: """Retrain soon; repeated calls within the delay are coalesced into one run.""" @@ -90,7 +93,7 @@ def _retrain_sync(self, sensor_id: int) -> None: {"sample_id": samples[item["index"]].id, **{k: v for k, v in item.items() if k != "index"}} for item in result.suspects ] - head_path = self.settings.heads_dir / f"{sensor_id}.joblib" + head_path = classifier.head_path(self.settings.heads_dir, sensor_id) if result.head is None: self.heads.pop(sensor_id, None) head_path.unlink(missing_ok=True) diff --git a/visionstate/backend/visionstate/settings.py b/visionstate/backend/visionstate/settings.py index da41d3d..08984db 100644 --- a/visionstate/backend/visionstate/settings.py +++ b/visionstate/backend/visionstate/settings.py @@ -508,7 +508,8 @@ # Logistic regression on L2-normalised embeddings. Higher C = more confident probabilities; # 30 keeps clear frames well above the default threshold while ambiguous frames stay near 50%. -CLASSIFIER = {"C": 30.0, "max_iter": 3000} +# Training stops when no gradient component exceeds tol (or after max_iter steps). +CLASSIFIER = {"C": 30.0, "max_iter": 3000, "tol": 1e-6} # --- Quality hints ---------------------------------------------------------- From 62742e75bc04cbed179f97b19fc11576be43691d Mon Sep 17 00:00:00 2001 From: oleost <5670710+oleost@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:23:41 +0200 Subject: [PATCH 12/18] 0.10.0b4: much less memory (no ONNX Runtime arena, classifier without scikit-learn) Co-Authored-By: Claude Opus 5.5 --- visionstate/CHANGELOG.md | 11 +++++++++++ visionstate/config.yaml | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/visionstate/CHANGELOG.md b/visionstate/CHANGELOG.md index 0bc241e..ab34261 100644 --- a/visionstate/CHANGELOG.md +++ b/visionstate/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## 0.10.0b4 + +### Better + +- **Uses much less memory**: memory an AI model needed for one image is now given back after + each check instead of being held for the rest of the run, and the state classifier no longer + loads scikit-learn. On a test installation with nine sensors and face recognition the app went + from about 940 MB to about 580 MB. A check takes a few percent longer. +- State sensors are retrained once after the update, from their stored training images (it takes + a moment, and the sensors keep their last state meanwhile). + ## 0.10.0b3 ### Fixed diff --git a/visionstate/config.yaml b/visionstate/config.yaml index c44f38f..3638340 100644 --- a/visionstate/config.yaml +++ b/visionstate/config.yaml @@ -1,5 +1,5 @@ name: VisionState (beta) -version: "0.10.0b3" +version: "0.10.0b4" slug: visionstate description: Teach a local AI to recognise states (open, closed, …), find objects (people, cars, pets) or read numbers in your camera images url: https://github.com/oleost/VisionState From 41967d4a425f5f8817ca0fbffcc753d45a64bd08 Mon Sep 17 00:00:00 2001 From: oleost <5670710+oleost@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:50:23 +0200 Subject: [PATCH 13/18] Heads: older versions retrain after going back instead of finding no model 0.10.0b4 saved heads only as .npz. Taken back to 0.10.0b3 or 0.9.2 (or restoring a 0.10 backup there), a state sensor had no .joblib and stayed untrained until its next label, or used an old pickle without the newer labels. Every training (and the startup, for heads trained before) now puts a marker in .joblib that older versions cannot read, so they retrain from the stored images. Coming forward again, a head older than the sensor's model is retrained too. Checked on the test Home Assistant: b4 -> b3 -> 0.9.2 -> b4. Co-Authored-By: Claude Opus 5.5 --- docs/SCOPE.md | 8 +++- visionstate/backend/tests/test_quality.py | 47 ++++++++++++++++--- visionstate/backend/visionstate/classifier.py | 6 ++- .../backend/visionstate/engine/training.py | 19 ++++++-- 4 files changed, 67 insertions(+), 13 deletions(-) diff --git a/docs/SCOPE.md b/docs/SCOPE.md index bbb6276..06e40db 100644 --- a/docs/SCOPE.md +++ b/docs/SCOPE.md @@ -117,8 +117,12 @@ so retraining only runs the backbone on new or changed samples. trained the heads until 0.10 — importing scikit-learn and SciPy cost about 90 MB of memory on Home Assistant OS. The problem is convex: on real embeddings the heads matched a fully converged scikit-learn to within 0.05 percentage points (scikit-learn's own default stopping point is - less exact). Its old heads (pickles, `.joblib`) are retrained once and left in place, so an - older version the app is taken back to still finds them. + less exact). Its old heads (pickles, `.joblib`) are retrained once. Each training puts + `classifier.LEGACY_MARKER` in that file: an older version the app is taken back to (or that + restores its backup) cannot read it and retrains from the stored images, as it does with a broken + head — finding no file, it would call the sensor untrained until the next label, and an old pickle + lacks the newer labels. Coming forward again, a head older than the sensor's model + (`ModelInfo.version`, which the older version raised) is retrained too. - **Quality check:** after each training, cross-validated predictions give the accuracy, the confusion matrix and a list of *possibly mislabelled* samples (label ≠ out-of-fold prediction). - Not implemented: training-time augmentation (not needed so far), fine-tuning. diff --git a/visionstate/backend/tests/test_quality.py b/visionstate/backend/tests/test_quality.py index 4718d25..9c139b4 100644 --- a/visionstate/backend/tests/test_quality.py +++ b/visionstate/backend/tests/test_quality.py @@ -4,6 +4,7 @@ from fastapi.testclient import TestClient from visionstate import classifier +from visionstate.db import ModelInfo from visionstate.main import create_app from visionstate.mqtt import REVIEW_TOPICS, hub_discovery_messages from visionstate.settings import Settings @@ -116,10 +117,12 @@ def test_relabel_and_delete_retrain_without_errors(tmp_path, model_dir): @requires_model -@pytest.mark.parametrize("outdated", ["legacy", "format"]) +@pytest.mark.parametrize("outdated", ["legacy", "format", "older version trained"]) def test_outdated_heads_are_retrained_on_startup(tmp_path, model_dir, outdated): - """A head from before HEAD_FORMAT (a scikit-learn pickle, .joblib) or of another format is - retrained automatically at startup; the pickle stays for an older version.""" + """A head from before HEAD_FORMAT (a scikit-learn pickle, .joblib), of another format, or + older than the sensor's model (an older version the app was taken back to trained it since) is + retrained automatically at startup. The .joblib then holds the marker that makes an older + version retrain too.""" settings = Settings( data_dir=tmp_path / "data", media_dir=tmp_path / "media", frontend_dir=tmp_path, bundled_models_dir=model_dir ) @@ -134,11 +137,16 @@ def test_outdated_heads_are_retrained_on_startup(tmp_path, model_dir, outdated): client.post(f"/api/v1/sensors/{sid}/capture", json={"state_key": state}) assert wait_for(lambda: client.get(f"/api/v1/sensors/{sid}").json()["model"] is not None) version = client.get(f"/api/v1/sensors/{sid}").json()["model"]["version"] + legacy = classifier.legacy_path(settings.heads_dir, sid) + assert legacy.read_bytes() == classifier.LEGACY_MARKER + if outdated == "older version trained": + with client.app.state.runtime.db.session() as s: + s.get(ModelInfo, sid).version = version = version + 1 path = classifier.head_path(settings.heads_dir, sid) if outdated == "legacy": - path.rename(classifier.legacy_path(settings.heads_dir, sid)) # no .npz: what 0.10.0b3 left - else: + path.replace(legacy) # no .npz: what 0.10.0b3 left + elif outdated == "format": head = classifier.load(path) assert head is not None head.format = 0 @@ -150,4 +158,31 @@ def test_outdated_heads_are_retrained_on_startup(tmp_path, model_dir, outdated): assert client.get(f"/api/v1/sensors/{sid}").json()["status"] == "ok" head = classifier.load(path) assert head is not None and head.format == classifier.HEAD_FORMAT - assert classifier.legacy_path(settings.heads_dir, sid).exists() == (outdated == "legacy") + assert legacy.read_bytes() == classifier.LEGACY_MARKER + + +@requires_model +def test_a_head_without_the_marker_gets_it_at_startup(tmp_path, model_dir): + """A head trained before the .joblib marker existed is kept (no retrain); the marker is added so an + older version the app is taken back to retrains instead of finding no head or an old one.""" + settings = Settings( + data_dir=tmp_path / "data", media_dir=tmp_path / "media", frontend_dir=tmp_path, bundled_models_dir=model_dir + ) + camera = FakeCamera() + body = {"name": "Door", "source_type": "http", "source": "x", "states": [{"name": "Open"}, {"name": "Closed"}]} + with TestClient(create_app(settings)) as client: + client.app.state.runtime.grabber = camera + sid = client.post("/api/v1/sensors", json=body).json()["id"] + for state in ("open", "closed"): + camera.state = state + for _ in range(3): + client.post(f"/api/v1/sensors/{sid}/capture", json={"state_key": state}) + assert wait_for(lambda: client.get(f"/api/v1/sensors/{sid}").json()["model"] is not None) + version = client.get(f"/api/v1/sensors/{sid}").json()["model"]["version"] + legacy = classifier.legacy_path(settings.heads_dir, sid) + legacy.write_bytes(b"an old scikit-learn pickle") + + with TestClient(create_app(settings)) as client: + client.app.state.runtime.grabber = camera + assert legacy.read_bytes() == classifier.LEGACY_MARKER + assert client.get(f"/api/v1/sensors/{sid}").json()["model"]["version"] == version diff --git a/visionstate/backend/visionstate/classifier.py b/visionstate/backend/visionstate/classifier.py index 4301d7e..121d03d 100644 --- a/visionstate/backend/visionstate/classifier.py +++ b/visionstate/backend/visionstate/classifier.py @@ -20,9 +20,11 @@ # Kept in each head file; a head of another format is retrained (from the stored embeddings). HEAD_FORMAT = 1 # Heads before HEAD_FORMAT were scikit-learn pickles (.joblib): retrained from the stored -# embeddings. The old file is left (a few kB) for an older version the app may be taken back to; -# deleting the sensor removes it. +# embeddings. Every training puts LEGACY_MARKER in that file: an older version the app is taken back to +# cannot read it and retrains from the stored images (as it does with a broken head), instead of +# finding no head (untrained until the next label) or an old one (without the newer labels). LEGACY_SUFFIX = ".joblib" +LEGACY_MARKER = b"VisionState: this sensor's model is in .npz; older versions retrain it.\n" def head_path(heads_dir: Path, sensor_id: int) -> Path: diff --git a/visionstate/backend/visionstate/engine/training.py b/visionstate/backend/visionstate/engine/training.py index 9abe879..312005d 100644 --- a/visionstate/backend/visionstate/engine/training.py +++ b/visionstate/backend/visionstate/engine/training.py @@ -21,8 +21,9 @@ class TrainingMixin(RuntimeBase): def heads_load(self, sensor_id: int) -> bool: - """Loads a sensor's head. Returns True when it must be retrained (unreadable, outdated or - from before HEAD_FORMAT); False also for a sensor that has none (untrained).""" + """Loads a sensor's head. Returns True when it must be retrained (unreadable, outdated, from + before HEAD_FORMAT, or older than the sensor's model); False also for a sensor that has none + (untrained).""" path = classifier.head_path(self.settings.heads_dir, sensor_id) head = classifier.load(path) if head is not None: @@ -30,7 +31,17 @@ def heads_load(self, sensor_id: int) -> bool: backbone = self.embedder.spec.id if self.embedder else None if not path.exists(): return classifier.legacy_path(self.settings.heads_dir, sensor_id).exists() - return head is None or not classifier.is_current(head, backbone) + if head is None or not classifier.is_current(head, backbone): + return True + # An older version the app was taken back to trained the sensor since (into its own file). + with self.db.session() as s: + info = s.get(ModelInfo, sensor_id) + if info is not None and info.version != head.version: + return True + legacy = classifier.legacy_path(self.settings.heads_dir, sensor_id) + if not legacy.exists() or legacy.read_bytes() != classifier.LEGACY_MARKER: # trained before the marker + legacy.write_bytes(classifier.LEGACY_MARKER) + return False def schedule_retrain(self, sensor_id: int, delay: float | None = None) -> None: """Retrain soon; repeated calls within the delay are coalesced into one run.""" @@ -97,8 +108,10 @@ def _retrain_sync(self, sensor_id: int) -> None: if result.head is None: self.heads.pop(sensor_id, None) head_path.unlink(missing_ok=True) + classifier.legacy_path(self.settings.heads_dir, sensor_id).unlink(missing_ok=True) else: classifier.save(result.head, head_path) + classifier.legacy_path(self.settings.heads_dir, sensor_id).write_bytes(classifier.LEGACY_MARKER) self.heads[sensor_id] = result.head with self.db.session() as s: info = s.get(ModelInfo, sensor_id) or ModelInfo(sensor_id=sensor_id) From 3edc4d1732d5e83abf713e7709de6ba43011ef79 Mon Sep 17 00:00:00 2001 From: oleost <5670710+oleost@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:50:55 +0200 Subject: [PATCH 14/18] 0.10.0b5: older versions retrain on data from 0.10.0b4 Co-Authored-By: Claude Opus 5.5 --- visionstate/CHANGELOG.md | 9 +++++++++ visionstate/config.yaml | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/visionstate/CHANGELOG.md b/visionstate/CHANGELOG.md index ab34261..9cdee97 100644 --- a/visionstate/CHANGELOG.md +++ b/visionstate/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## 0.10.0b5 + +### Fixed + +- **An older version on data from 0.10.0b4** (a backup made by 0.10.0b4 restored in an older + beta, or an older image started on its data): state sensors were shown as untrained until their + next label. The older version now trains them again from their stored images when it starts, + and coming back to the newer one does the same. + ## 0.10.0b4 ### Better diff --git a/visionstate/config.yaml b/visionstate/config.yaml index 3638340..c255abc 100644 --- a/visionstate/config.yaml +++ b/visionstate/config.yaml @@ -1,5 +1,5 @@ name: VisionState (beta) -version: "0.10.0b4" +version: "0.10.0b5" slug: visionstate description: Teach a local AI to recognise states (open, closed, …), find objects (people, cars, pets) or read numbers in your camera images url: https://github.com/oleost/VisionState From ba6e6fc5b390ecde6aa2facfba45b412c2bac234 Mon Sep 17 00:00:00 2001 From: oleost <5670710+oleost@users.noreply.github.com> Date: Tue, 20 Oct 2026 22:39:31 +0200 Subject: [PATCH 15/18] Restore: wait for the database, keep entities until it runs, watchdog on /alive - Database.close counts the connections still in use before the pool is disposed of (afterwards it counted none, so a restore never waited for them). - Entities of sensors and people the backup does not have are removed once the restored data runs and MQTT is connected; a restore that fails keeps them. - A value only outside today's limits (api.imports.OutOfLimits) no longer refuses a restore; settings this version does not know still do. - Free disk space is checked on the media folder's disk too when it is another one, and again once the upload is in (an upload without Content-Length was not checked). - The heads folder keeps only .npz and the marker files; an old pickle becomes the marker. - Home Assistant's watchdog asks /api/v1/alive, which is never held while a restore swaps the data. Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 2 +- docs/SCOPE.md | 22 +++-- visionstate/backend/tests/test_backup.py | 90 ++++++++++++++++- visionstate/backend/visionstate/api/backup.py | 99 ++++++++++++++----- .../backend/visionstate/api/imports.py | 30 +++++- visionstate/backend/visionstate/api/system.py | 7 ++ visionstate/backend/visionstate/backup.py | 20 ++++ visionstate/backend/visionstate/db.py | 4 +- .../backend/visionstate/engine/runtime.py | 6 ++ visionstate/backend/visionstate/main.py | 8 +- visionstate/config.yaml | 2 +- 11 files changed, 247 insertions(+), 43 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 01d97de..884b162 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -52,7 +52,7 @@ API as `request.app.state.runtime`. | `db.py` | SQLAlchemy tables and the schema migrations (`MIGRATIONS`, `SCHEMA_VERSION`) | | `storage.py` | Image files on disk: training samples, history frames, thumbnails | | `engine/` | The runtime — see below | -| `api/` | The REST API (`/api/v1/…`), one file per topic: `sensors.py` (CRUD, quality, export), `history.py` (the history: one filter for the list, its count and facets; history frames), `samples.py` (training images), `teach.py` (taught boxes; what is taught from a history frame is written into its row: `remember_taught`), `faces.py` (teaching a face on a sensor; the people and their faces, shared by every sensor), `cameras.py` (camera and entity lists, the light, previews), `review.py` (review queue and answers), `imports.py` (import), `backup.py` (backup of everything, restore), `system.py` (UI config, status, AI models and deleting downloaded ones, storage, review rules, review reminder, teaching and face limits), `common.py` (input models and views shared by them) | +| `api/` | The REST API (`/api/v1/…`), one file per topic: `sensors.py` (CRUD, quality, export), `history.py` (the history: one filter for the list, its count and facets; history frames), `samples.py` (training images), `teach.py` (taught boxes; what is taught from a history frame is written into its row: `remember_taught`), `faces.py` (teaching a face on a sensor; the people and their faces, shared by every sensor), `cameras.py` (camera and entity lists, the light, previews), `review.py` (review queue and answers), `imports.py` (import), `backup.py` (backup of everything, restore), `system.py` (UI config, status, `alive` for Home Assistant's watchdog, AI models and deleting downloaded ones, storage, review rules, review reminder, teaching and face limits), `common.py` (input models and views shared by them) | | `sources.py` | Camera sources: Home Assistant camera, HTTP snapshot, RTSP; address checks and size limits; the Home Assistant REST client (states, services, notify services, the app's page) | | `mqtt.py` | MQTT topics, Home Assistant discovery messages, the bridge (connect, publish, commands) | | `ha_events.py` | Home Assistant WebSocket: state changes of trigger entities, entity registry | diff --git a/docs/SCOPE.md b/docs/SCOPE.md index 06e40db..4b3e49b 100644 --- a/docs/SCOPE.md +++ b/docs/SCOPE.md @@ -734,12 +734,16 @@ installations is much code for a rare need. data migrations run twice. - **Safety**: the upload is checked and extracted next to the live data and its database migrated there first; only then is the runtime stopped, its database closed (its worker threads must not - write into the restored one: `Database.close`, waited for), the live data moved aside and the + write into the restored one: `Database.close`, waited for — counted before its pool is disposed + of, which counts none afterwards), the live data moved aside and the restored moved in, and a new runtime started. Any failure puts the old data back; a restore cut off by a stop is undone (or finished) at the next start (`backup.clean_leftovers`). Requests made - meanwhile wait for the new runtime (`BACKUP["hold_requests_s"]`) instead of failing. Archive paths + meanwhile wait for the new runtime (`BACKUP["hold_requests_s"]`) instead of failing — not Home + Assistant's watchdog, which asks `/api/v1/alive` and gives up after a few seconds. Archive paths outside the known folders are ignored. The download is streamed (no temporary file of its size); - the restore needs the upload and its contents on disk for a moment. Uploads reach the app + the restore needs the upload and its contents on disk for a moment: free space is checked on the + data folder's disk and on the media folder's when that is another one, before the upload when + its size is given and again once it is in. Uploads reach the app streamed (`ingress_stream: true` in `config.yaml`): without it the Supervisor reads a request into memory and refuses one over 16 MB, which had also kept videos and sensor bundles under that. The database files are moved aside first, so a restore cut off before that leaves the live @@ -748,15 +752,21 @@ installations is much code for a rare need. there while the app is stopped could outlast Home Assistant's watchdog; the swap only renames. The staging folder is left out of Home Assistant's backups (`backup_exclude`). - **A backup is not trusted more than a sensor bundle**: every sensor in it is validated like a bundle import (`api.imports.validated_sensor`, shared by both), which also catches a setting of a - newer version that needed no new database version (a new reading mode, a new display); its + newer version that needed no new database version (a new reading mode, a new display). A value + that is only outside today's limits (`api.imports.OutOfLimits`: a limit narrowed since) is kept, + as an update keeps it, so it never keeps anyone from restoring their own backup. Its database may hold only tables and indexes (a trigger would run with the app's writes); model files come back only when they are a registry's file with its SHA-256 (`_keep_known_models`: - the app loads a model it finds without checking it again). Not validated: the global settings + the app loads a model it finds without checking it again), and the heads folder keeps only + `.npz` and the marker files (`backup._keep_heads`; an old pickle becomes the marker, so no + version ever unpickles a file from a backup). Not validated: the global settings in `setting` (read through `merge_*` with defaults, like those of an older version). - **Bundle or backup**: the two archives tell each other apart (`manifest.json` / `backup.json`) and say where the other one goes (Import a sensor / Settings → Backup). - **Home Assistant**: sensors and people that are only in the app before the restore have their - entities removed; those in the backup keep their unique IDs and so their entity IDs. + entities removed — once the restored data runs and MQTT is connected (the runtime's + `on_first_connect`), so a restore that fails keeps them; those in the backup keep their unique + IDs and so their entity IDs. ## 13. Configuration diff --git a/visionstate/backend/tests/test_backup.py b/visionstate/backend/tests/test_backup.py index 343c750..1f42438 100644 --- a/visionstate/backend/tests/test_backup.py +++ b/visionstate/backend/tests/test_backup.py @@ -238,6 +238,9 @@ def test_restore_replaces_everything(tmp_path, removed): result = resp.json() assert result["sensors"] == 3 and result["people"] == 1 assert [m["name"] for m in result["masked"]] == ["Garage door", "Water meter"] + # Entities go once the restored data runs and MQTT is connected, not before. + assert removed == {"sensors": [], "people": [], "classes": []} + connected(client) assert removed["sensors"] == ["old_one"] # its entities go; the others keep theirs assert removed["classes"] == ["driveway/dog"] sensors = client.get("/api/v1/sensors").json() @@ -274,12 +277,19 @@ def test_restore_without_history_removes_the_history_here(tmp_path, removed): # Into the same app: its own history is gone with the rows that listed it. assert restore(client, data).status_code == 200 assert client.get("/api/v1/people").json()["people"] == [] + connected(client) assert removed["people"] == ["emma"] assert files_listed(source)["history"] == 0 assert not any(source.history_dir.rglob("*.jpg")) assert files_listed(source)["samples"] == 5 +def connected(client) -> None: + """The runtime running now connects to MQTT (the tests have no broker).""" + assert client.portal is not None + client.portal.call(client.app.state.runtime._on_connect) + + def test_restore_refuses_what_it_cannot_restore(tmp_path): settings = make_settings(tmp_path) populate(settings) @@ -322,6 +332,38 @@ def changed(**fields) -> bytes: assert not backup.stage_dir(settings).exists() +def test_a_value_outside_todays_limits_is_restored(tmp_path): + """A limit made narrower in a later version must not keep anyone from restoring their own + backup: the value is kept, as an update keeps it.""" + settings = make_settings(tmp_path) + good = make(tmp_path / "good.zip").read_bytes() + wider = with_db(good, tmp_path, "UPDATE sensor SET threshold = 1.5 WHERE kind = 'single_state'") + with TestClient(create_app(settings)) as client: + resp = restore(client, wider) + assert resp.status_code == 200, resp.text + sensors = client.get("/api/v1/sensors").json() + assert [s["threshold"] for s in sensors if s["kind"] == "single_state"] == [1.5] + + +def test_restore_checks_free_space(tmp_path, monkeypatch): + """Also an upload that does not say its size (no Content-Length) is checked once it is in.""" + from collections import namedtuple + + from visionstate.api import backup as backup_api + + settings = make_settings(tmp_path) + data = make(tmp_path / "b.zip").read_bytes() + usage = namedtuple("usage", "total used free") + monkeypatch.setattr(backup_api.shutil, "disk_usage", lambda _: usage(1, 1, 1_000)) + with TestClient(create_app(settings)) as client: + assert restore(client, data).status_code == 507 + chunked = client.post( + "/api/v1/backup/restore", content=iter([data]), headers={"content-type": "application/zip"} + ) + assert chunked.status_code == 507 and "Not enough free disk space" in chunked.json()["detail"] + assert not backup.stage_dir(settings).exists() + + def with_db(data: bytes, tmp_path: Path, sql: str) -> bytes: """The backup ``data`` with ``sql`` run on its database.""" db = tmp_path / "edit.db" @@ -379,6 +421,48 @@ def test_only_known_models_come_back(tmp_path): assert kept == [spec.filename] +def test_only_heads_come_back(tmp_path): + """The sensors' heads load without pickles; a pickle from before them (.joblib) becomes the + marker, so neither this version nor an older one taken back to ever unpickles a backup's file.""" + from visionstate.classifier import LEGACY_MARKER + + data = make(tmp_path / "b.zip").read_bytes() + out = io.BytesIO() + with zipfile.ZipFile(io.BytesIO(data)) as src, zipfile.ZipFile(out, "w") as dst: + for item in src.infolist(): + dst.writestr(item, src.read(item)) + dst.writestr("files/heads_dir/7.joblib", b"a pickle") + dst.writestr("files/heads_dir/8.joblib", LEGACY_MARKER) + dst.writestr("files/heads_dir/lib.so", b"x") + dst.writestr("files/heads_dir/1.npz.py", b"x") + dst.writestr("files/heads_dir/sub/2.npz", b"x") + staged = backup.stage(make_settings(tmp_path / "t"), _saved(tmp_path, out.getvalue())) + heads = staged.root / "heads_dir" + assert sorted(p.relative_to(heads).as_posix() for p in heads.rglob("*") if p.is_file()) == [ + "1.npz", + "7.joblib", + "8.joblib", + ] + assert (heads / "7.joblib").read_bytes() == (heads / "8.joblib").read_bytes() == LEGACY_MARKER + + +def test_closing_waits_for_connections_in_use(tmp_path): + """A restore waits for the stopped runtime's threads to let go of the database: the count of + connections still in use must be right (a pool disposed of first counted none).""" + db = Database(tmp_path / "x.db") + db.init() + first, second = db.engine.connect(), db.engine.connect() + in_use = db.close() + assert in_use() == 2 + with pytest.raises(RuntimeError): + with db.session(): + pass + first.close() + second.close() + assert in_use() == 0 + db.engine.dispose() + + def _saved(tmp_path: Path, data: bytes) -> Path: path = tmp_path / "upload.zip" path.write_bytes(data) @@ -402,7 +486,7 @@ def test_restore_ignores_paths_outside_its_folders(tmp_path): assert not (settings.thumbs_dir / "1.jpg").exists() -def test_a_failed_start_puts_the_old_data_back(tmp_path, monkeypatch): +def test_a_failed_start_puts_the_old_data_back(tmp_path, monkeypatch, removed): settings = make_settings(tmp_path) data = make(tmp_path / "b.zip").read_bytes() from visionstate.engine import Runtime @@ -430,6 +514,8 @@ async def start(self): resp = restore(client, data) assert resp.status_code == 500 and "broken" in resp.json()["detail"] assert [s["name"] for s in client.get("/api/v1/sensors").json()] == ["Mine"] + connected(client) + assert removed == {"sensors": [], "people": [], "classes": []} # the old data keeps its entities assert not list(settings.data_dir.glob("*.restore-old")) @@ -560,6 +646,8 @@ def test_requests_wait_while_the_data_is_swapped(tmp_path): asking.start() time.sleep(0.5) assert answers == [] # still waiting + # The watchdog is answered at once: it would restart the app in the middle of a restore. + assert client.get("/api/v1/alive").status_code == 200 client.portal.call(ready.set) asking.join(10) assert answers == [200] diff --git a/visionstate/backend/visionstate/api/backup.py b/visionstate/backend/visionstate/api/backup.py index f3cfe63..1e9ef0e 100644 --- a/visionstate/backend/visionstate/api/backup.py +++ b/visionstate/backend/visionstate/api/backup.py @@ -4,8 +4,11 @@ import asyncio import logging +import os import shutil import zipfile +from collections.abc import Awaitable, Callable +from dataclasses import dataclass from datetime import datetime from fastapi import APIRouter, HTTPException, Request @@ -13,14 +16,14 @@ from sqlalchemy import select from .. import backup -from ..db import Database, Sensor +from ..db import Database, Person, Sensor from ..engine import Runtime from ..engine.state import SensorConfig -from ..mqtt import SensorDescriptor +from ..mqtt import MqttBridge, SensorDescriptor from ..redact import redact -from ..settings import BACKUP, BACKUP_PARTS +from ..settings import BACKUP, BACKUP_PARTS, Settings from .common import API_PREFIX, runtime -from .imports import SENSOR_FIELDS, validated_sensor +from .imports import SENSOR_FIELDS, OutOfLimits, validated_sensor log = logging.getLogger(__name__) router = APIRouter(prefix=API_PREFIX, tags=["backup"]) @@ -76,12 +79,8 @@ async def _restore(request: Request) -> dict: length = int(request.headers.get("content-length") or 0) if length > limit: raise HTTPException(413, f"The backup is larger than {BACKUP['max_restore_gb']} GB") - # The upload, then what it holds, then the old data kept aside until the new runs. - need = 2 * length + BACKUP["free_margin_mb"] * 1_000_000 - if length and shutil.disk_usage(settings.data_dir).free < need: - raise HTTPException( - 507, f"Not enough free disk space: a restore of this backup needs about {need // 1_000_000} MB" - ) + if length: + _check_space(settings, length, uploaded=False) folder = backup.stage_dir(settings) await asyncio.to_thread(backup.clear_stage, settings) upload = folder / "upload.zip" @@ -94,6 +93,7 @@ async def _restore(request: Request) -> dict: if size > limit: raise HTTPException(413, f"The backup is larger than {BACKUP['max_restore_gb']} GB") await asyncio.to_thread(file.write, chunk) + _check_space(settings, size, uploaded=True) # also for an upload that did not say its size try: staged = await asyncio.to_thread(backup.stage, settings, upload) after = await asyncio.to_thread(_staged_sensors, staged) @@ -114,22 +114,53 @@ async def _restore(request: Request) -> dict: } +def _check_space(settings: Settings, size: int, uploaded: bool) -> None: + """Room for a restore of ``size`` bytes: the upload (unless ``uploaded``) and what it holds, + which is extracted next to the folders it replaces (backup._stage_root) — the history on the + media folder's disk, which may be another one. The data from before is only renamed.""" + margin = BACKUP["free_margin_mb"] * 1_000_000 + need = {settings.data_dir: size + margin + (0 if uploaded else size)} + if os.stat(settings.media_dir).st_dev != os.stat(settings.data_dir).st_dev: + need[settings.media_dir] = size + margin + for folder, amount in need.items(): + if shutil.disk_usage(folder).free < amount: + raise HTTPException( + 507, + f"Not enough free disk space in {folder}: a restore of this backup needs about " + f"{amount // 1_000_000} MB there", + ) + + +@dataclass +class _Gone: + """What the data before a restore had in Home Assistant and the backup does not. Forgotten once + the restored data runs, not before: a restore that fails keeps the old data, entities and all.""" + + sensors: list[SensorDescriptor] + changed: list[tuple[SensorDescriptor, SensorDescriptor]] # (before, after) of the same slug + people: list[str] + + async def forget(self, mqtt: MqttBridge) -> None: + # The sensors in the backup keep their unique IDs, and with them their entity IDs. + for descriptor in self.sensors: + await mqtt.remove_discovery(descriptor) + for before, after in self.changed: + await mqtt.remove_stale(before, after) + for key in self.people: + await mqtt.remove_person(key) + + async def _swap(request: Request, old: Runtime, staged: backup.Staged, after: dict[str, SensorDescriptor]) -> None: """Stops the runtime, swaps the data and starts a runtime on it; puts the old data back and starts on that when anything fails. ``after``: the entities of the backup's sensors.""" app = request.app settings = old.settings - # The entities of sensors and people the backup does not have go away in Home Assistant; - # the others keep their unique IDs, and with them their entity IDs. before = await asyncio.to_thread(_descriptors, old.db) - for slug, descriptor in before.items(): - if slug in after: - await old.mqtt.remove_stale(descriptor, after[slug]) - else: - await old.mqtt.remove_discovery(descriptor) - for key, _ in await asyncio.to_thread(old._people): - if key not in staged.people: - await old.mqtt.remove_person(key) + gone = _Gone( + sensors=[descriptor for slug, descriptor in before.items() if slug not in after], + changed=[(descriptor, after[slug]) for slug, descriptor in before.items() if slug in after], + people=[key for key in await asyncio.to_thread(_person_keys, old.db) if key not in staged.people], + ) app.state.ready.clear() # other requests wait for the new runtime await old.stop() await _closed(old.db) @@ -141,7 +172,7 @@ async def _swap(request: Request, old: Runtime, staged: backup.Staged, after: di await _start(app, settings) raise HTTPException(500, f"Restore failed: {redact(str(err))}") from err try: - await _start(app, settings) + await _start(app, settings, gone.forget) except Exception as err: log.exception("Restore: the restored data did not start; the data from before is put back") await _stop(app) @@ -158,10 +189,17 @@ def _descriptors(db: Database) -> dict[str, SensorDescriptor]: return {row.slug: SensorConfig.from_row(row).descriptor for row in s.scalars(select(Sensor))} +def _person_keys(db: Database) -> list[str]: + with db.session() as s: + return list(s.scalars(select(Person.key))) + + def _staged_sensors(staged: backup.Staged) -> dict[str, SensorDescriptor]: """The entities of the backup's sensors, by slug. Each sensor is checked like a sensor import (imports.validated_sensor): a setting this version does not know refuses the restore before - anything is stopped (the database version alone does not show a new choice in a setting).""" + anything is stopped (the database version alone does not show a new choice in a setting). A + value outside today's limits is kept, as an update keeps it: a limit made narrower since must + not keep anyone from restoring their own backup.""" db = Database(staged.root / backup.DATABASE) try: with db.session() as s: @@ -171,6 +209,8 @@ def _staged_sensors(staged: backup.Staged) -> dict[str, SensorDescriptor]: data["states"] = [{"key": st.key, "name": st.name, "color": st.color} for st in row.states] try: validated_sensor(data) + except OutOfLimits as err: + log.warning("Restore: the sensor %r has a setting outside the limits (%s), kept", row.name, err) except ValueError as err: raise ValueError( f"the sensor {row.name!r} has a setting this version does not know ({err}). " @@ -183,11 +223,14 @@ def _staged_sensors(staged: backup.Staged) -> dict[str, SensorDescriptor]: backup.single_file(staged.root / backup.DATABASE) -async def _start(app, settings) -> None: # noqa: ANN001 +async def _start(app, settings: Settings, on_connect: Callable[[MqttBridge], Awaitable[None]] | None = None) -> None: # noqa: ANN001 + """A runtime on the data there is now; ``on_connect``: done once its MQTT is connected.""" db = Database(settings.db_path) try: db.init() rt = Runtime(settings, db) + if on_connect is not None: + rt.on_first_connect.append(on_connect) app.state.runtime = rt await rt.start() except Exception: @@ -206,11 +249,13 @@ async def _stop(app) -> None: # noqa: ANN001 async def _closed(db: Database) -> None: - """Closes a stopped runtime's database and waits for the work still running in its threads - (a training, a clean-up) to let go of it.""" + """Closes a stopped runtime's database: waits for the work still running in its threads (a + training, a clean-up) to let go of it, then closes its connections.""" in_use = db.close() for _ in range(int(BACKUP["release_wait_s"] / 0.1)): if in_use() == 0: - return + break await asyncio.sleep(0.1) - log.warning("Restore: %d database connections still in use after %s s", in_use(), BACKUP["release_wait_s"]) + else: + log.warning("Restore: %d database connections still in use after %s s", in_use(), BACKUP["release_wait_s"]) + db.engine.dispose() diff --git a/visionstate/backend/visionstate/api/imports.py b/visionstate/backend/visionstate/api/imports.py index a6eb20c..9a74a16 100644 --- a/visionstate/backend/visionstate/api/imports.py +++ b/visionstate/backend/visionstate/api/imports.py @@ -138,10 +138,30 @@ def _import_sync(rt: Runtime, path: Path) -> tuple[int, int, bool]: ) +# Pydantic's errors for a value outside a limit (a number, a length), not one this version does not know. +LIMIT_ERRORS = frozenset( + { + "greater_than", + "greater_than_equal", + "less_than", + "less_than_equal", + "too_short", + "too_long", + "string_too_short", + "string_too_long", + } +) + + +class OutOfLimits(ValueError): + """Every setting is one this version knows, but a value is outside today's limits.""" + + def validated_sensor(data: dict) -> SensorIn: """A sensor's settings validated exactly like a sensor made in the UI (kind, limits, source type, states, objects, readings, triggers): a sensor bundle and every sensor of a restored - backup (api/backup.py). ValueError says what is wrong — often a setting of a newer version.""" + backup (api/backup.py). ValueError says what is wrong — often a setting of a newer version; + OutOfLimits (a ValueError too) when it is only a value outside the limits.""" from pydantic import ValidationError from .sensors import SensorIn @@ -149,9 +169,13 @@ def validated_sensor(data: dict) -> SensorIn: try: return SensorIn.model_validate({k: data[k] for k in SENSOR_FIELDS if data.get(k) is not None}).checked() except ValidationError as err: - error = err.errors()[0] + errors = err.errors() + error = errors[0] where = ".".join(str(part) for part in error.get("loc", ())) - raise ValueError(f"{where}: {error.get('msg')}" if where else str(error.get("msg"))) from err + message = f"{where}: {error.get('msg')}" if where else str(error.get("msg")) + if all(e.get("type") in LIMIT_ERRORS for e in errors): + raise OutOfLimits(message) from err + raise ValueError(message) from err except HTTPException as err: raise ValueError(str(err.detail)) from err diff --git a/visionstate/backend/visionstate/api/system.py b/visionstate/backend/visionstate/api/system.py index 340a578..12c2cc0 100644 --- a/visionstate/backend/visionstate/api/system.py +++ b/visionstate/backend/visionstate/api/system.py @@ -148,6 +148,13 @@ def ui_config() -> dict: } +@router.get("/alive") +async def alive() -> dict: + """Home Assistant's watchdog (config.yaml): answers at once, also while a restore swaps the + data (main.py does not hold it), so the watchdog never restarts the app in the middle of one.""" + return {"version": VERSION} + + @router.get("/status") async def status(request: Request) -> dict: """The header and the Settings page: version, counts, MQTT, Home Assistant and model state.""" diff --git a/visionstate/backend/visionstate/backup.py b/visionstate/backend/visionstate/backup.py index 38c49ee..eef0214 100644 --- a/visionstate/backend/visionstate/backup.py +++ b/visionstate/backend/visionstate/backup.py @@ -316,6 +316,7 @@ def stage(settings: Settings, upload: Path) -> Staged: with archive.open(info) as src, target.open("wb") as dst: shutil.copyfileobj(src, dst, 1024 * 1024) _keep_known_models(_stage_root(settings, "models_dir") / "models_dir") + _keep_heads(_stage_root(settings, "heads_dir") / "heads_dir") db_path = root / DATABASE try: con = sqlite3.connect(db_path) @@ -362,6 +363,25 @@ def _keep_known_models(folder: Path) -> None: path.unlink() +def _keep_heads(folder: Path) -> None: + """Only the sensors' heads (``.npz``, loaded without pickles) and the marker files next to + them. A head from before HEAD_FORMAT (a pickle, ``.joblib``) becomes the marker: this version + retrains that sensor all the same, and an older version the app is taken back to never unpickles + what a backup brought.""" + from .classifier import LEGACY_MARKER, LEGACY_SUFFIX + + for path in sorted(folder.rglob("*")) if folder.is_dir() else []: + if not path.is_file(): + continue + name = path.relative_to(folder).as_posix() + stem, dot, suffix = name.partition(".") + if not (stem.isascii() and stem.isdigit()) or dot + suffix not in (".npz", LEGACY_SUFFIX): + log.warning("Restore: left out %s from the sensors' models (not a model VisionState makes)", name[:200]) + path.unlink() + elif dot + suffix == LEGACY_SUFFIX and path.read_bytes() != LEGACY_MARKER: + path.write_bytes(LEGACY_MARKER) + + def single_file(db_path: Path) -> None: """Everything in the one database file (no -wal next to it): only that file is moved in.""" con = sqlite3.connect(db_path) diff --git a/visionstate/backend/visionstate/db.py b/visionstate/backend/visionstate/db.py index f1c1822..a52f432 100644 --- a/visionstate/backend/visionstate/db.py +++ b/visionstate/backend/visionstate/db.py @@ -367,10 +367,10 @@ def _pragmas(dbapi_conn, _record): # noqa: ANN001 def close(self) -> Callable[[], int]: """No session after this (a backup is restored under it: a worker thread of the stopped runtime must not write into the restored database). Returns how many connections are - still in use, as a function to wait on.""" + still in use, as a function to wait on; dispose of the engine once it is 0. Not before: + a pool disposed of counts nothing, and a connection handed back to it afterwards stays open.""" self.closed = True pool = self.engine.pool - self.engine.dispose() return pool.checkedout if isinstance(pool, QueuePool) else lambda: 0 def init(self) -> None: diff --git a/visionstate/backend/visionstate/engine/runtime.py b/visionstate/backend/visionstate/engine/runtime.py index 8c50635..d8bf7a3 100644 --- a/visionstate/backend/visionstate/engine/runtime.py +++ b/visionstate/backend/visionstate/engine/runtime.py @@ -5,6 +5,7 @@ import asyncio import logging import time +from collections.abc import Awaitable, Callable from sqlalchemy import select @@ -33,6 +34,8 @@ def __init__(self, settings: Settings, db: Database): super().__init__(settings, db) self.mqtt = MqttBridge(settings, self._on_command, self._on_connect) self.ha_events = HaEventListener(settings, self._on_ha_state) + # Done once MQTT is connected (a restored backup: the entities the data before it had). + self.on_first_connect: list[Callable[[MqttBridge], Awaitable[None]]] = [] async def start(self) -> None: """Load the settings and the backbone, retrain outdated heads, start every sensor's loop, @@ -78,6 +81,9 @@ async def _on_connect(self) -> None: """MQTT (re)connected: every sensor's entities, then each person's own entity.""" await self._on_mqtt_connect() await self.publish_people() + actions, self.on_first_connect = self.on_first_connect, [] + for action in actions: + await action(self.mqtt) async def stop(self) -> None: for handle in self._retrain_handles.values(): diff --git a/visionstate/backend/visionstate/main.py b/visionstate/backend/visionstate/main.py index 539ab51..05450fa 100644 --- a/visionstate/backend/visionstate/main.py +++ b/visionstate/backend/visionstate/main.py @@ -13,12 +13,15 @@ from . import backup from .api import backup as backup_api from .api import cameras, faces, history, imports, review, samples, sensors, system, teach +from .api.common import API_PREFIX from .db import Database from .engine import Runtime from .settings import BACKUP, INGRESS_PROXY_IP, VERSION, Settings, load_settings log = logging.getLogger("visionstate") +WATCHDOG_PATH = f"{API_PREFIX}/alive" # config.yaml "watchdog" + def create_app(settings: Settings | None = None) -> FastAPI: settings = settings or load_settings() @@ -45,9 +48,10 @@ async def lifespan(app: FastAPI): @app.middleware("http") async def while_restoring(request: Request, call_next): # A restore swaps the data under a stopped runtime for a few seconds: requests wait for - # the new runtime instead of failing. + # the new runtime instead of failing. Not the watchdog's: it gives up after a few seconds. ready: asyncio.Event = request.app.state.ready - if not ready.is_set() and request.url.path.startswith("/api/"): + path = request.url.path + if not ready.is_set() and path.startswith("/api/") and path != WATCHDOG_PATH: try: await asyncio.wait_for(ready.wait(), BACKUP["hold_requests_s"]) except TimeoutError: diff --git a/visionstate/config.yaml b/visionstate/config.yaml index c255abc..3f78cd9 100644 --- a/visionstate/config.yaml +++ b/visionstate/config.yaml @@ -28,7 +28,7 @@ backup_exclude: - "models/*" - "thumbs/*" - ".restore/*" # a VisionState backup being restored (Settings → Backup) -watchdog: "http://[HOST]:[PORT:8099]/api/v1/status" +watchdog: "http://[HOST]:[PORT:8099]/api/v1/alive" options: log_level: info discovery_prefix: homeassistant From 2193e9208d6d8cf58356177e8bb296836b8bd361 Mon Sep 17 00:00:00 2001 From: oleost <5670710+oleost@users.noreply.github.com> Date: Tue, 20 Oct 2026 22:39:51 +0200 Subject: [PATCH 16/18] 0.10.0b6: restoring a backup, watchdog and Home Assistant entities Co-Authored-By: Claude Opus 5.5 --- visionstate/CHANGELOG.md | 16 ++++++++++++++++ visionstate/config.yaml | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/visionstate/CHANGELOG.md b/visionstate/CHANGELOG.md index 9cdee97..3bece1b 100644 --- a/visionstate/CHANGELOG.md +++ b/visionstate/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## 0.10.0b6 + +### Fixed + +- **Restoring a backup**: + - Home Assistant's watchdog is answered at once while the backup is put in place, so it no + longer risks restarting the app in the middle of a restore. + - Entities of sensors and people that are not in the backup are removed only once the restored + data runs; if the restore fails they stay as they were. + - A sensor setting outside today's limits (a limit made narrower in a later version) no longer + refuses the restore of your own backup. + - Free disk space is also checked where the history goes when the media folder is on another + disk (network storage), and for uploads that do not say their size. + - The restore waits properly for the app's background work to let go of the old data. + - Only the trained sensor models VisionState itself makes come back from a backup. + ## 0.10.0b5 ### Fixed diff --git a/visionstate/config.yaml b/visionstate/config.yaml index 3f78cd9..8d3141c 100644 --- a/visionstate/config.yaml +++ b/visionstate/config.yaml @@ -1,5 +1,5 @@ name: VisionState (beta) -version: "0.10.0b5" +version: "0.10.0b6" slug: visionstate description: Teach a local AI to recognise states (open, closed, …), find objects (people, cars, pets) or read numbers in your camera images url: https://github.com/oleost/VisionState From 5511a6df6ea684f026478f90c758d278ef940221 Mon Sep 17 00:00:00 2001 From: oleost <5670710+oleost@users.noreply.github.com> Date: Thu, 22 Oct 2026 10:40:13 +0200 Subject: [PATCH 17/18] 0.10.0b7: Pillow 13.0.0; a full backup round trip before every promote pip-audit blocked the 0.10.0 tag: Pillow 12.3.0 has 24 known vulnerabilities, fixed in 13.0.0. RELEASING.md: the backup check now gives the test sensors what the release adds, restores into an emptied app and back, and compares everything, not only counts. Co-Authored-By: Claude Opus 5.5 --- docs/RELEASING.md | 11 ++++++++--- tools/wheelreader/requirements.txt | 2 +- visionstate/CHANGELOG.md | 7 +++++++ visionstate/backend/requirements.txt | 2 +- visionstate/config.yaml | 2 +- 5 files changed, 18 insertions(+), 6 deletions(-) diff --git a/docs/RELEASING.md b/docs/RELEASING.md index 75cca57..dcd22bb 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -77,9 +77,14 @@ all of it and tell the maintainer what was checked and what was found. - restart Home Assistant, reboot the host and restart the MQTT broker: the app and its entities come back by themselves; - make a backup of the app and restore it: the sensors are back, with their training images; - - move a VisionState backup (Settings → Backup) between the beta app and another app (the dev - app, or the stable app once it has the feature) both ways: the sensors, people, training - images and history come along, and the entities keep their IDs; + - **backup round trip** (Settings → Backup): first give the standing test sensors what the + release adds (a new setting, a new kind of data, a person…), so that the backup has to carry + it. Download a full backup from the beta app (camera passwords ticked), restore it into a + freshly installed app (a local app with its data and media folder emptied first) and back into + the beta app. Compare both sides: every sensor with all its settings, its training images and + what it was taught, people and their faces, the app's settings and limits, downloaded models, + history, and the Home Assistant entities with their IDs. Anything that does not come back + blocks the release; - **soak for 1 hour** with the sensors checking: the app's memory (Supervisor app stats) and its data on disk level off instead of growing, the log stays quiet; - open the app in Home Assistant at desktop width and on a phone (Ingress); diff --git a/tools/wheelreader/requirements.txt b/tools/wheelreader/requirements.txt index 963caef..9c394da 100644 --- a/tools/wheelreader/requirements.txt +++ b/tools/wheelreader/requirements.txt @@ -1,5 +1,5 @@ torch==2.11.0 numpy==2.5.3 -pillow==12.3.0 +pillow==13.0.0 onnx==1.23.2 onnxruntime==1.31.0 diff --git a/visionstate/CHANGELOG.md b/visionstate/CHANGELOG.md index 3bece1b..a8faaf6 100644 --- a/visionstate/CHANGELOG.md +++ b/visionstate/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## 0.10.0b7 + +### Fixed + +- Pillow, the library that reads and writes the camera images, is updated to 13.0.0 for 24 + security fixes. + ## 0.10.0b6 ### Fixed diff --git a/visionstate/backend/requirements.txt b/visionstate/backend/requirements.txt index 7ac18bc..31dd7d0 100644 --- a/visionstate/backend/requirements.txt +++ b/visionstate/backend/requirements.txt @@ -6,6 +6,6 @@ httpx==0.28.1 aiomqtt==2.5.1 onnxruntime==1.30.0 numpy==2.3.5 -pillow==12.3.0 +pillow==13.0.0 av==19.0.0 websockets==17.1 diff --git a/visionstate/config.yaml b/visionstate/config.yaml index 8d3141c..d3463cd 100644 --- a/visionstate/config.yaml +++ b/visionstate/config.yaml @@ -1,5 +1,5 @@ name: VisionState (beta) -version: "0.10.0b6" +version: "0.10.0b7" slug: visionstate description: Teach a local AI to recognise states (open, closed, …), find objects (people, cars, pets) or read numbers in your camera images url: https://github.com/oleost/VisionState From b83de969fef28cee55a5dbae22accc796352deb2 Mon Sep 17 00:00:00 2001 From: oleost <5670710+oleost@users.noreply.github.com> Date: Thu, 22 Oct 2026 13:12:28 +0200 Subject: [PATCH 18/18] 0.10.0: backup and restore, less memory Co-Authored-By: Claude Opus 5.5 --- docs/SCOPE.md | 6 +-- visionstate/CHANGELOG.md | 95 +++++++++++----------------------------- visionstate/config.yaml | 8 ++-- 3 files changed, 32 insertions(+), 77 deletions(-) diff --git a/docs/SCOPE.md b/docs/SCOPE.md index 4b3e49b..e46fb47 100644 --- a/docs/SCOPE.md +++ b/docs/SCOPE.md @@ -800,12 +800,12 @@ is part of backups and needs no restart. | **Recognising faces** ✅ | Faces in person boxes (YuNet), people taught by face and shared by every camera (SFace, or AuraFace for a fast PC), a People page, an entity per person saying where they are, unknown person, review questions, export opt-in — phases 1–3 of [`FACE_RECOGNITION_PLAN.md`](FACE_RECOGNITION_PLAN.md), and teaching a face from photos | 0.9.0 (betas 0.8.1b3–b5) | | **Counter limits** ✅ | A counter's limit grows with the time it was away (learned, a fixed rate or per reading), Review moves a counter on, readings held when the camera's picture changes size, training images in the app's storage (backups hold them) | 0.9.1 (betas 0.9.1b1–b6) | | **Reviewing readings** ✅ | A counter's value sent at its first reading, decimals for its rate, a rejected reading that repeats waits in Review once, Review shows a reading's region large and one sensor at a time | 0.9.2 (betas 0.9.2b1–b4) | -| **Backup of everything** | One file with everything (parts to leave out, camera passwords only when ticked), a restore that replaces it and keeps the entities, backups of every database version kept in the tests; uploads over 16 MB through Ingress | betas 0.10.0b1–b3 | +| **Backup of everything** ✅ | One file with everything (parts to leave out, camera passwords only when ticked), a restore that replaces it and keeps the entities, backups of every database version kept in the tests; uploads over 16 MB through Ingress; much less memory (no ONNX Runtime arena, classifier without scikit-learn) | 0.10.0 (betas 0.10.0b1–b7) | | **Better counter readings** | Benchmark, the last value as an expectation, calibrated confidence, extra decimal, self-calibrating regions, wheels-v2, learning per sensor — status per phase in [`WHEEL_READER_PLAN.md`](WHEEL_READER_PLAN.md) | planned | **Open ideas** (not scheduled): more on the History page (export of a filtered selection, a -summary per hour or day, "seen in the frame" as a filter, saved filters, bulk actions); full -export/import of everything; merge/replace import; +summary per hour or day, "seen in the frame" as a filter, saved filters, bulk actions); merge +import of a backup; less MQTT/camera traffic (throttle frame publishing, reuse the engine's latest frame in the UI); video de-duplication on the ROI instead of the full frame; light theme following Home Assistant; mechanical counters beyond [`WHEEL_READER_PLAN.md`](WHEEL_READER_PLAN.md): pointer diff --git a/visionstate/CHANGELOG.md b/visionstate/CHANGELOG.md index a8faaf6..3cf7e70 100644 --- a/visionstate/CHANGELOG.md +++ b/visionstate/CHANGELOG.md @@ -1,38 +1,21 @@ # Changelog -## 0.10.0b7 +## 0.10.0 -### Fixed - -- Pillow, the library that reads and writes the camera images, is updated to 13.0.0 for 24 - security fixes. - -## 0.10.0b6 - -### Fixed - -- **Restoring a backup**: - - Home Assistant's watchdog is answered at once while the backup is put in place, so it no - longer risks restarting the app in the middle of a restore. - - Entities of sensors and people that are not in the backup are removed only once the restored - data runs; if the restore fails they stay as they were. - - A sensor setting outside today's limits (a limit made narrower in a later version) no longer - refuses the restore of your own backup. - - Free disk space is also checked where the history goes when the media folder is on another - disk (network storage), and for uploads that do not say their size. - - The restore waits properly for the app's background work to let go of the old data. - - Only the trained sensor models VisionState itself makes come back from a backup. - -## 0.10.0b5 - -### Fixed - -- **An older version on data from 0.10.0b4** (a backup made by 0.10.0b4 restored in an older - beta, or an older image started on its data): state sensors were shown as untrained until their - next label. The older version now trains them again from their stored images when it starts, - and coming back to the newer one does the same. +### New -## 0.10.0b4 +- **Backup of everything** (Settings → Backup): one file with your sensors and settings, training + images, people and their faces, history and downloaded AI models — to move VisionState to + another Home Assistant, or between the stable and the beta app. Everything is ticked; untick what + you do not need (each shows its size). Camera passwords are left out unless you tick them. +- **Restore** replaces everything in the app with a backup. Sensors keep their entities and entity + IDs in Home Assistant; sensors and people that are not in the backup are removed only once the + restored data runs. The backup is checked first (its version, every sensor's settings, and the + free disk space, also where the history goes when the media folder is on network storage), and + if anything goes wrong, even if the app is stopped in the middle, it keeps what it had. A backup + from an older version is upgraded; one from a newer version is refused (update the app first). + The sensors whose camera password must be entered again are listed. Only the AI models and + trained sensor models VisionState itself makes come back from a backup. ### Better @@ -41,44 +24,8 @@ loads scikit-learn. On a test installation with nine sensors and face recognition the app went from about 940 MB to about 580 MB. A check takes a few percent longer. - State sensors are retrained once after the update, from their stored training images (it takes - a moment, and the sensors keep their last state meanwhile). - -## 0.10.0b3 - -### Fixed - -- **Restoring a backup with the media folder on network storage**: the history is now unpacked - on the disk it goes to, so the app is stopped only for a moment while the backup is put in - place — copying it over the network then could take long enough for Home Assistant to restart - the app, which put the old data back. -- A rare "ResizeObserver loop" error on the dashboard in Safari and the Home Assistant app on iPhone. - -## 0.10.0b2 - -### Fixed - -- **A restore cut off at the wrong moment could lose the database**: if the app stopped while a - backup was being restored, its next start could remove the database instead of putting it back. - The database is now always moved first, and one that was never moved is left alone. -- A backup from a newer version that adds a setting to a sensor (a new reading mode, say) is now - refused with the sensor's name, like a sensor export from a newer version, instead of being - restored into an app that does not know the setting. -- Only the AI models VisionState itself downloads come back from a backup, and only unchanged. -- Restoring a sensor export, or importing a backup as a sensor, now says where the file goes. - -## 0.10.0b1 - -### New - -- **Backup of everything** (Settings → Backup): one file with your sensors and settings, training - images, people and their faces, history and downloaded AI models — to move VisionState to - another Home Assistant, or between the stable and the beta app. Everything is ticked; untick what - you do not need (each shows its size). Camera passwords are left out unless you tick them. -- **Restore** replaces everything in the app with a backup. Sensors keep their entities and entity - IDs in Home Assistant; what is only in the app before is removed. The backup is checked first, - and if anything goes wrong the app keeps what it had. A backup from an older version is upgraded; - one from a newer version is refused (update the app first). The sensors whose camera password - must be entered again are listed. + a moment, and the sensors keep their last state meanwhile). Going back to 0.9.2 afterwards + works too: it retrains them the same way. ### Fixed @@ -86,6 +33,16 @@ Home Assistant. - An older version started on a newer database (going back to a backup made before an update) no longer makes the next update repeat its database upgrade. +- A rare "ResizeObserver loop" error on the dashboard in Safari and the Home Assistant app on iPhone. +- Pillow, the library that reads and writes the camera images, is updated to 13.0.0 for 24 + security fixes. + +### Help wanted: Raspberry Pi and other ARM boards + +The lower memory use may cost a little more CPU. On an x86 PC the difference is negligible, but on +ARM it has not been measured yet. If you run VisionState on a Raspberry Pi or another ARM board, +please share in [issue #52](https://github.com/oleost/VisionState/issues/52) how memory and CPU +compare with 0.9.2. Even rough numbers or "no difference I can see" help. ## 0.9.2 diff --git a/visionstate/config.yaml b/visionstate/config.yaml index d3463cd..4a40245 100644 --- a/visionstate/config.yaml +++ b/visionstate/config.yaml @@ -1,5 +1,5 @@ -name: VisionState (beta) -version: "0.10.0b7" +name: VisionState +version: "0.10.0" slug: visionstate description: Teach a local AI to recognise states (open, closed, …), find objects (people, cars, pets) or read numbers in your camera images url: https://github.com/oleost/VisionState @@ -16,7 +16,7 @@ ingress_port: 8099 # a request into memory and refuses one over 16 MB. ingress_stream: true panel_icon: mdi:eye-check-outline -panel_title: VisionState β +panel_title: VisionState homeassistant_api: true hassio_api: true services: @@ -39,5 +39,3 @@ schema: mqtt_port: port? mqtt_username: str? mqtt_password: password? -environment: - VISIONSTATE_MEDIA: /media/visionstate_beta