diff --git a/.agents/skills/extenddb/references/setup/04-init-stage.md b/.agents/skills/extenddb/references/setup/04-init-stage.md index b17d74d40..ff4688a83 100644 --- a/.agents/skills/extenddb/references/setup/04-init-stage.md +++ b/.agents/skills/extenddb/references/setup/04-init-stage.md @@ -14,7 +14,7 @@ First, check whether a config file is already present at the repo root. test -f extenddb.toml && echo "extenddb.toml already exists" || echo "extenddb.toml absent" ``` -`extenddb.toml` on its own does not block `extenddb init`. If `extenddb.toml` exists, `init` loads defaults from it and continues. The real blocker is whether the `extenddb_catalog` and `extenddb` data databases exist in PostgreSQL. +`extenddb.toml` on its own blocks `extenddb init`: if it exists, `init` refuses with `Error: Config file "extenddb.toml" already exists. Use --overwrite to delete and regenerate it.` and exits 255. With `--overwrite`, `init` loads the PostgreSQL connection defaults from the file and continues. The other blocker is whether the `extenddb_catalog` and `extenddb` data databases exist in PostgreSQL. Second, check whether a prior deployment is present in PostgreSQL. diff --git a/.agents/skills/extenddb/references/troubleshooting/07-runtime-symptoms.md b/.agents/skills/extenddb/references/troubleshooting/07-runtime-symptoms.md index 86c85875f..a7b14f69f 100644 --- a/.agents/skills/extenddb/references/troubleshooting/07-runtime-symptoms.md +++ b/.agents/skills/extenddb/references/troubleshooting/07-runtime-symptoms.md @@ -21,7 +21,7 @@ HTTP 500 on all requests under heavy load pool_size = 50 # default is 20 ``` -If the problem persists, check for long-running queries or connection leaks with `SELECT * FROM pg_stat_activity WHERE datname = 'extenddb_data';`. +If the problem persists, check for long-running queries or connection leaks with `SELECT * FROM pg_stat_activity WHERE datname = 'extenddb';` (the data database's default name). **Known limitation:** The HTTP status code should be 503 with a `Retry-After` header. This is tracked as technical debt. diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 071500c00..9c0881688 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -243,7 +243,7 @@ jobs: EXTENDDB_ADMIN_PASSWORD: ${{ steps.init.outputs.admin_password }} EXTENDDB_TEST_PG_ADMIN_CONNECTION_STRING: postgresql://postgres:devpass@127.0.0.1:5432 EXTENDDB_TEST_BACKUP_STORE: filesystem - run: devtools/run-tests --extenddb --pytest --comprehensive --terraform --parallel --filter "not import_export" + run: devtools/run-tests --extenddb --pytest --comprehensive --terraform --parallel # moto for the job suite's store-outage cell, the one cell that # exercises `spool_path` on PostgreSQL (an S3 download into the spool, @@ -422,7 +422,7 @@ jobs: # A file-backed SQLite server without [backup.store] writes to the # defaulted root beside the database file. EXTENDDB_TEST_BACKUP_STORE: filesystem - run: devtools/run-tests --extenddb --pytest --comprehensive --terraform --parallel --filter "not import_export" + run: devtools/run-tests --extenddb --pytest --comprehensive --terraform --parallel - name: Run the backup store and job suites run: python3 -m pytest tests/test_backup_store_config.py tests/test_backup_job.py -v -rP -p no:cacheprovider diff --git a/AGENTS.md b/AGENTS.md index 33908f768..3a42cbf8c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -151,7 +151,7 @@ cargo clippy --all-targets -- -D warnings ``` This creates: -- Databases (`extenddb_catalog`, `extenddb_data`) +- Databases (`extenddb_catalog` and `extenddb` on PostgreSQL; `extenddb_catalog` and `extenddb_data` on MongoDB) - Admin user credentials (printed to stdout — save the password!) - Self-signed TLS certificate at `~/.extenddb/tls/cert.pem` - Config file `extenddb.toml` @@ -397,6 +397,7 @@ Expression parsing lives in `crates/core/src/expression/`. This is pure sync Rus | Admin Guide | `docs/manuals/05-admin-guide.md` | Server lifecycle, configuration, IAM | | Security Model | `docs/manuals/10-security-model.md` | Threat model, auth, authz | | Deployment Guide | `docs/manuals/11-deployment-guide.md` | Self-hosted, multi-cloud, air-gapped | +| Backup and Restore | `docs/manuals/14-backup-and-restore.md` | Backup strategy, drills, recovery | | Differences from DynamoDB | `docs/differences-from-dynamodb.md` | Behavioral differences | | Troubleshooting | `docs/troubleshooting.md` | Common errors and solutions | | Storage Component Design | `docs/design/04-component-storage.md` | Storage trait design | @@ -406,7 +407,7 @@ Expression parsing lives in `crates/core/src/expression/`. This is pure sync Rus | Local MongoDB Setup | `docs/local-mongodb-setup.md` | MongoDB installation and config | | Cassandra Setup | `docs/cassandra-setup.md` | Cassandra install, config, and deployment (experimental) | | Testing Design | `docs/design/09-testing.md` | Test strategy and infrastructure | -| Extending Storage | `12-backend-plugin-architecture.md` | Guide to implementing storage backends | +| Extending Storage | `docs/manuals/12-extending-extenddb-storage.md` | Guide to implementing storage backends | Build PDFs: diff --git a/Cargo.lock b/Cargo.lock index faabdd021..409effa96 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1873,13 +1873,16 @@ dependencies = [ "chrono", "crc32fast", "extenddb-auth", + "extenddb-backup-store", "extenddb-core", + "extenddb-engine", "extenddb-storage", "extenddb-storage-cassandra", "futures", "rand 0.9.4", "serde", "serde_json", + "tempfile", "time", "tokio", "toml", diff --git a/README.md b/README.md index f5cf0ff55..7520033c4 100755 --- a/README.md +++ b/README.md @@ -259,6 +259,7 @@ docs/ | [Admin Guide](docs/manuals/05-admin-guide.md) | Server lifecycle, configuration, IAM management | | [Security Model](docs/manuals/10-security-model.md) | Threat model, authentication, authorization | | [Deployment Guide](docs/manuals/11-deployment-guide.md) | Self-hosted, multi-cloud, air-gapped deployments | +| [Backup and Restore](docs/manuals/14-backup-and-restore.md) | Backup strategy, recovery figures, restore drill, catalog-loss recovery | | [Differences from DynamoDB](docs/differences-from-dynamodb.md) | Behavioral differences and unsupported operations | | [Troubleshooting](docs/troubleshooting.md) | Common errors and solutions | | [Verifying Container Images](docs/verifying-images.md) | Build provenance and SBOM attestations for the published images | diff --git a/crates/app/src/cmd_backup.rs b/crates/app/src/cmd_backup.rs index 99a8f1551..39ce1220a 100644 --- a/crates/app/src/cmd_backup.rs +++ b/crates/app/src/cmd_backup.rs @@ -179,6 +179,43 @@ //! A row `--prune-missing` leaves because its manifest is back or another //! writer moved it is reported, and the exit stays 0. //! +//! Every sync that listed the root to its end then reports the rows of the +//! stores other than the configured one: per store (kind, root, id) the +//! rows per status and each CREATING and DELETING row by ARN, which no +//! sync changes (`extenddb_engine::store_maintenance`'s module doc states +//! all three options below). `--move-store ` re-points the rows +//! of a root whose tree was copied under the configured root: it needs the +//! configured store healthy, refuses the configured root itself and a root +//! with no rows (printing the stores the catalog knows), takes the claim +//! (on SQLite too) and waits for the jobs of the configured store and of +//! every store id of the old root, checks every row against the copy and +//! names each that stops the move, moves none while any does, and then +//! syncs the configured root; it exits 1 when a row stops it, is left +//! under the old root, or lands there during that sync (the closing report +//! of other stores names it). `--forget-store ` and `--forget-backup +//! ` (repeatable, in order, up to the first refusal) remove rows +//! outright, delete no object (but for a DELETING row of the configured +//! store, whose deletion is finished first when the store's check passed), +//! and run on a store in any state and with no store configured, under the +//! claim on every backend; a configured root that cannot be opened (a +//! filesystem directory that does not exist) stops them before the claim, +//! as it stops plain sync, and `kind = "disabled"` runs them. They open the +//! store writing nothing to it ([`Mode::Forget`]). They exit 1 for a row they leave (a restore reading +//! it, a live owner, another spelling of the ARN, an in-catalog backup, a +//! DELETING row of an unhealthy configured store, a row of a configured +//! root that has no identity file under its rows, or on SQLite of the +//! default root found missing with rows for it). After `--forget-backup` +//! the configured store's check runs once and its outcome is printed: a +//! store left unhealthy by the removed row's missing manifest reads +//! healthy, or names the next row whose manifest is gone. The three +//! options conflict with each other, with `--adopt-store` and with +//! `--prune-missing`. Each row they change is logged under +//! `extenddb::audit`. Like sync's insert, each step of theirs that cannot be +//! undone is preceded by a beat of the claim (`HeldClaim::recheck`; +//! `extenddb_engine::store_maintenance`'s module doc lists the steps and the +//! bound that remains), and a claim found lost there stops the command, +//! exit 1, naming the step. +//! //! Every `backup` command refuses an in-memory SQLite database (a //! `[storage.sqlite] path` naming no file) before it opens the catalog: it //! would open a new, empty catalog that ends with the command. A server on @@ -193,9 +230,10 @@ use extenddb_engine::maintenance::{ Interrupt, RemoveTarget, Stop, Wait, WaitOutcome, purge_rows, remove_target, wait_for_quiet, }; use extenddb_engine::{BackupRuntime, ConfiguredStore, HeldClaim, StoreHealth, StoreProblem}; +use extenddb_server::backup_store::NotConfigured; use extenddb_storage::StorageEngine; use extenddb_storage::backup_catalog::{ - BackupRow, BackupStatus, JOB_HEARTBEAT_INTERVAL, JobStatus, StoreIdentity, + BackupRow, BackupStatus, JOB_HEARTBEAT_INTERVAL, JobStatus, StoreIdentity, StoreKind, }; use extenddb_storage::error::StorageError; use extenddb_storage::hooks::ServingRegistrar; @@ -245,9 +283,9 @@ enum BackupCommand { /// Give a catalog row to every backup of the configured backup store that has none: each /// prefix with a verified manifest (its sidecar matching, every data file present at its /// size, its ARN naming the prefix's account, table and backup id) gets an AVAILABLE row. - /// Reports the prefixes it skips and the AVAILABLE rows whose manifest is gone. Refuses on - /// an unhealthy store. Beside servers (PostgreSQL, MongoDB, Cassandra) it takes the - /// maintenance claim; on SQLite it needs the server stopped. + /// Reports the prefixes it skips, the AVAILABLE rows whose manifest is gone and the rows of + /// other stores. Refuses on an unhealthy store. Beside servers (PostgreSQL, MongoDB, + /// Cassandra) it takes the maintenance claim; on SQLite it needs the server stopped. Sync { /// Make this catalog the owner of a store root whose extenddb-store.json names another /// catalog, then sync. Stop every server of the catalog the file names first. Takes the @@ -259,6 +297,48 @@ enum BackupCommand { /// of the same path). Takes the maintenance claim first and waits for running jobs. #[arg(long)] prune_missing: bool, + /// Move the rows of OLD_ROOT (its label as the catalog stores it, as backup list and + /// the report of other stores print it) to the configured store, after the old root's + /// tree was copied under the configured root. Reconfigure every server first. Checks + /// each row against the copy and moves none while any row stops the move (all are + /// named); then syncs. Takes the maintenance claim and waits for running jobs. + #[arg( + long, + value_name = "OLD_ROOT", + conflicts_with_all = ["adopt_store", "prune_missing", "forget_store", "forget_backup"] + )] + move_store: Option, + /// Remove every catalog row of ROOT, a root other than the configured one, outright: + /// no tombstone is left and no object is deleted, so a later sync where the bytes are + /// adopts them again. Runs on a store in any state and with no store configured; a + /// configured root that cannot be opened (a directory that does not exist) stops it, + /// and [backup.store] kind = "disabled" runs it. Takes the maintenance claim and waits + /// for running jobs, as purge does; a row whose job writes a heartbeat after the wait is + /// left and named. + #[arg( + long, + value_name = "ROOT", + conflicts_with_all = ["adopt_store", "prune_missing", "forget_backup"] + )] + forget_store: Option, + /// Remove the catalog row of the backup ARN outright, of any status and any store: no + /// tombstone is left and no object is deleted, except that a DELETING row of the + /// configured store has its deletion finished first; so a later sync where the bytes are + /// adopts the backup again (DeleteBackup is the way to delete one). Repeat the option for + /// more backups; they are removed in order, up to the first that is refused. Runs on a + /// store in any state and with no store configured (a configured root that cannot be + /// opened, such as a directory that does not exist, stops it, and [backup.store] kind = + /// "disabled" runs it), and runs the store's check afterwards unless the root had no + /// extenddb-store.json. Takes the maintenance claim and waits for + /// running jobs, as purge does; a row whose job writes a heartbeat after the wait is + /// left and named. + #[arg( + long, + value_name = "ARN", + action = clap::ArgAction::Append, + conflicts_with_all = ["adopt_store", "prune_missing"] + )] + forget_backup: Vec, }, } @@ -326,21 +406,67 @@ pub async fn run(args: BackupArgs, release: &'static str) -> anyhow::Result<()> catalog.close().await; return done; } + // The forget options run with no store configured too: the catalog is + // kept open without one. + let forgetting = matches!( + &args.command, + BackupCommand::Sync { forget_store, forget_backup, .. } + if forget_store.is_some() || !forget_backup.is_empty() + ); let (command, mode) = match args.command { BackupCommand::List { .. } => (LIST, Mode::Read), BackupCommand::Purge { confirm: false } => (PURGE, Mode::Read), BackupCommand::Purge { confirm: true } => (PURGE, Mode::Write), + BackupCommand::Sync { .. } if forgetting => (SYNC, Mode::Forget), BackupCommand::Sync { .. } => (SYNC, Mode::Sync), BackupCommand::RemoveTarget { .. } => unreachable!("handled above"), }; - let opening = open(&app_config, &args.config, release, command, mode).await?; + let opening = open( + &app_config, + &args.config, + release, + command, + mode, + forgetting, + ) + .await?; if let Some(signal) = interrupt.signalled() { - if let Opening::Store(opened) = opening { - opened.close().await; + match opening { + Opening::Store(opened) => opened.close().await, + Opening::CatalogOnly(catalog, _) => catalog.close().await, + Opening::NoStore(_) => {} } anyhow::bail!("{signal} received while opening the catalog; nothing was changed"); } match (args.command, opening) { + ( + BackupCommand::Sync { + forget_store, + forget_backup, + .. + }, + opening, + ) if forgetting => { + let (catalog, store) = match opening { + Opening::Store(opened) => { + let Opened { catalog, identity } = opened; + (catalog, Ok(identity)) + } + Opening::CatalogOnly(catalog, why) => (catalog, Err(why)), + Opening::NoStore(why) => anyhow::bail!("{why}"), + }; + let done = forget( + &catalog, + store.as_ref(), + &app_config, + forget_store.as_deref(), + &forget_backup, + interrupt, + ) + .await; + catalog.close().await; + done + } (BackupCommand::List { failed }, Opening::Store(opened)) => { let done = list(&opened, failed).await; opened.close().await; @@ -382,21 +508,39 @@ pub async fn run(args: BackupArgs, release: &'static str) -> anyhow::Result<()> BackupCommand::Sync { adopt_store, prune_missing, + move_store, + .. }, Opening::Store(opened), ) => { - let done = sync(&opened, &app_config, adopt_store, prune_missing, interrupt).await; + let done = sync( + &opened, + &app_config, + SyncOptions { + adopt_store, + prune_missing, + move_store: move_store.as_deref(), + }, + interrupt, + ) + .await; opened.close().await; done } (BackupCommand::Sync { .. }, Opening::NoStore(reason)) => anyhow::bail!("{reason}"), + (_, Opening::CatalogOnly(catalog, reason)) => { + catalog.close().await; + anyhow::bail!("{reason}") + } } } -/// What `open` found: the configured store, or why there is none. +/// What `open` found: the configured store, or why there is none (with the +/// catalog still open when the caller asked to keep it). enum Opening { Store(Opened), NoStore(String), + CatalogOnly(Catalog, NotConfigured), } /// How a command opens the catalog. @@ -421,6 +565,14 @@ enum Mode { /// (`remove-target`): the store is described as `Read` leaves it, so /// neither the default root nor an identity file is created. WriteCatalogOnly, + /// As `Write`, for `backup sync --forget-store` and `--forget-backup`, + /// which write nothing to the root: the store is described as `Read` + /// leaves it (no default root created, no `validate` and its probe + /// file), the identity file is read and never written, and when it + /// agrees with the rows the check that writes nothing + /// (`check_configured_store_read_only`) runs once and is recorded, so + /// a DELETING row of a healthy store can have its deletion finished. + Forget, } /// The open catalog: the storage engine, the backup runtime (with the @@ -453,10 +605,7 @@ async fn open_catalog( release: &'static str, command: &'static str, mode: Mode, -) -> anyhow::Result<( - Catalog, - Option, -)> { +) -> anyhow::Result<(Catalog, Option)> { let mut options = extenddb_storage::server_components::ServerComponentsOptions::default(); options.release = release; options.max_concurrent_jobs = app_config.backup.max_concurrent_jobs; @@ -502,7 +651,7 @@ async fn open_catalog( &backend, &*storage, match mode { - Mode::Read | Mode::WriteCatalogOnly => { + Mode::Read | Mode::WriteCatalogOnly | Mode::Forget => { extenddb_server::backup_store::Caller::ReadOnlyCommand } Mode::Write => extenddb_server::backup_store::Caller::Command, @@ -567,6 +716,7 @@ async fn open( release: &'static str, command: &'static str, mode: Mode, + keep_without_store: bool, ) -> anyhow::Result { let (catalog, not_configured) = open_catalog(app_config, config_path, release, command, mode).await?; @@ -575,6 +725,9 @@ async fn open( // absent or disabled, the default root not yet created, or the // default root failed its check with no rows for it. let why = not_configured.expect("a runtime without a store carries the reason"); + if keep_without_store { + return Ok(Opening::CatalogOnly(catalog, why)); + } catalog.close().await; return Ok(Opening::NoStore(why.to_string())); }; @@ -594,6 +747,32 @@ async fn open( Err(problem) => Identity::Problem(problem), } } + (Mode::Forget, _) => { + match extenddb_engine::read_identity_bounded(store, &**storage).await { + Ok(Some(identity)) => { + let check = extenddb_engine::check_configured_store_read_only( + store, + &**storage, + Some(&identity), + ) + .await; + // A file gone since the read leaves the store checking: + // not healthy, so nothing is deleted. + match check { + Ok(Some(checked)) => { + store.record_check(Ok(checked)); + } + Ok(None) => {} + Err(problem) => { + store.record_check(Err(problem)); + } + } + Identity::Known(identity) + } + Ok(None) => Identity::Empty, + Err(problem) => Identity::Problem(problem), + } + } (Mode::Write | Mode::Sync, StoreHealth::Checking) => { let outcome = extenddb_engine::check_configured_store(store, &**storage, backend).await; store.record_check(outcome.clone()); @@ -856,7 +1035,7 @@ async fn purge( // live owner is told apart by is zero. On the other backends purge runs // beside live servers, whose jobs heartbeat within `stale_after`. let owner_window = if opened.catalog.backend == "sqlite" { - resolve_dead_owners(opened).await; + resolve_dead_owners(&opened.catalog).await; Duration::ZERO } else { stale_after @@ -867,7 +1046,7 @@ async fn purge( let started = Instant::now(); let outcome = wait_for_quiet( storage, - identity, + std::slice::from_ref(identity), &claim, Wait { owner_window, @@ -1029,13 +1208,26 @@ async fn signal_name() -> &'static str { /// command's own claim would read as held only within one instant of its /// heartbeat). What was resolved is printed; a failure is printed and the /// wait decides. -async fn resolve_dead_owners(opened: &Opened) { +async fn resolve_dead_owners(catalog: &Catalog) -> Vec { + resolve_dead_owners_with( + catalog, + extenddb_engine::reconciler::ReconcilerState::for_maintenance_holder(), + ) + .await +} + +/// [`resolve_dead_owners`] with the cycle's `state`; returns the ARNs of +/// the DELETING rows whose deletion the cycle finished. +async fn resolve_dead_owners_with( + catalog: &Catalog, + mut state: extenddb_engine::reconciler::ReconcilerState, +) -> Vec { match extenddb_engine::reconciler::cycle( - &opened.catalog.storage, - &opened.catalog.runtime, + &catalog.storage, + &catalog.runtime, Duration::ZERO, - &opened.catalog.backend, - &mut extenddb_engine::reconciler::ReconcilerState::for_maintenance_holder(), + &catalog.backend, + &mut state, ) .await { @@ -1090,8 +1282,12 @@ async fn resolve_dead_owners(opened: &Opened) { command's write" ); } + report.deletions_finished + } + Err(e) => { + println!("could not resolve the jobs the stopped server left: {e}"); + Vec::new() } - Err(e) => println!("could not resolve the jobs the stopped server left: {e}"), } } @@ -1157,15 +1353,29 @@ async fn remove_restore_target( Ok(()) } -/// `backup sync [--adopt-store] [--prune-missing]` (module doc). +/// The options `sync` was given (`--forget-store` and `--forget-backup` +/// are [`forget`]'s). +#[derive(Clone, Copy)] +struct SyncOptions<'a> { + adopt_store: bool, + prune_missing: bool, + move_store: Option<&'a str>, +} + +/// `backup sync [--adopt-store] [--prune-missing] [--move-store ]` +/// (module doc). #[allow(clippy::too_many_lines)] async fn sync( opened: &Opened, app_config: &config::AppConfig, - adopt_store: bool, - prune_missing: bool, + options: SyncOptions<'_>, mut interrupt: Interrupt, ) -> anyhow::Result<()> { + let SyncOptions { + adopt_store, + prune_missing, + move_store, + } = options; let storage = &*opened.catalog.storage; let configured = opened.store(); println!("=== extenddb backup sync ==="); @@ -1215,9 +1425,10 @@ async fn sync( } Identity::Problem(problem) => anyhow::bail!( "the {} backup store {} is unhealthy: {problem}; sync changes no row while the store \ - is unhealthy", + is unhealthy{}", configured.kind, - configured.root + configured.root, + manifest_missing_hint(problem) ), Identity::Empty => anyhow::bail!( "the {} backup store {} has no {}: its check did not run", @@ -1227,6 +1438,24 @@ async fn sync( ), }; + // The old root `--move-store` names: refused before the claim when it is + // the configured root or has no rows. + let moving = match move_store { + Some(root) => { + match extenddb_engine::store_maintenance::locate_root( + storage, + root, + Some(&configured.root), + ) + .await + { + Ok(located) => Some(located), + Err(refusal) => return Err(root_refusal("--move-store", root, refusal)), + } + } + None => None, + }; + // On SQLite the command holds the serve lock, which already excludes // every server and every other command, so plain sync takes no claim // and resolves nothing a stopped server left: its rows are skipped, and @@ -1235,44 +1464,42 @@ async fn sync( // maintenance claim, so no purge removes a row between this sync's row // read and its insert and then deletes the bytes it adopted; plain sync // waits for no job, since its insert-if-absent orders it against a - // CreateBackup. The options wait for running jobs as purge does. + // CreateBackup. The options wait for running jobs as purge does, + // `--move-store` for those of the old root's store ids too. let sqlite = opened.catalog.backend == "sqlite"; - let waits = adopt_store || prune_missing; + let waits = adopt_store || prune_missing || moving.is_some(); let mut claim = None; if !sqlite || waits { let backup = &app_config.backup; let stale_after = Duration::from_secs(backup.stale_job_after_secs); - let holder = format!( - "extenddb backup sync on {} (pid {})", - extenddb_server::backup_store::hostname(), - std::process::id() - ); - let taken = storage - .maintenance_claim(&holder, stale_after) - .await - .map_err(|e| match e { - StorageError::AlreadyExists(m) => anyhow::anyhow!( - "{m}; one maintenance command runs at a time, and a claim whose holder died \ - expires after stale_job_after_secs ({} s)", - backup.stale_job_after_secs - ), - other => anyhow::anyhow!("cannot take the maintenance claim: {other}"), - })?; - println!("Took the maintenance claim as {holder:?}."); - let held = HeldClaim::start(taken, JOB_HEARTBEAT_INTERVAL); - if waits - && let Err(refusal) = wait_for_jobs( - opened, - configured, - known.as_ref(), + let held = take_claim(storage, app_config).await?; + if waits { + let mut identities = match &known { + Some(identity) => vec![identity.clone()], + None => match adopt_wait_identity(configured).await { + Ok(identity) => vec![identity], + Err(refusal) => { + release(held).await; + return Err(refusal); + } + }, + }; + if let Some(located) = &moving { + identities.extend(located.identities()); + } + if let Err(refusal) = wait_for_jobs( + &opened.catalog, + &identities, &held, &mut interrupt, stale_after, + false, ) .await - { - release(held).await; - return Err(refusal); + { + release(held).await; + return Err(refusal); + } } claim = Some(held); } @@ -1283,6 +1510,7 @@ async fn sync( known, adopting, prune_missing, + moving.as_ref(), claim.as_ref(), &interrupt, ) @@ -1293,59 +1521,131 @@ async fn sync( done } -/// The wait `--adopt-store` and `--prune-missing` run under the claim, as -/// purge's: on SQLite after the dead-owner resolutions with a zero window. -/// The refusal is the command's error, the claim still held. +/// Take the maintenance claim for `backup sync` and start its heartbeat. +async fn take_claim( + storage: &dyn StorageEngine, + app_config: &config::AppConfig, +) -> anyhow::Result { + let backup = &app_config.backup; + let stale_after = Duration::from_secs(backup.stale_job_after_secs); + let holder = format!( + "extenddb backup sync on {} (pid {})", + extenddb_server::backup_store::hostname(), + std::process::id() + ); + let taken = storage + .maintenance_claim(&holder, stale_after) + .await + .map_err(|e| match e { + StorageError::AlreadyExists(m) => anyhow::anyhow!( + "{m}; one maintenance command runs at a time, and a claim whose holder died \ + expires after stale_job_after_secs ({} s)", + backup.stale_job_after_secs + ), + other => anyhow::anyhow!("cannot take the maintenance claim: {other}"), + })?; + println!("Took the maintenance claim as {holder:?}."); + Ok(HeldClaim::start(taken, JOB_HEARTBEAT_INTERVAL)) +} + +/// The command's error for a root `--move-store` or `--forget-store` +/// (`option`) refused, the known stores printed first when the root has no +/// rows. +fn root_refusal( + option: &str, + root: &str, + refusal: extenddb_engine::store_maintenance::RootRefusal, +) -> anyhow::Error { + use extenddb_engine::store_maintenance::RootRefusal; + match refusal { + RootRefusal::ConfiguredRoot if option == "--move-store" => anyhow::anyhow!( + "{option} {root}: that is the configured store's root, so nothing would move. Give \ + the root the rows were written under, as backup list printed it before the \ + configuration changed or as backup sync's report of other stores prints it" + ), + RootRefusal::ConfiguredRoot => anyhow::anyhow!( + "{option} {root}: that is the configured store's root, which {option} refuses so \ + the store in use cannot lose its rows by a slip; extenddb backup purge deletes the \ + configured store's backups, and backup sync --prune-missing removes its AVAILABLE \ + rows whose manifest is gone" + ), + RootRefusal::NoRows(known) => { + println!(); + println!("--- The stores the catalog holds backup rows for:"); + for line in known_store_lines(&known) { + println!("{line}"); + } + anyhow::anyhow!( + "{option} {root}: the catalog has no backup row under that root (a root matches \ + as the catalog stores it, byte for byte: a trailing / or an S3 root's \ + ?endpoint= suffix makes another root); the roots it holds rows for are listed \ + above" + ) + } + RootRefusal::Catalog(e) => anyhow::anyhow!("the catalog could not be read: {e}"), + } +} + +/// The identity `--adopt-store`'s wait reads rows of: the root's file id, +/// before the adoption as after it. +async fn adopt_wait_identity(configured: &ConfiguredStore) -> anyhow::Result { + let file = tokio::time::timeout( + configured.check_timeout(), + extenddb_engine::store_identity::read_identity_file(&*configured.store), + ) + .await + .map_err(|_| { + anyhow::anyhow!( + "the store's {} could not be read within {} s", + extenddb_engine::STORE_IDENTITY_KEY, + configured.check_timeout().as_secs() + ) + })? + .map_err(|e| anyhow::anyhow!("{e}"))? + .ok_or_else(|| { + anyhow::anyhow!( + "the root's {} is gone; run the command again", + extenddb_engine::STORE_IDENTITY_KEY + ) + })?; + Ok(StoreIdentity { + store_kind: configured.kind, + store_root: configured.root.clone(), + store_id: file.store_id, + }) +} + +/// The wait the options run under the claim, as purge's, over the rows of +/// `identities` and every RUNNING restore: on SQLite after the dead-owner +/// resolutions with a zero window, whose cycle runs the store's check +/// unless `writes_nothing` (the forget options, which record their own +/// check that writes nothing). Returns the DELETING rows whose deletion +/// that cycle finished. The refusal is the command's error, the claim +/// still held. async fn wait_for_jobs( - opened: &Opened, - configured: &ConfiguredStore, - known: Option<&StoreIdentity>, + catalog: &Catalog, + identities: &[StoreIdentity], claim: &HeldClaim, interrupt: &mut Interrupt, stale_after: Duration, -) -> anyhow::Result<()> { - let storage = &*opened.catalog.storage; - // The identity the wait reads rows of: the file's id, before the - // adoption as after it. - let wait_identity = match known { - Some(identity) => identity.clone(), - None => { - let file = tokio::time::timeout( - configured.check_timeout(), - extenddb_engine::store_identity::read_identity_file(&*configured.store), + writes_nothing: bool, +) -> anyhow::Result> { + let storage = &*catalog.storage; + let mut finished = Vec::new(); + if catalog.backend == "sqlite" { + let state = if writes_nothing { + extenddb_engine::reconciler::ReconcilerState::for_maintenance_holder_without_store_check( ) - .await - .map_err(|_| { - anyhow::anyhow!( - "the store's {} could not be read within {} s", - extenddb_engine::STORE_IDENTITY_KEY, - configured.check_timeout().as_secs() - ) - })? - .map_err(|e| anyhow::anyhow!("{e}"))? - .ok_or_else(|| { - anyhow::anyhow!( - "the root's {} is gone; run the command again", - extenddb_engine::STORE_IDENTITY_KEY - ) - })?; - StoreIdentity { - store_kind: configured.kind, - store_root: configured.root.clone(), - store_id: file.store_id, - } - } - }; - let owner_window = if opened.catalog.backend == "sqlite" { - resolve_dead_owners(opened).await; - Duration::ZERO - } else { - stale_after - }; + } else { + extenddb_engine::reconciler::ReconcilerState::for_maintenance_holder() + }; + finished = resolve_dead_owners_with(catalog, state).await; + } + let owner_window = owner_window(&catalog.backend, stale_after); let started = Instant::now(); let outcome = wait_for_quiet( storage, - &wait_identity, + identities, claim, Wait { owner_window, @@ -1362,7 +1662,7 @@ async fn wait_for_jobs( ) .await; match outcome { - WaitOutcome::Quiet => Ok(()), + WaitOutcome::Quiet => Ok(finished), WaitOutcome::Busy(remaining) => Err(anyhow::anyhow!( "jobs were still running after waiting {} s (2 * stale_job_after_secs): {}. The \ maintenance claim is released; retry once they finish", @@ -1383,16 +1683,17 @@ async fn wait_for_jobs( } } -/// The adoption, the sync and the prune, with the claim held (on +/// The adoption, the move, the sync and the prune, with the claim held (on /// PostgreSQL, MongoDB and Cassandra) or under the serve lock (SQLite); the /// caller releases the claim. -#[allow(clippy::too_many_lines)] +#[allow(clippy::too_many_lines, clippy::too_many_arguments)] async fn sync_held( opened: &Opened, app_config: &config::AppConfig, known: Option, adopting: bool, prune_missing: bool, + moving: Option<&extenddb_engine::store_maintenance::LocatedRoot>, claim: Option<&HeldClaim>, interrupt: &Interrupt, ) -> anyhow::Result<()> { @@ -1462,6 +1763,52 @@ async fn sync_held( .map(|signal| format!("{signal} received")) }) }; + if let Some(located) = moving { + let owner_window = owner_window( + &opened.catalog.backend, + Duration::from_secs(app_config.backup.stale_job_after_secs), + ); + let report = extenddb_engine::store_maintenance::move_store( + storage, + extenddb_engine::store_maintenance::MoveTarget { + configured, + identity: &identity, + from_kind: located.kind, + from_root: &located.root, + owner_window, + stop: &stop, + claim, + }, + ) + .await + .map_err(|e| { + anyhow::anyhow!( + "--move-store: the catalog's rows could not be read: {e}; no row changed" + ) + })?; + for line in move_report_lines(&report, located) { + println!("{line}"); + } + move_outcome(&report)?; + println!(); + if report.gone.is_empty() { + println!( + "Every row of {} names the configured store now; syncing it.", + located.root + ); + } else { + // The rows gone during the move name no store, or another + // one: what the report established is that the old root has + // no row left. + println!( + "No row is left under {} ({} moved, {} removed, or moved to another store, by \ + another writer during the move); syncing the configured store.", + located.root, + report.moved.len(), + report.gone.len() + ); + } + } let report = extenddb_engine::sync::sync( storage, extenddb_engine::sync::SyncTarget { @@ -1474,14 +1821,17 @@ async fn sync_held( }, ) .await; + // After a completed move, the run to repeat is plain sync: the old + // root's rows are moved, and `--move-store` would refuse it as having no + // rows. + let again = run_again(moving.is_some()); let report = match report { Ok(report) => report, Err(extenddb_engine::sync::SyncError::Listing { error, partial }) => { print_sync_report(&partial, prune_missing, false); anyhow::bail!( "the store's listing failed: {error}. {} backup(s) listed before the failure \ - were adopted, and no AVAILABLE row was examined for a missing manifest; run the \ - command again", + were adopted, and no AVAILABLE row was examined for a missing manifest; {again}", partial.adopted.len() ); } @@ -1491,10 +1841,50 @@ async fn sync_held( if let Some(reason) = &report.stopped { anyhow::bail!( "stopped: {reason}. {} backup(s) were adopted before the stop, and no AVAILABLE row \ - was examined for a missing manifest; run the command again for the rest", + was examined for a missing manifest; {again} for the rest", report.adopted.len() ); } + // The rows of the other stores, read once the listing has ended. + let others = extenddb_engine::store_maintenance::other_stores(storage, &identity).await; + println!(); + let others_error = match &others { + Ok(others) => { + for line in other_stores_lines(others, &identity.store_root) { + println!("{line}"); + } + None + } + Err(e) => { + println!("--- Rows of other stores: not read ({e})"); + Some(e.to_string()) + } + }; + // After a move, a row of the old root in that report landed during the + // move's sync (a CreateBackup of a server still configured with the old + // root whose insert came after the move's re-list): named, and the + // command exits 1. + let landed = match (moving, &others) { + (Some(located), Ok(others)) if old_root_listed(others, located) => { + let rows = named_rows_of_root(storage, located).await; + println!(); + println!("--- Rows that landed under the old root during the move:"); + match &rows { + Ok(rows) if rows.is_empty() => println!(" (none listed now)"), + Ok(rows) => { + for row in rows { + println!( + " {} ({}, account {})", + row.backup_arn, row.status, row.account_id + ); + } + } + Err(e) => println!(" (not read: {e})"), + } + Some(rows.map_or(0, |rows| rows.len())) + } + _ => None, + }; let mut counts = PruneCounts::default(); let mut stopped = None; @@ -1548,6 +1938,17 @@ async fn sync_held( anyhow::bail!("--prune-missing stopped before the remaining rows: {reason}"); } counts.left_for_a_later_run()?; + others_outcome(others_error.as_deref(), again)?; + if let (Some(n), Some(located)) = (landed, moving) { + anyhow::bail!( + "a row landed under the old root during the move ({n} listed above): a CreateBackup \ + of a server still configured with {}. Reconfigure every server to the new root, \ + then, once the row's job has ended, copy its prefix under the new root (or remove \ + the row with --forget-backup) and run --move-store {} again", + located.root, + located.root + ); + } complete_run(&report) } @@ -1615,6 +2016,131 @@ impl PruneCounts { } } +/// Whether the report of other stores lists a row of `located`'s root (an +/// entry count with no row behind it is not one). +fn old_root_listed( + others: &[extenddb_engine::store_maintenance::OtherStore], + located: &extenddb_engine::store_maintenance::LocatedRoot, +) -> bool { + others.iter().any(|other| { + other.store.store_kind == located.kind + && other.store.store_root == located.root + && !other.counts.is_empty() + }) +} + +/// The rows of the old root, named for the operator. +async fn named_rows_of_root( + storage: &dyn StorageEngine, + located: &extenddb_engine::store_maintenance::LocatedRoot, +) -> Result, StorageError> { + storage + .list_backup_rows_by_root(located.kind, &located.root, None) + .map_ok(|row| extenddb_engine::store_maintenance::NamedRow { + status: row.status, + backup_arn: row.backup_arn, + account_id: row.account_id, + }) + .try_collect() + .await +} + +/// The exit of a sync whose report of other stores failed (`error`, its +/// read's error): 1, since the report is part of the run; `again` is +/// [`run_again`]'s advice. +fn others_outcome(error: Option<&str>, again: &str) -> anyhow::Result<()> { + match error { + Some(e) => { + anyhow::bail!("the rows of other stores could not be read: {e}; {again}") + } + None => Ok(()), + } +} + +/// The advice a sync that did not finish gives: run the command again, or, +/// after a completed `--move-store` (`moved`), plain `backup sync`, since +/// the old root's rows are moved and the option would refuse the root as +/// having no rows. +fn run_again(moved: bool) -> &'static str { + if moved { + "run extenddb backup sync (plain, without --move-store: the rows are moved, and the \ + option would refuse the old root as having no rows)" + } else { + "run the command again" + } +} + +/// The tail of plain sync's refusal on an unhealthy store: for a missing +/// manifest, the `--forget-backup` command for that backup. +fn manifest_missing_hint(problem: &StoreProblem) -> String { + match problem { + StoreProblem::ManifestMissing { backup_arn } => format!( + "; check first that the root is the one that holds the backups: a root not shared \ + by every server (another host's directory at the same path) shows this too. When \ + that backup is gone for good, remove its row with extenddb backup sync \ + --forget-backup {backup_arn}, which runs on an unhealthy store and prints the \ + store's check afterwards" + ), + _ => String::new(), + } +} + +/// The window the options tell a live owner apart by: `stale_after` beside +/// servers; zero on SQLite, where the command holds the serve lock and no +/// job of another process can be alive. +fn owner_window(backend: &str, stale_after: Duration) -> Duration { + if backend == "sqlite" { + Duration::ZERO + } else { + stale_after + } +} + +/// The configured root's stores, one per store id the catalog records for +/// it (`ids`, the answer of `read_store_ids`): what the forget options wait +/// on when the configured store's identity is not known. A failed read +/// refuses the command before the claim, as the other catalog reads there +/// do, rather than leaving the wait with only the RUNNING restores to read. +fn configured_root_identities( + kind: StoreKind, + root: &str, + ids: Result, StorageError>, +) -> anyhow::Result> { + let ids = ids.map_err(|e| { + anyhow::anyhow!( + "the catalog's store ids for the configured root {root} could not be read: {e}; \ + nothing was changed" + ) + })?; + Ok(ids + .into_iter() + .map(|store_id| StoreIdentity { + store_kind: kind, + store_root: root.to_owned(), + store_id, + }) + .collect()) +} + +/// The stores the forget options wait on: the configured store when its +/// identity is known, otherwise every store id the catalog records for the +/// configured root (`configured_ids`), and every store id it records for +/// `--forget-store`'s root. +fn forget_wait_identities( + known: Option<&StoreIdentity>, + configured_ids: &[StoreIdentity], + located: Option<&extenddb_engine::store_maintenance::LocatedRoot>, +) -> Vec { + let mut identities: Vec = match known { + Some(known) => vec![known.clone()], + None => configured_ids.to_vec(), + }; + if let Some(located) = located { + identities.extend(located.identities()); + } + identities +} + /// The sync's last read of `extenddb-store.json` (`file`, `None` when the /// read did not finish within `bound`): `Ok` when it still names this /// catalog (`catalog_id`), and otherwise the error naming what it found, so @@ -1718,46 +2244,837 @@ fn print_sync_report( } } -#[cfg(test)] -mod tests { - use std::sync::Arc; - use std::sync::atomic::{AtomicUsize, Ordering}; - use std::time::Duration; +/// The counts of one store as a line's tail: `3 AVAILABLE, 1 DELETING`. +fn counts_text(counts: &[(BackupStatus, u64)]) -> String { + counts + .iter() + .map(|(status, n)| format!("{n} {status}")) + .collect::>() + .join(", ") +} - use async_trait::async_trait; - use extenddb_storage::hooks::{Reregistration, ServingRegistrar}; +/// The other-stores section of a sync's report: per store its counts per +/// status (on Cassandra, the status entries no row stands behind as such) +/// and its CREATING and DELETING rows, then the two options that act on +/// them. A store of `configured_root` under another store id is one the +/// two options refuse (the root is the configured one), so its line names +/// `--forget-backup` instead. +fn other_stores_lines( + others: &[extenddb_engine::store_maintenance::OtherStore], + configured_root: &str, +) -> Vec { + let mut lines = vec![ + "--- Rows of other stores (left as they are; sync reads and changes none of them):" + .to_owned(), + ]; + if others.is_empty() { + lines.push(" (none)".to_owned()); + return lines; + } + let mut other_roots = false; + for other in others { + let mut tail = counts_text(&other.counts); + if !other.entries.is_empty() { + if !tail.is_empty() { + tail.push_str("; "); + } + tail.push_str(&format!( + "{} (status entries with no row of that status behind them, which readers skip; \ + with no row of the root left, --forget-store removes them)", + counts_text(&other.entries) + )); + } + if other.store.store_root == configured_root { + tail.push_str( + " (the configured root under another store id: --move-store and --forget-store \ + refuse the configured root; remove such a row with extenddb backup sync \ + --forget-backup )", + ); + } else { + other_roots = true; + } + lines.push(format!( + " {} {} (store id {}): {tail}", + other.store.store_kind, other.store.store_root, other.store.store_id + )); + for row in &other.unresolved { + lines.push(format!( + " {} {} (account {})", + row.status, row.backup_arn, row.account_id + )); + } + } + if other_roots { + lines.push( + " (to keep a root's backups, copy its tree under the configured root and run \ + extenddb backup sync --move-store ; to drop its rows, extenddb backup sync \ + --forget-store )" + .to_owned(), + ); + } + lines +} - use super::{ - PruneCounts, complete_run, owner_after_sync, release_registration, - release_registration_within, run_word, - }; +/// The stores the catalog holds rows for, one line each, for a root that +/// has none. +fn known_store_lines(counts: &[extenddb_storage::backup_catalog::StoreRootCount]) -> Vec { + let stores = extenddb_engine::store_maintenance::group_counts(counts, None); + if stores.is_empty() { + return vec![" (none)".to_owned()]; + } + stores + .iter() + .map(|other| { + format!( + " {} {} (store id {}): {}", + other.store.store_kind, + other.store.store_root, + other.store.store_id, + counts_text(&other.counts) + ) + }) + .collect() +} - /// The identity file as a store holds it, naming `catalog_id`. - fn file(catalog_id: &str) -> extenddb_engine::StoreIdentityFile { - extenddb_engine::StoreIdentityFile { - format_version: 1, - store_id: "store-1".to_owned(), - catalog_id: catalog_id.to_owned(), - backend: "postgres".to_owned(), - created_at_ms: 0, - ephemeral_catalog: false, +/// The report of `--move-store`. +fn move_report_lines( + report: &extenddb_engine::store_maintenance::MoveReport, + located: &extenddb_engine::store_maintenance::LocatedRoot, +) -> Vec { + let mut lines = vec![ + String::new(), + format!( + "--- --move-store {} ({}): {} row(s) under the old root, checked against the copy \ + under the configured root (the copy must be complete before the move)", + located.root, located.kind, report.rows + ), + ]; + let mut section = |title: &str, items: Vec| { + lines.push(format!("--- {title}:")); + if items.is_empty() { + lines.push(" (none)".to_owned()); } + lines.extend(items.into_iter().map(|item| format!(" {item}"))); + }; + section( + "Resolved (CREATING rows whose owner is dead, against the configured store)", + report + .resolved + .iter() + .map(|(arn, status)| format!("{arn}: now {status}")) + .collect(), + ); + section( + "Finished (DELETING rows, their objects deleted under the configured root)", + report + .finished + .iter() + .map(|arn| format!("{arn}: now DELETED")) + .collect(), + ); + section( + "Stopping the move (no row is moved while any is listed here)", + report + .stopping + .iter() + .map(|(row, why)| { + format!( + "{} ({}, account {}): {why}", + row.backup_arn, row.status, row.account_id + ) + }) + .collect(), + ); + section( + "Moved (each row now names the configured store)", + report.moved.clone(), + ); + if !report.gone.is_empty() { + section( + "Gone during the move (removed, or moved to another store, by another writer; \ + nothing moved)", + report.gone.clone(), + ); } + section( + "Left under the old root", + report + .left + .iter() + .map(|row| { + format!( + "{} ({}, account {})", + row.backup_arn, row.status, row.account_id + ) + }) + .collect(), + ); + lines.push(String::new()); + lines.push(format!( + "=== extenddb backup sync --move-store {}: {} row(s) moved, {} resolved, {} \ + deletion(s) finished, {} row(s) stopping the move, {} gone during the move, {} left \ + under the old root ===", + if report.complete() { + "complete" + } else { + "stopped" + }, + report.moved.len(), + report.resolved.len(), + report.finished.len(), + report.stopping.len(), + report.gone.len(), + report.left.len() + )); + lines +} - /// The adopting sync's last read: the file still naming this catalog - /// passes; another catalog (an adopter whose put landed during the - /// sync), no file, an unreadable file and a read past the bound each - /// fail, the first naming the new owner. - #[test] - fn the_adopting_syncs_last_read_fails_unless_the_file_names_this_catalog() { - let bound = Duration::from_secs(10); - owner_after_sync(Some(Ok(Some(file("catalog-1")))), "catalog-1", bound).unwrap(); - let err = owner_after_sync(Some(Ok(Some(file("adopter")))), "catalog-1", bound) - .unwrap_err() - .to_string(); - assert!( - err.starts_with( - "after the sync, extenddb-store.json names catalog adopter (store id \ +/// The exit of `--move-store`: `Ok` when every row of the old root was +/// moved (the sync of the configured root follows), otherwise why not. +fn move_outcome(report: &extenddb_engine::store_maintenance::MoveReport) -> anyhow::Result<()> { + if !report.stopping.is_empty() { + anyhow::bail!( + "--move-store moved no row: {} row(s) stop the move (see above). Resolve each and \ + run the command again; the CREATING rows resolved and the deletions finished are \ + kept", + report.stopping.len() + ); + } + if let Some(reason) = &report.stopped { + if report.rewritten == 0 { + anyhow::bail!( + "--move-store stopped before the rewrite: {reason}; no row was moved{}", + relist_failure(report) + ); + } + anyhow::bail!( + "--move-store stopped during the rewrite: {reason}{}", + after_a_partial_rewrite(report) + ); + } + if let Some(e) = &report.rewrite_error { + anyhow::bail!( + "--move-store's rewrite failed: {e}{}", + after_a_partial_rewrite(report) + ); + } + if let Some(e) = &report.relist_error { + anyhow::bail!( + "{e}; the rewrite counted {} row(s). Run extenddb backup sync, which lists any row \ + still under the old root in its report of other stores and syncs the configured \ + root; run --move-store again only if that report shows rows of the old root (with \ + none, it refuses the root as having no rows)", + report.rewritten + ); + } + if !report.left.is_empty() { + anyhow::bail!( + "{} row(s) are still under the old root (see above): a CreateBackup of a server \ + still configured with that root, or a row another writer changed during the move. \ + Reconfigure every server to the new root, copy each such row's prefix under the new \ + root (or remove the row with --forget-backup), and run the command again", + report.left.len() + ); + } + Ok(()) +} + +/// The listing's error after a stop or a failed rewrite, when the old +/// root's rows could not be listed again, with the sentence's separator. +fn relist_failure(report: &extenddb_engine::store_maintenance::MoveReport) -> String { + report + .relist_error + .as_ref() + .map_or_else(String::new, |e| format!("; {e}")) +} + +/// What a stop or a failed rewrite leaves: the rows rewritten before it +/// (the count is what the rewrite returned, `moved` is empty when the +/// listing after it failed), and where the rows left under the old root +/// are. +fn after_a_partial_rewrite(report: &extenddb_engine::store_maintenance::MoveReport) -> String { + format!( + "; {} row(s) were rewritten{}, and the rows left under the old root {}: run the command \ + again for them", + report.rewritten, + relist_failure(report), + if report.relist_error.is_some() { + "could not be listed" + } else { + "are listed above" + } + ) +} + +/// The report of `--forget-store`. +fn forget_store_lines( + report: &extenddb_engine::store_maintenance::ForgetStoreReport, + located: &extenddb_engine::store_maintenance::LocatedRoot, +) -> Vec { + let mut lines = vec![ + String::new(), + format!( + "--- --forget-store {} ({}): rows removed outright; no object is deleted:", + located.root, located.kind + ), + ]; + if report.rows.is_empty() { + lines.push(" (none)".to_owned()); + } + for (row, outcome) in &report.rows { + lines.push(format!( + " {} ({}, account {}): {outcome}", + row.backup_arn, row.status, row.account_id + )); + } + let swept = report.entries_swept.unwrap_or_default(); + if swept.removed > 0 { + lines.push(format!( + " ({} status entry(ies) of the root with no row of the root behind them removed: \ + entries a deleter wrote after a move of its row, and the earlier-status entries of \ + rows removed above)", + swept.removed + )); + } + if swept.repaired > 0 { + lines.push(format!( + " ({} status entry(ies) of rows left under the root repaired: each was under an \ + earlier status than its row's, from a status change whose entry batch failed or \ + whose writer stopped before it, and is now under the row's status)", + swept.repaired + )); + } + if !report.entries_left.is_empty() { + lines.push(format!( + " (the catalog still records store id(s) {} for the root: status entries written \ + after the root was listed, by a writer still configured with it; plain sync's \ + report of other stores shows them, and running the command again removes their \ + rows)", + report.entries_left.join(", ") + )); + } + // The rows the listing after the removals holds that the first listing + // did not: each landed during the run, from a writer still configured + // with the root, and is counted as left. + let landed: Vec<&extenddb_engine::store_maintenance::NamedRow> = report + .rows_left + .as_deref() + .unwrap_or_default() + .iter() + .filter(|left| { + !report + .rows + .iter() + .any(|(row, _)| row.backup_arn == left.backup_arn) + }) + .collect(); + if !landed.is_empty() { + lines.push( + "--- Rows that landed under the root during the run (left; a writer is still \ + configured with the root):" + .to_owned(), + ); + for row in landed { + lines.push(format!( + " {} ({}, account {})", + row.backup_arn, row.status, row.account_id + )); + } + } + lines.push(String::new()); + lines.push(format!( + "=== extenddb backup sync --forget-store {}: {} row(s) removed, {} left ===", + if report.complete() { + "complete" + } else if report.stopped.is_some() { + "stopped" + } else { + "incomplete" + }, + report.removed(), + report.rows_left.as_ref().map_or_else( + || "an unknown number".to_owned(), + |left| left.len().to_string() + ) + )); + lines +} + +/// The exit of `--forget-store`. +fn forget_store_outcome( + report: &extenddb_engine::store_maintenance::ForgetStoreReport, +) -> anyhow::Result<()> { + if let Some(reason) = &report.stopped { + anyhow::bail!( + "--forget-store stopped before the remaining rows: {reason}; run the command again \ + for the rest" + ); + } + if let Some(e) = &report.recheck_error { + anyhow::bail!("{e}; run the command again to see what is left"); + } + if !report.complete() { + anyhow::bail!( + "{} row(s) of the root are left (see above, each with why); run the command again \ + once their jobs have ended, and for a row in use once its restore has ended or its \ + target is removed with extenddb backup remove-target --account ", + report.rows_left.as_ref().map_or(0, Vec::len) + ); + } + Ok(()) +} + +/// The report of `--forget-backup`. +fn forget_backups_lines( + report: &extenddb_engine::store_maintenance::ForgetBackupsReport, + asked: &[String], +) -> Vec { + let mut lines = vec![ + String::new(), + "--- --forget-backup (rows removed outright; no object is deleted, but a DELETING row \ + of the configured store has its deletion finished first):" + .to_owned(), + ]; + for (arn, outcome) in &report.done { + lines.push(format!(" {arn}: {outcome}")); + } + for arn in asked.iter().skip(report.done.len()) { + lines.push(format!(" {arn}: not reached")); + } + let removed = report + .done + .iter() + .filter(|(_, o)| { + matches!( + o, + extenddb_engine::store_maintenance::ForgetBackup::Row(_, forgotten) + if forgotten.is_removed() + ) + }) + .count(); + lines.push(String::new()); + lines.push(format!( + "=== extenddb backup sync --forget-backup {}: {removed} row(s) removed, {} not ===", + if report.complete(asked.len()) { + "complete" + } else { + "stopped" + }, + asked.len() - report.done.iter().filter(|(_, o)| o.is_done()).count() + )); + lines +} + +/// The exit of `--forget-backup`. +fn forget_backups_outcome( + report: &extenddb_engine::store_maintenance::ForgetBackupsReport, + asked: usize, +) -> anyhow::Result<()> { + if let Some(reason) = &report.stopped { + anyhow::bail!( + "--forget-backup stopped before the remaining ARNs: {reason}; run the command again \ + for them" + ); + } + if !report.complete(asked) { + let (arn, outcome) = report + .done + .last() + .expect("an incomplete run without a stop ends at a refused ARN"); + anyhow::bail!( + "--forget-backup did not remove the row of {arn} ({outcome}); the ARNs after it were \ + not reached" + ); + } + Ok(()) +} + +/// `backup sync --forget-store ` and `--forget-backup ` (module +/// doc): on a store in any state, or with none configured (`identity` is +/// then why), under the maintenance claim on every backend. +#[allow(clippy::too_many_lines)] +async fn forget( + catalog: &Catalog, + identity: Result<&Identity, &NotConfigured>, + app_config: &config::AppConfig, + forget_store: Option<&str>, + forget_backup: &[String], + mut interrupt: Interrupt, +) -> anyhow::Result<()> { + let storage = &*catalog.storage; + let configured = catalog.runtime.configured_store(); + // The defaulted root missing (moved away, or a link to a volume that is + // gone) while the catalog holds rows for it: its rows are refused as an + // unmounted root's, and the header says so. + let unmounted = match identity { + Err(NotConfigured::DefaultRootMissingWithRows { + label, store_ids, .. + }) => Some((label.clone(), store_ids.clone())), + _ => None, + }; + println!("=== extenddb backup sync ==="); + // The check after the removals runs for `--forget-backup` alone, and + // not when the root had no identity file as the command opened it + // (`forget_check`), so the note says so only when it will run. + let check_note = + if forget_store.is_none() && !matches!(forget_check(identity), ForgetCheck::Skip) { + "the removals read no store; the check after them does" + } else { + "the removals read no store" + }; + let known = match (configured, identity) { + (Some(store), Ok(Identity::Known(identity))) => { + match store.health() { + StoreHealth::Unhealthy(problem) => println!( + "Store: {} {} (unhealthy: {problem}; {check_note})", + store.kind, store.root + ), + // The opening's read-only check gave no answer (the file + // read, then gone before the check): not healthy, so a + // DELETING row of this store is refused. + StoreHealth::Checking => println!( + "Store: {} {} (store id {}; check pending: the opening's check did not \ + pass, so a DELETING row of this store is left; {check_note})", + store.kind, store.root, identity.store_id + ), + StoreHealth::Healthy => println!( + "Store: {} {} (store id {})", + store.kind, store.root, identity.store_id + ), + } + Some(identity.clone()) + } + (Some(store), Ok(Identity::Problem(problem))) => { + println!( + "Store: {} {} (unhealthy: {problem}; {check_note})", + store.kind, store.root + ); + None + } + (Some(store), Ok(Identity::Empty)) => { + println!( + "Store: {} {} (no {})", + store.kind, + store.root, + extenddb_engine::STORE_IDENTITY_KEY + ); + None + } + (_, Err(_)) | (None, Ok(_)) => { + match &unmounted { + Some((root, ids)) => println!( + "Store: none (the default root {root} is missing while the catalog holds \ + rows for it, store id {}: a volume that is not mounted, or a directory moved \ + away; this command refuses its rows and reads no store)", + ids.join(", ") + ), + None => println!( + "Store: none ({}; the removals read no store)", + identity + .err() + .map_or_else(|| "not configured".to_owned(), ToString::to_string) + ), + } + None + } + }; + // Every ARN's form before the claim: a typo costs no wait. + let refused = extenddb_engine::store_maintenance::check_arns(forget_backup); + if !refused.is_empty() { + println!(); + println!("--- --forget-backup: ARNs refused before any change:"); + for (arn, why) in &refused { + println!(" {arn:?}: {why}"); + } + anyhow::bail!( + "--forget-backup: {} ARN(s) given are not backup ARNs (see above); nothing was \ + changed", + refused.len() + ); + } + let located = match forget_store { + Some(root) => match extenddb_engine::store_maintenance::locate_root( + storage, + root, + configured + .map(|store| store.root.as_str()) + .or_else(|| unmounted.as_ref().map(|(root, _)| root.as_str())), + ) + .await + { + Ok(located) => Some(located), + Err(refusal) => return Err(root_refusal("--forget-store", root, refusal)), + }, + None => None, + }; + + // The configured root's store ids when its identity is not known, so + // the wait covers its rows too. + let configured_ids = match (configured, &known) { + (Some(store), None) => configured_root_identities( + store.kind, + &store.root, + storage.read_store_ids(store.kind, &store.root).await, + )?, + _ => Vec::new(), + }; + let stale_after = Duration::from_secs(app_config.backup.stale_job_after_secs); + let claim = take_claim(storage, app_config).await?; + let claimed_at = tokio::time::Instant::now(); + let identities = forget_wait_identities(known.as_ref(), &configured_ids, located.as_ref()); + let deletions_finished = match wait_for_jobs( + catalog, + &identities, + &claim, + &mut interrupt, + stale_after, + true, + ) + .await + { + Ok(finished) => finished, + Err(refusal) => { + release(claim).await; + return Err(refusal); + } + }; + let stop = || { + claim.lost().or_else(|| { + interrupt + .signalled() + .map(|signal| format!("{signal} received")) + }) + }; + let on_wait = |remaining: Duration| { + println!( + "Waiting {} s before the first removal of a FAILED or DELETED row (or the \ + finished deletion of a DELETING one): a deletion a server's reconciler started before the claim was taken runs at most {} s, and its \ + last store call at most {} s more.", + remaining.as_secs().max(1), + extenddb_engine::reconciler::DUTY_DELETION_BOUND.as_secs(), + extenddb_engine::S3_CHECK_TIMEOUT.as_secs() + ); + }; + // The configured store's identity when its check passed (recorded at + // the opening, writing nothing): a DELETING row of it has its deletion + // finished before its removal. + let healthy = configured.and_then(|store| store.identity().filter(|_| store.is_healthy())); + let run = extenddb_engine::store_maintenance::ForgetRun { + owner_window: owner_window(&catalog.backend, stale_after), + removals_from: removals_from(&catalog.backend, claimed_at), + on_wait: &on_wait, + stop: &stop, + claim: Some(&claim), + configured: configured.map( + |store| extenddb_engine::store_maintenance::ForgetConfigured { + store, + healthy: healthy.as_ref(), + mismatch_row_ids: match identity { + Ok(Identity::Problem(problem)) => problem.mismatch_row_ids(), + _ => &[], + }, + file_missing_with_rows: matches!( + identity, + Ok(Identity::Problem(StoreProblem::FileMissingWithRows(_))) + ), + }, + ), + unmounted_root: unmounted.as_ref().map(|(root, _)| root.as_str()), + deletions_finished: &deletions_finished, + }; + let outcome = match &located { + Some(located) => { + let report = extenddb_engine::store_maintenance::forget_store( + storage, + located.kind, + &located.root, + run, + ) + .await; + match report { + Ok(report) => { + for line in forget_store_lines(&report, located) { + println!("{line}"); + } + forget_store_outcome(&report) + } + Err(e) => Err(anyhow::anyhow!( + "--forget-store: the root's rows could not be read: {e}; no row was removed" + )), + } + } + None => { + let report = extenddb_engine::store_maintenance::forget_backups( + storage, + forget_backup, + &app_config.server.region, + run, + ) + .await; + for line in forget_backups_lines(&report, forget_backup) { + println!("{line}"); + } + forget_backups_outcome(&report, forget_backup.len()) + } + }; + release(claim).await; + + // The store's check once more, as the next server check runs it but + // writing nothing: a removed row may have been what kept the store + // unhealthy. + if let Some(store) = configured + && forget_store.is_none() + { + println!(); + match forget_check(identity) { + ForgetCheck::Skip => { + let fresh = matches!(identity, Ok(Identity::Empty)); + for line in skipped_check_lines(&store.root, fresh) { + println!("{line}"); + } + } + ForgetCheck::Run(known) => { + let check = extenddb_engine::check_configured_store_read_only( + store, + storage, + known.as_ref(), + ) + .await; + for line in store_check_lines(&check) { + println!("{line}"); + } + } + } + } + outcome +} + +/// The instant before which a forget option removes no FAILED or DELETED +/// row: `FORGET_WAIT` after the claim was taken, beside servers; none on +/// SQLite, whose serve lock excludes every server. +fn removals_from(backend: &str, claimed_at: tokio::time::Instant) -> Option { + (backend != "sqlite").then(|| claimed_at + extenddb_engine::store_maintenance::FORGET_WAIT) +} + +/// Whether the store's check runs after `--forget-backup`, and against +/// which identity. +#[derive(Debug, Clone, PartialEq, Eq)] +enum ForgetCheck { + /// The opening found no identity file under the root (with rows for + /// it: an unmounted volume, say, whose rows the command refused; or + /// none: a fresh root): the check is not run, so nothing is written to + /// such a root. + Skip, + /// The read-only check, against the id the opening read when it read + /// one. + Run(Option), +} + +/// The check after `--forget-backup` for the store as the opening found +/// it. +fn forget_check(identity: Result<&Identity, &NotConfigured>) -> ForgetCheck { + match identity { + Ok(Identity::Known(identity)) => ForgetCheck::Run(Some(identity.clone())), + Ok(Identity::Empty | Identity::Problem(StoreProblem::FileMissingWithRows(_))) => { + ForgetCheck::Skip + } + Ok(Identity::Problem(_)) | Err(_) => ForgetCheck::Run(None), + } +} + +/// What `--forget-backup` prints when the opening found no identity file: +/// at a fresh root (`fresh`: no rows for it either), or at a root with rows +/// for it. +fn skipped_check_lines(root: &str, fresh: bool) -> Vec { + let key = extenddb_engine::STORE_IDENTITY_KEY; + let opening = format!( + "The configured store's check was not run: the root {root} had no {key} when the command \ + opened it, and this command writes nothing to a root." + ); + if fresh { + return vec![format!( + "{opening} The root is fresh (the catalog has no rows for it either): a server's \ + first check creates its {key}." + )]; + } + vec![format!( + "{opening} A server's check creates the file at a root the catalog has no rows for and \ + reports the store healthy, so mount the volume (or point [backup.store] at the root \ + that holds the backups) before the root's last row is gone." + )] +} + +/// The store's check after `--forget-backup`. +fn store_check_lines(check: &Result, StoreProblem>) -> Vec { + match check { + Ok(Some(identity)) => vec![format!( + "The configured store's check passes (store id {}); each server finds it so at its \ + next reconciler cycle.", + identity.store_id + )], + Ok(None) => vec![format!( + "The configured store's root has no {} and the catalog no rows for it; the check \ + writes nothing, and a server's next check creates the file.", + extenddb_engine::STORE_IDENTITY_KEY + )], + Err(problem @ StoreProblem::ManifestMissing { backup_arn }) => vec![ + format!("The configured store's check fails: {problem}."), + format!( + " If that backup is gone for good too, run extenddb backup sync \ + --forget-backup {backup_arn}; then backup sync --prune-missing removes the \ + remaining rows whose manifest is gone." + ), + ], + Err(problem) => vec![format!("The configured store's check fails: {problem}.")], + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::time::Duration; + + use async_trait::async_trait; + use extenddb_engine::StoreProblem; + use extenddb_engine::store_maintenance::{ + ForgetBackup, ForgetBackupsReport, ForgetStoreReport, Forgotten, LocatedRoot, MoveReport, + NamedRow, OtherStore, + }; + use extenddb_storage::backup_catalog::{BackupStatus, StoreIdentity, StoreKind}; + use extenddb_storage::hooks::{Reregistration, ServingRegistrar}; + + use super::{ + PruneCounts, complete_run, owner_after_sync, release_registration, + release_registration_within, run_word, + }; + + /// The identity file as a store holds it, naming `catalog_id`. + fn file(catalog_id: &str) -> extenddb_engine::StoreIdentityFile { + extenddb_engine::StoreIdentityFile { + format_version: 1, + store_id: "store-1".to_owned(), + catalog_id: catalog_id.to_owned(), + backend: "postgres".to_owned(), + created_at_ms: 0, + ephemeral_catalog: false, + } + } + + /// The adopting sync's last read: the file still naming this catalog + /// passes; another catalog (an adopter whose put landed during the + /// sync), no file, an unreadable file and a read past the bound each + /// fail, the first naming the new owner. + #[test] + fn the_adopting_syncs_last_read_fails_unless_the_file_names_this_catalog() { + let bound = Duration::from_secs(10); + owner_after_sync(Some(Ok(Some(file("catalog-1")))), "catalog-1", bound).unwrap(); + let err = owner_after_sync(Some(Ok(Some(file("adopter")))), "catalog-1", bound) + .unwrap_err() + .to_string(); + assert!( + err.starts_with( + "after the sync, extenddb-store.json names catalog adopter (store id \ store-1): another catalog adopted the root during the sync" ), "{err}" @@ -1963,4 +3280,714 @@ mod tests { release_registration(None).await; } + + /// `backup sync`'s options as clap parses them: the three store options + /// conflict with each other, with `--adopt-store` and with + /// `--prune-missing`; `--forget-backup` repeats, its ARNs in order. + #[test] + fn the_store_options_conflict_and_forget_backup_repeats() { + use clap::Parser; + #[derive(Parser)] + struct Cli { + #[command(subcommand)] + command: super::BackupCommand, + } + let parse = |args: &[&str]| { + Cli::try_parse_from(std::iter::once("backup").chain(args.iter().copied())) + .map(|cli| cli.command) + }; + for refused in [ + &["sync", "--move-store", "/a", "--prune-missing"][..], + &["sync", "--move-store", "/a", "--adopt-store"], + &["sync", "--move-store", "/a", "--forget-store", "/b"], + &["sync", "--move-store", "/a", "--forget-backup", "x"], + &["sync", "--forget-store", "/b", "--forget-backup", "x"], + &["sync", "--forget-store", "/b", "--prune-missing"], + &["sync", "--forget-store", "/b", "--adopt-store"], + &["sync", "--forget-backup", "x", "--prune-missing"], + &["sync", "--forget-backup", "x", "--adopt-store"], + ] { + let Err(e) = parse(refused) else { + panic!("{refused:?} parsed"); + }; + assert_eq!( + e.kind(), + clap::error::ErrorKind::ArgumentConflict, + "{refused:?}" + ); + } + match parse(&["sync", "--forget-backup", "a", "--forget-backup", "b"]).unwrap() { + super::BackupCommand::Sync { + forget_backup, + move_store: None, + forget_store: None, + adopt_store: false, + prune_missing: false, + } => assert_eq!(forget_backup, ["a", "b"]), + _ => panic!("not a sync"), + } + match parse(&["sync", "--move-store", "s3://b/p?endpoint=http://h:1"]).unwrap() { + super::BackupCommand::Sync { + move_store: Some(root), + .. + } => assert_eq!(root, "s3://b/p?endpoint=http://h:1"), + _ => panic!("not a move"), + } + } + + fn named(status: BackupStatus, arn: &str) -> NamedRow { + NamedRow { + status, + backup_arn: arn.to_owned(), + account_id: "123456789012".to_owned(), + } + } + + fn old_root() -> LocatedRoot { + LocatedRoot { + kind: StoreKind::Filesystem, + root: "/srv/old".to_owned(), + store_ids: vec!["store-0".to_owned()], + } + } + + /// The other-stores section: `(none)` without any; otherwise a line per + /// store with its counts, its CREATING and DELETING rows below it, and + /// the line naming the two options. A store of the configured root + /// under another store id, which both options refuse, names + /// `--forget-backup` on its own line instead, and the options' line is + /// printed only when a root other than the configured one is listed. + #[test] + fn the_other_stores_section_counts_names_and_points_to_the_options() { + assert_eq!( + super::other_stores_lines(&[], "/srv/new"), + [ + "--- Rows of other stores (left as they are; sync reads and changes none of them):", + " (none)", + ] + ); + let old = OtherStore { + store: StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/srv/old".to_owned(), + store_id: "store-0".to_owned(), + }, + counts: vec![(BackupStatus::Available, 3), (BackupStatus::Deleting, 1)], + unresolved: vec![named(BackupStatus::Deleting, "arn:d")], + entries: Vec::new(), + }; + let lines = super::other_stores_lines(std::slice::from_ref(&old), "/srv/new"); + assert_eq!( + lines[1], + " filesystem /srv/old (store id store-0): 3 AVAILABLE, 1 DELETING" + ); + assert_eq!(lines[2], " DELETING arn:d (account 123456789012)"); + assert!( + lines[3].contains("--move-store ") && lines[3].contains("--forget-store "), + "{}", + lines[3] + ); + assert_eq!(lines.len(), 4); + + // The configured root under another id, alone: no options' line. + let same_root = OtherStore { + store: StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/srv/new".to_owned(), + store_id: "store-9".to_owned(), + }, + counts: vec![(BackupStatus::Available, 2)], + unresolved: Vec::new(), + entries: Vec::new(), + }; + let lines = super::other_stores_lines(std::slice::from_ref(&same_root), "/srv/new"); + assert_eq!(lines.len(), 2, "{lines:?}"); + assert!( + lines[1].starts_with( + " filesystem /srv/new (store id store-9): 2 AVAILABLE (the \ + configured root under another store id: --move-store and \ + --forget-store refuse the configured root; remove such a row \ + with extenddb backup sync --forget-backup )" + ), + "{}", + lines[1] + ); + assert!(!lines.iter().any(|l| l.contains("--move-store "))); + + // Beside another root: the options' line names that root's options. + let lines = super::other_stores_lines(&[old, same_root], "/srv/new"); + assert_eq!(lines.len(), 5, "{lines:?}"); + assert!(lines[3].contains("--forget-backup "), "{}", lines[3]); + assert!(lines[4].contains("--move-store "), "{}", lines[4]); + } + + /// The move exits 0 only when every row moved; a stopping row, a stop, + /// a failed rewrite and a row left each exit 1 with their own sentence, + /// and the summary line says which. + #[test] + fn the_move_exits_1_unless_every_row_moved() { + let moved = MoveReport { + rows: 2, + moved: vec!["arn:1".to_owned(), "arn:2".to_owned()], + rewritten: 2, + ..MoveReport::default() + }; + super::move_outcome(&moved).unwrap(); + let summary = super::move_report_lines(&moved, &old_root()); + assert_eq!( + summary.last().unwrap(), + "=== extenddb backup sync --move-store complete: 2 row(s) moved, 0 resolved, 0 \ + deletion(s) finished, 0 row(s) stopping the move, 0 gone during the move, 0 left \ + under the old root ===" + ); + let gone = MoveReport { + rows: 3, + moved: vec!["arn:1".to_owned(), "arn:2".to_owned()], + gone: vec!["arn:3".to_owned()], + rewritten: 3, + ..MoveReport::default() + }; + assert!( + super::move_report_lines(&gone, &old_root()) + .last() + .unwrap() + .contains( + "2 row(s) moved, 0 resolved, 0 deletion(s) finished, 0 row(s) stopping \ + the move, 1 gone during the move, 0 left under the old root" + ), + "the summary line counts the rows gone during the move" + ); + let stopping = MoveReport { + stopping: vec![(named(BackupStatus::Available, "arn:1"), "why".to_owned())], + ..MoveReport::default() + }; + let err = super::move_outcome(&stopping).unwrap_err().to_string(); + assert!( + err.starts_with("--move-store moved no row: 1 row(s) stop the move"), + "{err}" + ); + assert!( + super::move_report_lines(&stopping, &old_root()) + .contains(&" arn:1 (AVAILABLE, account 123456789012): why".to_owned()) + ); + for (report, says) in [ + ( + MoveReport { + stopped: Some("SIGINT received".to_owned()), + ..MoveReport::default() + }, + "stopped before the rewrite: SIGINT received", + ), + ( + MoveReport { + rewrite_error: Some("lost".to_owned()), + ..MoveReport::default() + }, + "rewrite failed: lost", + ), + ( + MoveReport { + left: vec![named(BackupStatus::Creating, "arn:3")], + ..MoveReport::default() + }, + "1 row(s) are still under the old root", + ), + ] { + let err = super::move_outcome(&report).unwrap_err().to_string(); + assert!(err.contains(says), "{err}"); + assert!( + super::move_report_lines(&report, &old_root()) + .last() + .unwrap() + .starts_with("=== extenddb backup sync --move-store stopped:") + ); + } + } + + /// A stop between two slices with the old root's listing then failing: + /// `moved` is empty (the listing is what fills it) while the rewrite + /// counted 500 keys, so the exit says that 500 rows were rewritten, not + /// that no row was moved, and prints the listing's error; a stop + /// before the first slice says no row was moved and prints it too. A + /// failed rewrite with the listing failing says the same of its count. + #[test] + fn a_stop_with_a_failed_listing_counts_the_rewritten_rows_and_prints_the_listings_error() { + let relist = "the old root's rows could not be read after the rewrite: connection reset"; + let stopped = MoveReport { + rows: 1000, + stopped: Some("the maintenance claim was lost".to_owned()), + rewritten: 500, + relist_error: Some(relist.to_owned()), + ..MoveReport::default() + }; + let err = super::move_outcome(&stopped).unwrap_err().to_string(); + assert_eq!( + err, + format!( + "--move-store stopped during the rewrite: the maintenance claim was lost; 500 \ + row(s) were rewritten; {relist}, and the rows left under the old root could not \ + be listed: run the command again for them" + ) + ); + let before = MoveReport { + rows: 1000, + stopped: Some("SIGTERM received".to_owned()), + relist_error: Some(relist.to_owned()), + ..MoveReport::default() + }; + let err = super::move_outcome(&before).unwrap_err().to_string(); + assert_eq!( + err, + format!( + "--move-store stopped before the rewrite: SIGTERM received; no row was moved; {relist}" + ) + ); + let listed = MoveReport { + rows: 1000, + stopped: Some("SIGTERM received".to_owned()), + moved: (0..500).map(|i| format!("arn:{i}")).collect(), + rewritten: 500, + ..MoveReport::default() + }; + let err = super::move_outcome(&listed).unwrap_err().to_string(); + assert_eq!( + err, + "--move-store stopped during the rewrite: SIGTERM received; 500 row(s) were \ + rewritten, and the rows left under the old root are listed above: run the command \ + again for them" + ); + let failed = MoveReport { + rows: 1000, + rewrite_error: Some("lost".to_owned()), + rewritten: 500, + relist_error: Some(relist.to_owned()), + ..MoveReport::default() + }; + let err = super::move_outcome(&failed).unwrap_err().to_string(); + assert_eq!( + err, + format!( + "--move-store's rewrite failed: lost; 500 row(s) were rewritten; {relist}, and \ + the rows left under the old root could not be listed: run the command again for \ + them" + ) + ); + } + + /// `--forget-store` exits 0 once the root lists no row, entries left + /// behind (Cassandra) reported with the exit kept; a row left, a stop + /// and a failed re-read exit 1. + #[test] + fn forget_store_exits_1_while_a_row_of_the_root_is_left() { + let done = ForgetStoreReport { + rows: vec![( + named(BackupStatus::Failed, "arn:1"), + Forgotten::Removed(BackupStatus::Failed), + )], + rows_left: Some(Vec::new()), + entries_left: vec!["store-0".to_owned()], + ..ForgetStoreReport::default() + }; + super::forget_store_outcome(&done).unwrap(); + let lines = super::forget_store_lines(&done, &old_root()); + assert!( + lines.iter().any(|l| l.contains("store id(s) store-0")), + "{lines:?}" + ); + assert_eq!( + lines.last().unwrap(), + "=== extenddb backup sync --forget-store complete: 1 row(s) removed, 0 left ===" + ); + let left = ForgetStoreReport { + rows: vec![( + named(BackupStatus::Creating, "arn:2"), + Forgotten::LiveOwner(BackupStatus::Creating), + )], + rows_left: Some(vec![ + named(BackupStatus::Creating, "arn:2"), + named(BackupStatus::Available, "arn:3"), + ]), + ..ForgetStoreReport::default() + }; + let err = super::forget_store_outcome(&left).unwrap_err().to_string(); + assert!(err.starts_with("2 row(s) of the root are left"), "{err}"); + assert!( + err.contains("extenddb backup remove-target --account"), + "{err}" + ); + let lines = super::forget_store_lines(&left, &old_root()); + // The row the listing after the removals holds and the first did + // not (arn:3) is named as landed; arn:2 is named above it, once. + let landed = lines + .iter() + .position(|l| l.starts_with("--- Rows that landed under the root during the run")) + .expect("the landed section"); + assert_eq!( + lines[landed + 1], + " arn:3 (AVAILABLE, account 123456789012)" + ); + assert_eq!( + lines.iter().filter(|l| l.contains("arn:2")).count(), + 1, + "{lines:?}" + ); + assert_eq!( + lines.last().unwrap(), + "=== extenddb backup sync --forget-store incomplete: 0 row(s) removed, 2 left ===" + ); + let swept = ForgetStoreReport { + rows_left: Some(Vec::new()), + entries_swept: Some(extenddb_storage::backup_catalog::StatusEntrySweep { + removed: 2, + repaired: 1, + }), + ..ForgetStoreReport::default() + }; + let lines = super::forget_store_lines(&swept, &old_root()); + assert!( + lines + .iter() + .any(|l| l.starts_with(" (2 status entry(ies) of the root")) + && lines + .iter() + .any(|l| l.starts_with(" (1 status entry(ies) of rows left")), + "{lines:?}" + ); + let stopped = ForgetStoreReport { + stopped: Some("SIGTERM received".to_owned()), + ..ForgetStoreReport::default() + }; + assert!(super::forget_store_outcome(&stopped).is_err()); + let unread = ForgetStoreReport { + recheck_error: Some("the root's rows could not be read: down".to_owned()), + ..ForgetStoreReport::default() + }; + assert!(super::forget_store_outcome(&unread).is_err()); + } + + /// `--forget-backup` exits 0 when every ARN's row is gone, and 1 naming + /// the ARN it stopped at (a legacy ARN included); the ARNs after it are + /// listed as not reached. + #[test] + fn forget_backup_exits_1_naming_the_arn_it_stopped_at() { + let asked = ["arn:1".to_owned(), "arn:2".to_owned(), "arn:3".to_owned()]; + let removed = |arn: &str| { + ( + arn.to_owned(), + ForgetBackup::Row( + BackupStatus::Available, + Forgotten::Removed(BackupStatus::Available), + ), + ) + }; + let all = ForgetBackupsReport { + done: asked.iter().map(|a| removed(a)).collect(), + stopped: None, + }; + super::forget_backups_outcome(&all, 3).unwrap(); + // An ARN no row carries counts as done: a run given again after a + // stop goes past the ARNs it removed. + let again = ForgetBackupsReport { + done: vec![ + ("arn:1".to_owned(), ForgetBackup::NoRow), + removed("arn:2"), + removed("arn:3"), + ], + stopped: None, + }; + super::forget_backups_outcome(&again, 3).unwrap(); + let lines = super::forget_backups_lines(&again, &asked); + assert!( + lines.contains(&" arn:1: no row carries this ARN; nothing to remove".to_owned()), + "{lines:?}" + ); + assert_eq!( + lines.last().unwrap(), + "=== extenddb backup sync --forget-backup complete: 2 row(s) removed, 0 not ===" + ); + let refused = ForgetBackupsReport { + done: vec![removed("arn:1"), ("arn:2".to_owned(), ForgetBackup::Legacy)], + stopped: None, + }; + let err = super::forget_backups_outcome(&refused, 3) + .unwrap_err() + .to_string(); + assert!( + err.starts_with( + "--forget-backup did not remove the row of arn:2 (an in-catalog backup" + ), + "{err}" + ); + let lines = super::forget_backups_lines(&refused, &asked); + assert!( + lines.contains(&" arn:3: not reached".to_owned()), + "{lines:?}" + ); + assert_eq!( + lines.last().unwrap(), + "=== extenddb backup sync --forget-backup stopped: 1 row(s) removed, 2 not ===" + ); + } + + /// The store's check after `--forget-backup`: passing, or the problem; + /// a manifest still missing names the next ARN to forget. + #[test] + fn the_check_after_forget_backup_names_the_next_backup_to_forget() { + let ok = super::store_check_lines(&Ok(Some(StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/r".to_owned(), + store_id: "store-1".to_owned(), + }))); + assert!(ok[0].starts_with("The configured store's check passes (store id store-1)")); + let empty = super::store_check_lines(&Ok(None)); + assert!( + empty[0].contains("the check writes nothing, and a server's next check creates"), + "{empty:?}" + ); + let missing = super::store_check_lines(&Err(StoreProblem::ManifestMissing { + backup_arn: "arn:next".to_owned(), + })); + assert!( + missing[1].contains("--forget-backup arn:next"), + "{missing:?}" + ); + } + + /// The check after `--forget-backup` runs against the id the opening + /// read, without one when the opening read none, and not at all when + /// the opening found no identity file (`Empty`, which the forget + /// options' opening reads for a fresh root since it writes nothing, or + /// `FileMissingWithRows`); the line printed then says to mount the + /// volume before the root's last row is gone. + #[test] + fn the_check_after_forget_backup_runs_only_where_the_opening_found_a_file() { + use super::{ForgetCheck, Identity, forget_check}; + let identity = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/r".to_owned(), + store_id: "store-1".to_owned(), + }; + assert_eq!( + forget_check(Ok(&Identity::Known(identity.clone()))), + ForgetCheck::Run(Some(identity)) + ); + assert_eq!(forget_check(Ok(&Identity::Empty)), ForgetCheck::Skip); + assert_eq!( + forget_check(Ok(&Identity::Problem(StoreProblem::FileMissingWithRows( + vec!["store-1".to_owned()] + )))), + ForgetCheck::Skip + ); + assert_eq!( + forget_check(Ok(&Identity::Problem(StoreProblem::ManifestMissing { + backup_arn: "arn:gone".to_owned() + }))), + ForgetCheck::Run(None) + ); + assert_eq!( + forget_check(Err(&super::NotConfigured::Absent)), + ForgetCheck::Run(None) + ); + let skipped = &super::skipped_check_lines("/mnt/b", false)[0]; + assert!(skipped.contains("/mnt/b had no"), "{skipped}"); + assert!( + skipped.contains("before the root's last row is gone") + && !skipped.contains("each server's next check reports the store"), + "{skipped}" + ); + let fresh = &super::skipped_check_lines("/mnt/c", true)[0]; + assert!( + fresh.contains("/mnt/c had no") + && fresh.contains("The root is fresh") + && !fresh.contains("mount the volume"), + "{fresh}" + ); + } + + /// After a move, a row of the old root in the closing report of other + /// stores makes the command exit 1; counts of status entries with no + /// row behind them, or rows of another root, do not. + #[test] + fn a_row_of_the_old_root_in_the_closing_report_is_one_that_landed() { + let root = old_root(); + let other = |root: &str, counts: Vec<(BackupStatus, u64)>, entries| OtherStore { + store: StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: root.to_owned(), + store_id: "store-0".to_owned(), + }, + counts, + unresolved: Vec::new(), + entries, + }; + assert!(super::old_root_listed( + &[other( + "/srv/old", + vec![(BackupStatus::Creating, 1)], + Vec::new() + )], + &root + )); + assert!(!super::old_root_listed( + &[other( + "/srv/old", + Vec::new(), + vec![(BackupStatus::Deleting, 1)] + )], + &root + )); + assert!(!super::old_root_listed( + &[other( + "/srv/elsewhere", + vec![(BackupStatus::Available, 2)], + Vec::new() + )], + &root + )); + let lines = super::other_stores_lines( + &[other( + "/srv/old", + vec![(BackupStatus::Available, 1)], + vec![(BackupStatus::Deleting, 1)], + )], + "/srv/new", + ); + assert_eq!( + lines[1], + " filesystem /srv/old (store id store-0): 1 AVAILABLE; 1 DELETING (status entries \ + with no row of that status behind them, which readers skip; with no row of the \ + root left, --forget-store removes them)" + ); + } + + /// The decisions the options take from the backend and the located + /// root: the owner window (zero on SQLite under the serve lock), the + /// wait before the first FAILED or DELETED removal (none on SQLite), and + /// the stores the forget options wait on (the configured store's and + /// the root's ids). + #[test] + fn the_options_take_the_window_the_wait_and_the_stores_from_the_backend() { + let stale = std::time::Duration::from_secs(300); + assert_eq!( + super::owner_window("sqlite", stale), + std::time::Duration::ZERO + ); + for backend in ["postgres", "mongodb", "cassandra"] { + assert_eq!(super::owner_window(backend, stale), stale, "{backend}"); + } + let now = tokio::time::Instant::now(); + assert_eq!(super::removals_from("sqlite", now), None); + assert_eq!( + super::removals_from("postgres", now), + Some(now + extenddb_engine::store_maintenance::FORGET_WAIT) + ); + let configured = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/srv/new".to_owned(), + store_id: "store-1".to_owned(), + }; + let root = old_root(); + assert_eq!( + super::forget_wait_identities(Some(&configured), &[], Some(&root)), + [configured.clone(), root.identities()[0].clone()] + ); + assert_eq!( + super::forget_wait_identities(None, &[], Some(&root)), + root.identities() + ); + assert_eq!( + super::forget_wait_identities(Some(&configured), &[], None), + [configured] + ); + assert!(super::forget_wait_identities(None, &[], None).is_empty()); + let ids = [ + StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/srv/new".to_owned(), + store_id: "store-1".to_owned(), + }, + StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/srv/new".to_owned(), + store_id: "store-2".to_owned(), + }, + ]; + assert_eq!( + super::forget_wait_identities(None, &ids, Some(&root)), + [ids[0].clone(), ids[1].clone(), root.identities()[0].clone()], + "the configured root's ids when its identity is not known" + ); + assert_eq!( + super::forget_wait_identities(Some(&ids[0]), &ids, None), + [ids[0].clone()] + ); + } + + /// The configured root's stores the forget options wait on when its + /// identity is not known: one per id the catalog records, and a failed + /// read of those ids refuses the command. + #[test] + fn a_failed_read_of_the_configured_roots_ids_refuses_the_forget() { + let ids = super::configured_root_identities( + StoreKind::Filesystem, + "/srv/new", + Ok(vec!["store-1".to_owned(), "store-2".to_owned()]), + ) + .unwrap(); + assert_eq!( + ids.iter().map(|i| i.store_id.as_str()).collect::>(), + ["store-1", "store-2"] + ); + assert!(ids.iter().all(|i| i.store_root == "/srv/new")); + let err = super::configured_root_identities( + StoreKind::Filesystem, + "/srv/new", + Err(extenddb_storage::error::StorageError::Internal( + "pool exhausted".to_owned(), + )), + ) + .unwrap_err() + .to_string(); + assert!( + err.contains("store ids for the configured root /srv/new could not be read") + && err.contains("pool exhausted") + && err.ends_with("nothing was changed"), + "{err}" + ); + } + + /// A sync whose report of other stores failed exits 1; plain sync's + /// refusal on a missing manifest names the backup's own + /// `--forget-backup` command. + #[test] + fn a_failed_report_of_other_stores_exits_1_and_the_refusal_names_the_arn() { + super::others_outcome(None, super::run_again(false)).unwrap(); + let err = super::others_outcome(Some("down"), super::run_again(false)) + .unwrap_err() + .to_string(); + assert_eq!( + err, + "the rows of other stores could not be read: down; run the command again" + ); + // After a completed move the advice is plain sync: `--move-store` + // would refuse the old root as having no rows. + let err = super::others_outcome(Some("down"), super::run_again(true)) + .unwrap_err() + .to_string(); + assert!( + err.ends_with( + "; run extenddb backup sync (plain, without --move-store: the rows are moved, \ + and the option would refuse the old root as having no rows)" + ), + "{err}" + ); + let hint = super::manifest_missing_hint(&StoreProblem::ManifestMissing { + backup_arn: "arn:aws:dynamodb:us-east-1:1:table/T/backup/01".to_owned(), + }); + assert!( + hint.contains("--forget-backup arn:aws:dynamodb:us-east-1:1:table/T/backup/01,"), + "{hint}" + ); + assert!(super::manifest_missing_hint(&StoreProblem::Transport("x".to_owned())).is_empty()); + } } diff --git a/crates/app/src/cmd_init.rs b/crates/app/src/cmd_init.rs index 5ec78a390..970642b35 100755 --- a/crates/app/src/cmd_init.rs +++ b/crates/app/src/cmd_init.rs @@ -21,11 +21,13 @@ pub struct InitArgs { #[arg(long)] backend: Option, - /// Data database name (default: extenddb) + /// Data database name (default: the catalog's name without its + /// `_catalog` suffix). PostgreSQL backend only. #[arg(long)] data_db: Option, - /// Catalog database name (default: _catalog) + /// Catalog database name (default: extenddb_catalog, or the config + /// file's). PostgreSQL backend only. #[arg(long)] catalog_db: Option, diff --git a/crates/backup-store/src/fs.rs b/crates/backup-store/src/fs.rs index fff5547dc..ec9d24c3e 100644 --- a/crates/backup-store/src/fs.rs +++ b/crates/backup-store/src/fs.rs @@ -54,7 +54,9 @@ //! blocking pool, where dropping the future does not stop it; a guard in //! the future sets a flag when it is dropped, and the walk checks the flag //! before each removal, so a dropped `delete_prefix` stops at its next -//! entry. +//! entry. `delete_prefix_before` checks its deadline at the same point, so a +//! walk that reaches the deadline stops before its next removal, whatever +//! the caller's timer has done by then. //! //! Writes go to a temporary sibling file and rename into place, so a crash //! never leaves a truncated file under its final name. The temporary is @@ -126,7 +128,7 @@ use futures::{FutureExt, StreamExt, TryStreamExt}; use tokio::io::AsyncWriteExt; use crate::key::{normalize_prefix, validate_key, validate_prefix}; -use crate::{BackupStore, ByteStream, ObjectMeta, StoreError}; +use crate::{BackupStore, ByteStream, ObjectMeta, PrefixDeletion, StoreError}; /// Prefix and suffix marking in-flight temporary files, hidden from listings. const TEMP_PREFIX: &str = ".put-"; @@ -600,6 +602,19 @@ impl FilesystemStore { } async fn delete_prefix_impl(&self, prefix: &str) -> Result { + self.delete_walk(prefix, None) + .await + .map(|done| done.deleted) + } + + /// The walk of `delete_prefix` and `delete_prefix_before`: with a + /// `deadline`, the walk starts no removal at or after it and returns + /// what it removed with `finished` false. + async fn delete_walk( + &self, + prefix: &str, + deadline: Option, + ) -> Result { validate_prefix(prefix)?; let normalized = normalize_prefix(prefix).to_owned(); let root = self.root.clone(); @@ -614,10 +629,25 @@ impl FilesystemStore { cancel: cancel.0.clone(), removal_hook: self.removal_hook.clone(), removals: std::sync::atomic::AtomicU64::new(0), + deadline, + out_of_time: std::sync::atomic::AtomicBool::new(false), + removed: std::sync::atomic::AtomicU64::new(0), }; - tokio::task::spawn_blocking(move || delete_objects(&root, &normalized, &walk)) - .await - .map_err(|e| StoreError::Other(format!("deletion task failed: {e}")))? + tokio::task::spawn_blocking(move || { + let outcome = delete_objects(&root, &normalized, &walk); + if walk.out_of_time.load(std::sync::atomic::Ordering::Acquire) { + return Ok(PrefixDeletion { + deleted: walk.removed.load(std::sync::atomic::Ordering::Acquire), + finished: false, + }); + } + outcome.map(|deleted| PrefixDeletion { + deleted, + finished: true, + }) + }) + .await + .map_err(|e| StoreError::Other(format!("deletion task failed: {e}")))? } async fn validate_impl(&self) -> Result<(), StoreError> { @@ -742,6 +772,14 @@ impl BackupStore for FilesystemStore { self.delete_prefix_impl(prefix).boxed() } + fn delete_prefix_before<'a>( + &'a self, + prefix: &'a str, + deadline: tokio::time::Instant, + ) -> BoxFuture<'a, Result> { + self.delete_walk(prefix, Some(deadline)).boxed() + } + fn validate(&self) -> BoxFuture<'_, Result<(), StoreError>> { self.validate_impl().boxed() } @@ -1254,23 +1292,40 @@ impl Drop for CancelOnDrop { } /// What one `delete_prefix` walk carries down the tree: the flag its -/// `CancelOnDrop` sets, and the store's removal hook (set only in the -/// crate's own tests) with the count it is called with. +/// `CancelOnDrop` sets, the store's removal hook (set only in the crate's +/// own tests) with the count it is called with, and for +/// `delete_prefix_before` the deadline, whether the walk reached it, and +/// the files it removed. struct DeleteWalk { cancel: std::sync::Arc, removal_hook: Option, removals: std::sync::atomic::AtomicU64, + deadline: Option, + out_of_time: std::sync::atomic::AtomicBool, + removed: std::sync::atomic::AtomicU64, } impl DeleteWalk { - /// The error a cancelled walk ends with; nobody receives it, since the - /// caller's future is gone. + /// The error a cancelled walk, or one past its deadline, ends with; + /// nobody receives the first, since the caller's future is gone, and + /// `delete_walk` turns the second into an unfinished deletion. Called + /// before each removal. fn check_cancel(&self) -> Result<(), StoreError> { if self.cancel.load(std::sync::atomic::Ordering::Acquire) { return Err(StoreError::Other( "delete_prefix was dropped before it finished".to_owned(), )); } + if self + .deadline + .is_some_and(|deadline| tokio::time::Instant::now() >= deadline) + { + self.out_of_time + .store(true, std::sync::atomic::Ordering::Release); + return Err(StoreError::Other( + "delete_prefix_before reached its deadline".to_owned(), + )); + } Ok(()) } @@ -1368,7 +1423,11 @@ fn check_tree(dir: &Path, dir_meta: &std::fs::Metadata) -> Result<(), StoreError fn remove_file_counted(path: &Path, walk: &DeleteWalk) -> Result { walk.before_removal(); match std::fs::remove_file(path) { - Ok(()) => Ok(1), + Ok(()) => { + walk.removed + .fetch_add(1, std::sync::atomic::Ordering::AcqRel); + Ok(1) + } Err(e) if vanished(&e) => Ok(0), Err(e) => Err(StoreError::io(format!("remove {}", path.display()), e)), } diff --git a/crates/backup-store/src/lib.rs b/crates/backup-store/src/lib.rs index 23e7526ad..7724ebcc7 100644 --- a/crates/backup-store/src/lib.rs +++ b/crates/backup-store/src/lib.rs @@ -69,6 +69,17 @@ pub struct ObjectMeta { pub etag: Option, } +/// What one [`BackupStore::delete_prefix_before`] did. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PrefixDeletion { + /// The objects removed, counted as [`BackupStore::delete_prefix`] + /// counts them. + pub deleted: u64, + /// Whether the deletion ran to its end; `false` when it stopped at the + /// deadline, with the rest of the prefix left. + pub finished: bool, +} + /// A destination for backup objects. /// /// Implementations must be safe to share across tasks; the engine holds one @@ -189,6 +200,44 @@ pub trait BackupStore: Send + Sync { /// the dropped future sets). fn delete_prefix<'a>(&'a self, prefix: &'a str) -> BoxFuture<'a, Result>; + /// [`BackupStore::delete_prefix`] that starts no removal at or after + /// `deadline`: the filesystem store's walk checks the deadline beside its + /// cancel flag before each removal, the S3 store before each + /// `ListObjectsV2`, `DeleteObjects`, `ListMultipartUploads` and + /// `AbortMultipartUpload`. A call that reaches the deadline returns what + /// it removed so far with [`PrefixDeletion::finished`] false; what + /// remains is the step in flight at the deadline (one removal on a + /// filesystem root, one request on S3, up to 1,000 keys for a + /// `DeleteObjects`). The check reads the clock in the process, so it + /// bounds a process that keeps running; one paused between a check and + /// the step it admits (a stopped process, a frozen VM) runs that step + /// when it resumes. The caller still bounds the call with a timeout for + /// a step that never answers. + /// + /// The default checks the deadline once and then runs `delete_prefix` + /// whole: for a store whose deletion is one step (an in-memory test + /// store), the same rule. + fn delete_prefix_before<'a>( + &'a self, + prefix: &'a str, + deadline: tokio::time::Instant, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + if tokio::time::Instant::now() >= deadline { + return Ok(PrefixDeletion { + deleted: 0, + finished: false, + }); + } + self.delete_prefix(prefix) + .await + .map(|deleted| PrefixDeletion { + deleted, + finished: true, + }) + }) + } + /// Startup validation. The filesystem store applies the owner-only rule /// to the root and the ancestor rule to the directories above it (the /// `fs` module doc states both), checks that the root is still where diff --git a/crates/backup-store/src/s3.rs b/crates/backup-store/src/s3.rs index 465a23f32..9d724501c 100644 --- a/crates/backup-store/src/s3.rs +++ b/crates/backup-store/src/s3.rs @@ -102,7 +102,7 @@ use futures::{FutureExt, StreamExt, TryStreamExt}; use crate::config::S3StoreConfig; use crate::key::{key_matches_prefix, normalize_prefix, validate_key, validate_prefix}; -use crate::{BackupStore, ByteStream, ObjectMeta, StoreError}; +use crate::{BackupStore, ByteStream, ObjectMeta, PrefixDeletion, StoreError}; /// The smallest part size S3 accepts for any part except the last. pub const MIN_PART_SIZE_BYTES: u64 = 5 * 1024 * 1024; @@ -925,12 +925,34 @@ impl S3Store { } async fn delete_prefix_impl(&self, prefix: &str) -> Result { + self.delete_requests(prefix, None) + .await + .map(|done| done.deleted) + } + + /// The requests of `delete_prefix` and `delete_prefix_before`: with a + /// `deadline`, no request is sent at or after it, and the call returns + /// what it deleted with `finished` false. + async fn delete_requests( + &self, + prefix: &str, + deadline: Option, + ) -> Result { validate_prefix(prefix)?; let match_prefix = normalize_prefix(prefix); let request_prefix = format!("{}{match_prefix}", self.prefix); let mut deleted: u64 = 0; let mut token: Option = None; + let out_of_time = + || deadline.is_some_and(|deadline| tokio::time::Instant::now() >= deadline); + let unfinished = |deleted| PrefixDeletion { + deleted, + finished: false, + }; loop { + if out_of_time() { + return Ok(unfinished(deleted)); + } // Raw bucket-side keys, not `list_page`'s filtered view: an // object whose key fails the store rule is still under the // prefix and still has to go. @@ -964,6 +986,9 @@ impl S3Store { }) .collect::, _>>()?; let count = objects.len() as u64; + if out_of_time() { + return Ok(unfinished(deleted)); + } let delete = Delete::builder() .set_objects(Some(objects)) .quiet(true) @@ -1006,9 +1031,16 @@ impl S3Store { // does not see them and S3 bills their parts until they are aborted. // A process that died mid-put left one; the reconciler's prefix // deletion is the one place that sweeps it. - self.abort_uploads_under(&request_prefix, match_prefix) - .await?; - Ok(deleted) + if !self + .abort_uploads_under(&request_prefix, match_prefix, deadline) + .await? + { + return Ok(unfinished(deleted)); + } + Ok(PrefixDeletion { + deleted, + finished: true, + }) } /// Abort every in-progress multipart upload under the normalized store @@ -1027,14 +1059,22 @@ impl S3Store { /// backstop for such uploads, and `validate` reports either missing /// grant at startup (it probes both). A token code (`ExpiredToken` and /// the others in `is_token_code`) is not a denial and fails the call. + /// With a `deadline`, no request is sent at or after it, and the sweep + /// answers `false` (not finished); otherwise `true`. async fn abort_uploads_under( &self, request_prefix: &str, match_prefix: &str, - ) -> Result<(), StoreError> { + deadline: Option, + ) -> Result { + let out_of_time = + || deadline.is_some_and(|deadline| tokio::time::Instant::now() >= deadline); let mut key_marker: Option = None; let mut id_marker: Option = None; loop { + if out_of_time() { + return Ok(false); + } let output = match self .client .list_multipart_uploads() @@ -1056,7 +1096,7 @@ impl S3Store { there stay open until the bucket's AbortIncompleteMultipartUpload \ lifecycle rule removes them" ); - return Ok(()); + return Ok(true); } Err(e) => return Err(map_sdk_error("ListMultipartUploads", e)), }; @@ -1070,6 +1110,9 @@ impl S3Store { if !matches { continue; } + if out_of_time() { + return Ok(false); + } match self .client .abort_multipart_upload() @@ -1110,7 +1153,7 @@ impl S3Store { break; } } - Ok(()) + Ok(true) } async fn validate_impl(&self) -> Result<(), StoreError> { @@ -1325,6 +1368,14 @@ impl BackupStore for S3Store { self.delete_prefix_impl(prefix).boxed() } + fn delete_prefix_before<'a>( + &'a self, + prefix: &'a str, + deadline: tokio::time::Instant, + ) -> BoxFuture<'a, Result> { + self.delete_requests(prefix, Some(deadline)).boxed() + } + /// Lists one key under the prefix (`ListObjectsV2`, `s3:ListBucket`) /// rather than calling `HeadBucket`, since only a GET-shaped error has a /// body: a 403 keeps S3's code, a 404 names the missing bucket, a 301 the diff --git a/crates/backup-store/tests/conformance.rs b/crates/backup-store/tests/conformance.rs index 30adcdd53..f44cb6887 100644 --- a/crates/backup-store/tests/conformance.rs +++ b/crates/backup-store/tests/conformance.rs @@ -24,7 +24,7 @@ use bytes::Bytes; use extenddb_backup_store::{BackupStore, ByteStream, StoreError, byte_stream_from}; -use extenddb_backup_store::{MIN_PART_SIZE_BYTES, ObjectMeta}; +use extenddb_backup_store::{MIN_PART_SIZE_BYTES, ObjectMeta, PrefixDeletion}; use futures::TryStreamExt; /// Part size every conformance run uses: the smallest S3 accepts, so the @@ -270,6 +270,47 @@ mod suite { ); } + /// `delete_prefix_before` with its deadline passed removes nothing and + /// says it did not finish, a prefix that names one object included; + /// with a deadline ahead it deletes the prefix as `delete_prefix` does. + pub async fn delete_prefix_before_honours_its_deadline(store: &dyn BackupStore) { + for key in ["dl/a/1", "dl/a/2", "dl/b/3", "keepdl/4"] { + store.put(key, chunked_body(vec![9])).await.unwrap(); + } + let passed = tokio::time::Instant::now(); + let done = store.delete_prefix_before("dl", passed).await.unwrap(); + assert_eq!( + done, + PrefixDeletion { + deleted: 0, + finished: false + }, + "a deadline already passed starts no removal" + ); + assert_eq!(list_keys(store, "dl").await.len(), 3); + let done = store.delete_prefix_before("dl/a/1", passed).await.unwrap(); + assert_eq!( + done, + PrefixDeletion { + deleted: 0, + finished: false + }, + "a prefix that names one object, past its deadline" + ); + assert_eq!(list_keys(store, "dl").await.len(), 3); + let ahead = tokio::time::Instant::now() + std::time::Duration::from_secs(60); + let done = store.delete_prefix_before("dl", ahead).await.unwrap(); + assert_eq!( + done, + PrefixDeletion { + deleted: 3, + finished: true + } + ); + assert!(list_keys(store, "dl").await.is_empty()); + assert_eq!(list_keys(store, "keepdl").await, vec!["keepdl/4"]); + } + pub async fn missing_object_is_not_found(store: &dyn BackupStore) { let err = store.get("no/such/object").await.err().expect("get fails"); assert!(matches!(err, StoreError::NotFound), "{err}"); @@ -657,6 +698,12 @@ mod filesystem { suite::delete_prefix_removes_and_counts(&store).await; } + #[tokio::test] + async fn delete_prefix_before_honours_its_deadline() { + let (_dir, store) = fs_store().await; + suite::delete_prefix_before_honours_its_deadline(&store).await; + } + #[tokio::test] async fn missing_object_is_not_found() { let (_dir, store) = fs_store().await; @@ -931,6 +978,42 @@ mod filesystem { } } + /// A walk held past its deadline right before a removal (the hook, as a + /// process paused there would be) removes the entry in flight and no + /// other, and says it did not finish. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn delete_prefix_before_stops_at_its_deadline() { + use std::os::unix::fs::OpenOptionsExt as _; + const HOLD: std::time::Duration = std::time::Duration::from_millis(600); + let (dir, mut store) = fs_store().await; + store.set_removal_hook(std::sync::Arc::new(move |removals| { + if removals == 1 { + std::thread::sleep(HOLD); + } + })); + let prefix = dir.path().join("held"); + private_dir(&prefix); + for f in 0..6 { + std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(prefix.join(format!("f{f}"))) + .expect("create an object"); + } + let deadline = tokio::time::Instant::now() + HOLD / 2; + let done = store.delete_prefix_before("held", deadline).await.unwrap(); + assert_eq!( + done, + PrefixDeletion { + deleted: 2, + finished: false + }, + "the removal before the hold and the one in flight at the deadline" + ); + assert_eq!(std::fs::read_dir(&prefix).unwrap().count(), 4); + } + /// `validate` beside a `delete_prefix("")`, which removes temporaries /// and so the probe too: every `validate` succeeds. A round in which /// the deletion removed something (the probe is the only file under the @@ -2801,6 +2884,262 @@ mod s3 { suite::delete_prefix_removes_and_counts(&store).await; } + #[tokio::test] + async fn delete_prefix_before_honours_its_deadline() { + let Some(store) = store().await else { return }; + suite::delete_prefix_before_honours_its_deadline(&store).await; + } + + /// Holds the first request of one operation of the deletion on the + /// sending thread past the caller's deadline, as a process paused + /// between the deadline's check and the request would: `ListObjectsV2` + /// (mode 1), `DeleteObjects` (mode 2) or `AbortMultipartUpload` (mode + /// 3), once armed. Counts the `ListObjectsV2`, `DeleteObjects`, + /// `ListMultipartUploads` and `AbortMultipartUpload` requests sent. + #[derive(Debug, Default)] + struct HoldOneDeletionRequest { + mode: std::sync::Arc, + lists: std::sync::Arc, + deletes: std::sync::Arc, + upload_lists: std::sync::Arc, + aborts: std::sync::Arc, + } + + /// How long [`HoldOneDeletionRequest`] holds its request. + const DELETION_HOLD: std::time::Duration = std::time::Duration::from_millis(600); + + impl aws_sdk_s3::config::Intercept for HoldOneDeletionRequest { + fn name(&self) -> &'static str { + "HoldOneDeletionRequest" + } + + fn read_before_execution( + &self, + context: &aws_sdk_s3::config::interceptors::BeforeSerializationInterceptorContextRef< + '_, + >, + _cfg: &mut aws_sdk_s3::config::ConfigBag, + ) -> Result<(), aws_sdk_s3::error::BoxError> { + use std::sync::atomic::Ordering; + let input = context.input(); + let is_list = input + .downcast_ref::() + .is_some(); + let is_delete = input + .downcast_ref::() + .is_some(); + let is_upload_list = input + .downcast_ref::< + aws_sdk_s3::operation::list_multipart_uploads::ListMultipartUploadsInput, + >() + .is_some(); + let is_abort = input + .downcast_ref::< + aws_sdk_s3::operation::abort_multipart_upload::AbortMultipartUploadInput, + >() + .is_some(); + if is_delete { + self.deletes.fetch_add(1, Ordering::SeqCst); + } + if is_list { + self.lists.fetch_add(1, Ordering::SeqCst); + } + if is_upload_list { + self.upload_lists.fetch_add(1, Ordering::SeqCst); + } + if is_abort { + self.aborts.fetch_add(1, Ordering::SeqCst); + } + let held = match self.mode.load(Ordering::SeqCst) { + 1 => is_list, + 2 => is_delete, + 3 => is_abort, + _ => false, + }; + if held && self.mode.swap(0, Ordering::SeqCst) != 0 { + std::thread::sleep(DELETION_HOLD); + } + Ok(()) + } + } + + /// A deletion whose process is held past its deadline right after a + /// check sends the one request that check admitted and no other: held + /// at its first `ListObjectsV2`, it deletes nothing; held at its first + /// `DeleteObjects`, that batch (two keys, the page size) is deleted and + /// no further listing or batch is sent. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn delete_prefix_before_sends_no_request_after_its_deadline() { + use std::sync::atomic::Ordering; + let Some(endpoint) = endpoint_or_skip() else { + return; + }; + let config = test_config(endpoint, test_bucket(), unique_prefix()); + let hold = HoldOneDeletionRequest::default(); + let (mode, lists, deletes) = (hold.mode.clone(), hold.lists.clone(), hold.deletes.clone()); + let mut store = S3Store::open_with_interceptor(&config, hold) + .await + .expect("S3Store::open_with_interceptor"); + store.set_page_size(2); + ensure_bucket(&config).await; + for (cell, armed, deleted) in [ + ("held at its first ListObjectsV2", 1u8, 0u64), + ("held at its first DeleteObjects", 2, 2), + ] { + let prefix = format!("held{armed}"); + for n in 0..5 { + store + .put(&format!("{prefix}/{n}"), chunked_body(vec![7])) + .await + .unwrap(); + } + deletes.store(0, Ordering::SeqCst); + lists.store(0, Ordering::SeqCst); + mode.store(armed, Ordering::SeqCst); + let deadline = tokio::time::Instant::now() + DELETION_HOLD / 2; + let done = store.delete_prefix_before(&prefix, deadline).await.unwrap(); + assert_eq!( + done, + PrefixDeletion { + deleted, + finished: false + }, + "{cell}" + ); + assert_eq!( + deletes.load(Ordering::SeqCst), + usize::try_from(deleted / 2).unwrap(), + "{cell}: the DeleteObjects requests sent" + ); + assert_eq!( + lists.load(Ordering::SeqCst), + 1, + "{cell}: the ListObjectsV2 requests sent" + ); + assert_eq!( + list_keys(&store, &prefix).await.len(), + usize::try_from(5 - deleted).unwrap(), + "{cell}: the objects left" + ); + store.delete_prefix(&prefix).await.unwrap(); + } + } + + /// The multipart half of the deadline. Held at the last `DeleteObjects` + /// of a single page past its deadline, with an upload open under the + /// prefix: the page is deleted, no `ListMultipartUploads` is sent, the + /// upload stays open and the deletion answers unfinished. Held at its + /// first `AbortMultipartUpload` past the deadline, with two uploads open + /// and no object: that abort is sent, the second upload is not, and the + /// deletion answers unfinished. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn delete_prefix_before_aborts_no_upload_after_its_deadline() { + use std::sync::atomic::Ordering; + let Some(endpoint) = endpoint_or_skip() else { + return; + }; + let config = test_config(endpoint, test_bucket(), unique_prefix()); + let hold = HoldOneDeletionRequest::default(); + let (mode, upload_lists, aborts) = ( + hold.mode.clone(), + hold.upload_lists.clone(), + hold.aborts.clone(), + ); + let store = S3Store::open_with_interceptor(&config, hold) + .await + .expect("S3Store::open_with_interceptor"); + ensure_bucket(&config).await; + let client = raw_client(&config).await; + let open_uploads = |prefix: String| { + let client = client.clone(); + let bucket = config.bucket.clone(); + async move { + client + .list_multipart_uploads() + .bucket(bucket) + .prefix(prefix) + .send() + .await + .expect("list_multipart_uploads") + .uploads() + .iter() + .map(|u| { + ( + u.key().unwrap_or_default().to_owned(), + u.upload_id().unwrap_or_default().to_owned(), + ) + }) + .collect::>() + } + }; + for (cell, armed, objects, uploads, deleted, aborted) in [ + ( + "held at its last DeleteObjects", + 2u8, + 3u64, + 1usize, + 3u64, + 0usize, + ), + ("held at its first AbortMultipartUpload", 3, 0, 2, 0, 1), + ] { + let prefix = format!("mp{armed}"); + for n in 0..objects { + store + .put(&format!("{prefix}/{n}"), chunked_body(vec![7])) + .await + .unwrap(); + } + let bucket_prefix = format!("{}{prefix}/", config.prefix); + for n in 0..uploads { + client + .create_multipart_upload() + .bucket(&config.bucket) + .key(format!("{bucket_prefix}up{n}")) + .send() + .await + .expect("create an upload"); + } + assert_eq!(open_uploads(bucket_prefix.clone()).await.len(), uploads); + upload_lists.store(0, Ordering::SeqCst); + aborts.store(0, Ordering::SeqCst); + mode.store(armed, Ordering::SeqCst); + let deadline = tokio::time::Instant::now() + DELETION_HOLD / 2; + let done = store.delete_prefix_before(&prefix, deadline).await.unwrap(); + assert_eq!( + done, + PrefixDeletion { + deleted, + finished: false + }, + "{cell}" + ); + assert_eq!( + upload_lists.load(Ordering::SeqCst), + usize::from(armed == 3), + "{cell}: the ListMultipartUploads requests sent" + ); + assert_eq!( + aborts.load(Ordering::SeqCst), + aborted, + "{cell}: the AbortMultipartUpload requests sent" + ); + let left = open_uploads(bucket_prefix).await; + assert_eq!(left.len(), uploads - aborted, "{cell}: the uploads left"); + for (key, id) in left { + client + .abort_multipart_upload() + .bucket(&config.bucket) + .key(key) + .upload_id(id) + .send() + .await + .expect("abort a left upload"); + } + store.delete_prefix(&prefix).await.unwrap(); + } + } + #[tokio::test] async fn missing_object_is_not_found() { let Some(store) = store().await else { return }; diff --git a/crates/engine/src/backup.rs b/crates/engine/src/backup.rs index 213d3be53..776dc2e7f 100755 --- a/crates/engine/src/backup.rs +++ b/crates/engine/src/backup.rs @@ -41,7 +41,8 @@ pub use maintenance::HeldClaim; pub use restore::{OTHER_STORE, RESTORE_REGISTRATION_TIMEOUT, TARGET_NOT_EMPTY}; pub use runtime::{ BackupRuntime, ConfiguredStore, JobSettings, LOCAL_CHECK_TIMEOUT, S3_CHECK_TIMEOUT, Spool, - StoreFence, StoreHealth, StoreHealthReport, check_configured_store, read_identity_bounded, + StoreFence, StoreHealth, StoreHealthReport, check_configured_store, + check_configured_store_read_only, read_identity_bounded, }; pub use store_identity::{STORE_IDENTITY_KEY, StoreIdentityFile, StoreProblem}; @@ -555,12 +556,23 @@ where /// engine has no point between the row and the copy to check at, and the /// store-backed create checks here too rather than after its insert, since /// a row written FAILED for the refusal carried the insert's heartbeat and -/// `backup purge`'s wait counted it as a live job for the stale window. A -/// claim taken between this check and the row's insert is therefore not -/// seen by the job; the in-catalog create that slips through writes no -/// object of the store the command deletes from, and a restore or a -/// store-backed create that slips through is a RUNNING record or a CREATING -/// row the command's wait lists, so the window costs nothing. +/// `backup purge`'s wait counted it as a live job for the stale window. The +/// store-backed create reads the claim again immediately before its +/// `claim_backup`, and the store-backed restore immediately before its +/// registration (and once more after it), so a claim a command took while +/// the request opened the table or read the manifest refuses the job with +/// no row. What remains is the window from that last read to the insert, +/// however long the insert takes: a claim taken in it is not seen by the +/// job, and its CREATING row or RUNNING record lands under the claim. One +/// that lands during the command's wait is listed by it, and one that lands +/// before `--move-store`'s rewrite stays under the old root, which the +/// move's re-list reports (the rewrite skips CREATING rows). A CreateBackup +/// whose insert lands after the move's re-list, during its sync of the +/// configured root, is reported by the sync's closing report of other +/// stores if it lands before that report (and the command exits 1), and +/// only by a later plain sync otherwise. The in-catalog +/// create, which commits its row inside the backend, writes no object of +/// the store a command deletes from. async fn refuse_while_maintenance_held( storage: &S, runtime: &BackupRuntime, @@ -2020,6 +2032,7 @@ pub mod reconciler; mod restore; mod runtime; pub mod store_identity; +pub mod store_maintenance; pub mod sync; #[cfg(test)] mod tests; diff --git a/crates/engine/src/backup/job.rs b/crates/engine/src/backup/job.rs index bca4db180..f00c2d23b 100644 --- a/crates/engine/src/backup/job.rs +++ b/crates/engine/src/backup/job.rs @@ -19,9 +19,11 @@ //! pipeline's item and chunk channels are the job's own), and a client that //! disconnects mid-request cannot leave a row whose owner was never //! spawned. The maintenance claim is checked by the request before the -//! permit, as the in-catalog paths check it, so a held claim refuses the -//! request and inserts no row (a FAILED row written for the refusal was -//! counted by `backup purge`'s wait as a live job for the stale window). +//! permit, as the in-catalog paths check it, and again immediately before +//! `claim_backup`, so a held claim refuses the request and inserts no row +//! (a FAILED row written for the refusal was counted by `backup purge`'s +//! wait as a live job for the stale window), and a claim taken while the +//! open waited (on a lock the table holds, say) refuses it too. //! The open and the insert are selected against the shutdown token, so a //! server that is stopping neither waits on an open the table's lock holds //! nor inserts a row it would fail at once. The `ServiceUnavailable` that @@ -102,7 +104,7 @@ use tokio::sync::oneshot; use tokio::time::Instant; use super::runtime::{BackupRuntime, ConfiguredStore, JobSettings, Spool, StoreFence}; -use super::{create_backup_storage_err_to_dynamo, storage_err_to_dynamo}; +use super::{create_backup_storage_err_to_dynamo, maintenance_held, storage_err_to_dynamo}; use failure::{FailurePath, Written}; use heartbeat::{HeartbeatHandle, HeartbeatMetric}; use writer::{DataFiles, Pipeline, Stop}; @@ -355,6 +357,21 @@ where )) })); } + // The maintenance claim once more, immediately before the insert: the + // request's check ran before the open, the TTL read and the fence, any + // of which can wait (an open behind the table's lock), and a command + // that took the claim since gets no row of this job. A claim taken + // between this read and the insert is not seen: the CREATING row lands + // under the claim, where the command's wait lists it, the move's re-list + // of the old root reports it, or (an insert after that re-list) the + // move's closing report of other stores or a later plain sync does. + if storage + .check_maintenance_claim(env.settings.stale_job_after) + .await + .map_err(storage_err_to_dynamo)? + { + return Err(maintenance_held()); + } let claimed_at = Instant::now(); // The insert too is selected against the token: a server that is // stopping inserts no row it would have to fail at once. diff --git a/crates/engine/src/backup/maintenance.rs b/crates/engine/src/backup/maintenance.rs index afd3f62bf..1e4d899bc 100644 --- a/crates/engine/src/backup/maintenance.rs +++ b/crates/engine/src/backup/maintenance.rs @@ -387,14 +387,24 @@ pub struct Wait { pub poll: Duration, } -/// Wait until no CREATING or FAILED backup row of `identity` has a live -/// owner and no restore job is RUNNING, re-reading every `wait.poll`, for -/// at most `wait.limit`; the claim lost, a catalog read failing, or a +/// Wait until no CREATING or FAILED backup row of any of `identities` has a +/// live owner and no restore job is RUNNING, re-reading every `wait.poll`, +/// for at most `wait.limit`; the claim lost, a catalog read failing, or a /// signal ends it early. The jobs found running are reported through -/// `announce` the first time they are seen. +/// `announce` the first time they are seen. Purge and `backup sync`'s +/// `--adopt-store` and `--prune-missing` pass the configured store; +/// `--move-store` and `--forget-store` add every store id the catalog +/// records for the old root (`read_store_ids`), so a CreateBackup of a +/// server still configured with that root is waited on too; `--forget-store` +/// and `--forget-backup` pass, when the configured store's identity is not +/// known (an unhealthy store, a root with no identity file), every store id +/// the catalog records for the configured root instead of the configured +/// store. An empty slice waits on the RUNNING restores alone: with no store +/// configured, or when the catalog records no store id for the configured +/// root (a failed read of those ids refuses the command before the wait). pub async fn wait_for_quiet( storage: &S, - identity: &StoreIdentity, + identities: &[StoreIdentity], claim: &HeldClaim, wait: Wait, interrupt: &mut Interrupt, @@ -409,7 +419,7 @@ pub async fn wait_for_quiet( if let Some(signal) = interrupt.signalled() { return WaitOutcome::Interrupted(signal); } - let remaining = match busy_jobs(storage, identity, wait.owner_window).await { + let remaining = match busy_jobs(storage, identities, wait.owner_window).await { Ok(remaining) => remaining, Err(e) => return WaitOutcome::CatalogError(e), }; @@ -440,34 +450,36 @@ pub async fn wait_for_quiet( /// (`list_running_restore_jobs`), with no age. async fn busy_jobs( storage: &S, - identity: &StoreIdentity, + identities: &[StoreIdentity], owner_window: Duration, ) -> Result, String> { let mut busy = Vec::new(); - for status in [BackupStatus::Creating, BackupStatus::Failed] { - let rows: Vec = storage - .list_backup_rows(Some(status), identity) - .try_collect() - .await - .map_err(|e| format!("cannot list the catalog's backup rows: {e}"))?; - let dead: Vec = storage - .list_backup_rows_older_than(status, identity, StaleClock::Heartbeat, owner_window) - .try_collect() - .await - .map_err(|e| format!("cannot list the catalog's backup rows by age: {e}"))?; - for row in rows { - if dead.iter().any(|d| d.backup_arn == row.backup_arn) { - continue; + for identity in identities { + for status in [BackupStatus::Creating, BackupStatus::Failed] { + let rows: Vec = storage + .list_backup_rows(Some(status), identity) + .try_collect() + .await + .map_err(|e| format!("cannot list the catalog's backup rows: {e}"))?; + let dead: Vec = storage + .list_backup_rows_older_than(status, identity, StaleClock::Heartbeat, owner_window) + .try_collect() + .await + .map_err(|e| format!("cannot list the catalog's backup rows by age: {e}"))?; + for row in rows { + if dead.iter().any(|d| d.backup_arn == row.backup_arn) { + continue; + } + busy.push(format!( + "{} {} has a live owner (heartbeat within stale_job_after_secs)", + if status == BackupStatus::Creating { + "CreateBackup" + } else { + "the FAILED backup" + }, + row.backup_arn + )); } - busy.push(format!( - "{} {} has a live owner (heartbeat within stale_job_after_secs)", - if status == BackupStatus::Creating { - "CreateBackup" - } else { - "the FAILED backup" - }, - row.backup_arn - )); } } // Every RUNNING restore job, by status alone: a record stamped in the @@ -642,7 +654,7 @@ pub async fn purge_row( .map_err(|e| format!("cannot mark the row DELETED: {e}"))?; } if storage - .remove_backup_row(&key) + .remove_backup_row(&key, row.store.store_kind, &row.store.store_root) .await .map_err(|e| format!("cannot remove the row: {e}"))? { diff --git a/crates/engine/src/backup/reconciler.rs b/crates/engine/src/backup/reconciler.rs index 70605ea5e..81606cd17 100644 --- a/crates/engine/src/backup/reconciler.rs +++ b/crates/engine/src/backup/reconciler.rs @@ -75,8 +75,11 @@ //! command's wait is satisfied either way: the record is no longer //! RUNNING); a record of a store other than the //! configured one (no store configured included) is failed with -//! [`OTHER_STORE`] (configure that store and restore again; no command -//! re-points a record yet), its target removed by the FAILED-target duty +//! [`OTHER_STORE`] (configure that store, or copy its tree under the +//! configured root and move the backup's rows to the configured store +//! with `extenddb backup sync --move-store`, and +//! restore again: the move re-points backup rows, never a restore +//! record), its target removed by the FAILED-target duty //! next; while the configured store is unhealthy, or not yet checked, //! the record is skipped (one line per cycle, naming which); with no free //! job permit it is skipped to the next cycle; then the backend's @@ -162,7 +165,34 @@ //! after the refusal (the dead-owner duty's reads and writes among //! them) do not run on the root; the refusing duty's later rows are //! refused with no read of the file (the FAILED-prefix and -//! DELETED-prefix duties still list each prefix first). +//! DELETED-prefix duties still list each prefix first). The FAILED, +//! DELETED and tombstone duties then read the row once more after the +//! fence passed, skip a row that is gone or whose status or store +//! changed since the duty listed it, and read the maintenance claim +//! again, starting nothing while a command holds it. The claim is read +//! once at the cycle's start, so a cycle that began just before a +//! maintenance command took it may still be running its duties when +//! the command removes a row (`backup sync --forget-backup`, +//! `--forget-store`) or moves it (`--move-store`). A deletion or +//! removal that read the claim free must end by its deadline, +//! [`DUTY_DELETION_BOUND`] after that read was sent: a read answered +//! after the deadline starts nothing, the prefix deletion checks the +//! deadline inside the store before each removal (each request on S3, +//! `delete_prefix_before`), and a call still running at it is dropped; +//! the duty then stops for this cycle, the row left for the next. A +//! command that removes FAILED or DELETED rows beside servers waits the +//! bound and one store call after it takes the claim, so what a +//! deletion duty can still do to a row such a command removes is the +//! step in flight at the deadline: one removal on a filesystem root, one +//! request on S3 (up to 1,000 keys for a `DeleteObjects`). The bound, +//! like every lease in the design, assumes the server's process keeps +//! running: one paused past the wait (SIGSTOP, a frozen cgroup, a VM +//! pause) between its last check of the deadline and a call it has +//! already sent runs that call when it resumes. The exception is a +//! DELETING row of the configured store: its deletion is finished +//! before its removal (by the command, or by `finish_deleting_rows`, +//! which runs under the claim), the deletion its DeleteBackup asked +//! for. //! - Not while the maintenance claim is held, on every backend: the //! backend's `sweep_restore_registrations` (a no-op but on Cassandra); //! the progress-row sweep (`sweep_restore_progress` with the table ids of @@ -361,6 +391,11 @@ pub struct ReconcilerState { /// takes the held branch and does not read the claim. `None` reads it /// through `check_maintenance_claim`. maintenance_held: Option, + /// Whether the cycle leaves the store's check out and reads the health + /// as the caller recorded it: a command that writes nothing to the root + /// (`backup sync --forget-backup`, `--forget-store`), whose own check + /// writes no identity file. + skip_store_check: bool, /// How many times this instance took over each store-backed restore /// record, by target table id, for the bound [`RESTORE_TAKEOVERS`]; an /// entry is removed when the record is failed by the bound. @@ -389,6 +424,18 @@ impl ReconcilerState { ..Self::default() } } + + /// [`Self::for_maintenance_holder`] for a command that writes nothing + /// to the root: the cycle runs no store check (which may write + /// `extenddb-store.json` or a probe file), and the duties read the + /// store's health as the command's own check recorded it. + #[must_use] + pub fn for_maintenance_holder_without_store_check() -> Self { + Self { + skip_store_check: true, + ..Self::for_maintenance_holder() + } + } } /// Run cycles every `settings.interval` until `shutdown` fires. One cycle @@ -510,7 +557,9 @@ where // The store's check first, so every duty of this cycle that reads the // store's health reads what this cycle's check left. - if let Some(store) = runtime.configured_store() { + if let Some(store) = runtime.configured_store() + && !state.skip_store_check + { let check = super::check_configured_store(store, &**storage, backend); let outcome = match &state.shutdown { Some(shutdown) => tokio::select! { @@ -728,6 +777,7 @@ where storage: &**storage, store: calls, fence: &fence, + stale_job_after, }; report.failed_prefixes_deleted = delete_stale_failed_prefixes(duty, &identity, stale_job_after).await?; @@ -1299,87 +1349,158 @@ async fn resolve_dead_creating_rows( let mut resolved = Vec::new(); let mut left_to_owner = Vec::new(); for row in rows { - let claim = match storage.claim_dead_backup(&row.key(), stale_job_after).await { - Ok(Some(claim)) => claim, - Ok(None) => continue, - Err(e) => { - tracing::error!( - "could not test whether the owner of CREATING backup {} (account {}) is \ - dead; the next cycle retries: {e}", - row.backup_arn, - row.account_id - ); - continue; - } - }; - let arn = match BackupArn::parse(&row.backup_arn) { - Ok(arn) => arn, - Err(e) => { - tracing::error!( - "CREATING row {} carries an ARN this build cannot parse; left as it is: {e}", - row.backup_arn - ); - continue; - } - }; - let (to, fields) = match probe_manifest(store, &arn).await { - Ok(Probe::Complete(fields)) => (JobStatus::Available, fields), - Ok(Probe::Incomplete(why)) => { - tracing::warn!( - "backup {} (account {}): its owner is dead and the store holds no complete \ - manifest ({why}); the row is FAILED and its prefix is deleted once the row's \ - heartbeat is stale", - row.backup_arn, - row.account_id - ); - ( - JobStatus::Failed, - StatusFields { - failure_reason: Some(OWNER_DEAD_NO_MANIFEST.to_owned()), - completed: true, - ..StatusFields::default() - }, - ) - } - Err(e) => { - tracing::error!( - "backup {} (account {}): its owner is dead and the store could not be \ - probed for its manifest; the next cycle retries: {e}", - row.backup_arn, - row.account_id - ); - continue; - } - }; - match write_status(&*claim, JobStatus::Creating, to, &fields).await { - Written::Matched => { - tracing::info!( - "backup {} (account {}): its owner was dead; the row is now {to}", - row.backup_arn, - row.account_id - ); - let status = match to { - JobStatus::Available => BackupStatus::Available, - _ => BackupStatus::Failed, - }; - resolved.push((row.backup_arn, status)); - } - Written::NotMatched(why) => { - tracing::info!( - "backup {} was no longer this cycle's to resolve ({why}); left to its owner", - row.backup_arn - ); - left_to_owner.push(row.backup_arn); - } - Written::Error(e) => tracing::error!( + match resolve_dead_creating_row(storage, store, &row, stale_job_after, OnIncomplete::Fail) + .await + { + CreatingOutcome::Resolved(status) => resolved.push((row.backup_arn, status)), + CreatingOutcome::LeftToOwner(_) => left_to_owner.push(row.backup_arn), + CreatingOutcome::NotHandedOver + | CreatingOutcome::Incomplete(_) + | CreatingOutcome::Failed(_) => {} + } + } + Ok((resolved, left_to_owner)) +} + +/// What [`resolve_dead_creating_row`] did with one CREATING row. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum CreatingOutcome { + /// The backend did not hand the row over: its owner is alive, or + /// another taker has it. + NotHandedOver, + /// The row was written AVAILABLE (the store holds its complete + /// manifest) or FAILED ([`OWNER_DEAD_NO_MANIFEST`]). + Resolved(BackupStatus), + /// The write matched no row: another owner took it over since the + /// takeover (why). + LeftToOwner(String), + /// With [`OnIncomplete::Leave`], the store holds no complete manifest + /// (why): no status was written, the row is CREATING under the + /// takeover's generation with its heartbeat as it was. + Incomplete(String), + /// A catalog or store call failed, or the row's ARN does not parse + /// (why); the row is left as it was and logged. + Failed(String), +} + +/// What [`resolve_dead_creating_row`] writes when the store holds no +/// complete manifest. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum OnIncomplete { + /// FAILED ([`OWNER_DEAD_NO_MANIFEST`]): the store probed is the row's. + Fail, + /// Nothing ([`CreatingOutcome::Incomplete`]): the store probed holds a + /// copy, and the row's own store may hold the complete backup. + Leave, +} + +/// One CREATING row whose owner may be dead: the backend's takeover +/// (`claim_dead_backup` with `stale_job_after`), the manifest probe of +/// `store` ([`probe_manifest`]), and the CREATING to AVAILABLE write under +/// the taken generation, or to FAILED when the probe finds no complete +/// manifest and `on_incomplete` is [`OnIncomplete::Fail`]. The dead-owner +/// duty runs it on the configured store's rows with `Fail`; `backup sync +/// --move-store` on the old root's rows with `Leave`, probing the +/// configured store, to which the copied prefix belongs: a copy without a +/// complete backup proves nothing about the old root's prefix (a copy's +/// order is arbitrary), so the move writes no FAILED. +pub(super) async fn resolve_dead_creating_row( + storage: &S, + store: StoreCalls<'_>, + row: &BackupRow, + stale_job_after: Duration, + on_incomplete: OnIncomplete, +) -> CreatingOutcome { + let claim = match storage.claim_dead_backup(&row.key(), stale_job_after).await { + Ok(Some(claim)) => claim, + Ok(None) => return CreatingOutcome::NotHandedOver, + Err(e) => { + tracing::error!( + "could not test whether the owner of CREATING backup {} (account {}) is dead; \ + the next cycle retries: {e}", + row.backup_arn, + row.account_id + ); + return CreatingOutcome::Failed(format!( + "could not test whether its owner is dead: {e}" + )); + } + }; + let arn = match BackupArn::parse(&row.backup_arn) { + Ok(arn) => arn, + Err(e) => { + tracing::error!( + "CREATING row {} carries an ARN this build cannot parse; left as it is: {e}", + row.backup_arn + ); + return CreatingOutcome::Failed(format!("the row's ARN does not parse: {e}")); + } + }; + let (to, fields) = match probe_manifest(store, &arn).await { + Ok(Probe::Complete(fields)) => (JobStatus::Available, fields), + Ok(Probe::Incomplete(why)) if on_incomplete == OnIncomplete::Leave => { + drop(claim); + return CreatingOutcome::Incomplete(why); + } + Ok(Probe::Incomplete(why)) => { + tracing::warn!( + "backup {} (account {}): its owner is dead and the store holds no complete \ + manifest ({why}); the row is FAILED and its prefix is deleted once the row's \ + heartbeat is stale", + row.backup_arn, + row.account_id + ); + ( + JobStatus::Failed, + StatusFields { + failure_reason: Some(OWNER_DEAD_NO_MANIFEST.to_owned()), + completed: true, + ..StatusFields::default() + }, + ) + } + Err(e) => { + tracing::error!( + "backup {} (account {}): its owner is dead and the store could not be probed \ + for its manifest; the next cycle retries: {e}", + row.backup_arn, + row.account_id + ); + return CreatingOutcome::Failed(format!( + "the store could not be probed for its manifest: {e}" + )); + } + }; + let outcome = match write_status(&*claim, JobStatus::Creating, to, &fields).await { + Written::Matched => { + tracing::info!( + "backup {} (account {}): its owner was dead; the row is now {to}", + row.backup_arn, + row.account_id + ); + CreatingOutcome::Resolved(match to { + JobStatus::Available => BackupStatus::Available, + _ => BackupStatus::Failed, + }) + } + Written::NotMatched(why) => { + tracing::info!( + "backup {} was no longer this cycle's to resolve ({why}); left to its owner", + row.backup_arn + ); + CreatingOutcome::LeftToOwner(why) + } + Written::Error(e) => { + tracing::error!( "backup {} (account {}): the {to} write failed; the next cycle retries: {e}", row.backup_arn, row.account_id - ), + ); + CreatingOutcome::Failed(format!("the {to} write failed: {e}")) } - drop(claim); - } - Ok((resolved, left_to_owner)) + }; + drop(claim); + outcome } /// The configured store as the reconciler's duties call it: each failed @@ -1431,16 +1552,22 @@ impl StoreCalls<'_> { } } - async fn delete_prefix(self, prefix: &str) -> Result { + /// The prefix deleted, no removal started at or after `deadline` + /// ([`BackupStore::delete_prefix_before`]). + async fn delete_prefix_before( + self, + prefix: &str, + deadline: tokio::time::Instant, + ) -> Result { self.store - .delete_prefix(prefix) + .delete_prefix_before(prefix, deadline) .await .map_err(|e| self.failed("delete_prefix", e)) } } /// What the manifest probe found under a dead owner's prefix. -enum Probe { +pub(super) enum Probe { /// A complete, verified manifest: the AVAILABLE write's fields. Complete(StatusFields), /// No manifest, or one the reader refuses, or a data file missing or of @@ -1452,9 +1579,14 @@ enum Probe { /// [`verify_prefix`], with the manifest's ARN required to name the row's /// backup (its partition included; the region may differ, as it does for a /// backup `backup sync` adopted from a manifest another region's server -/// wrote, `BackupArn::names_same_backup`) before the prefix is listed. A -/// store error other than an absent object is the `Err`. -async fn probe_manifest(store: StoreCalls<'_>, arn: &BackupArn) -> Result { +/// wrote, `BackupArn::names_same_backup`) before the prefix is listed. +/// `backup sync --move-store` probes each AVAILABLE row of the old root with +/// it in the configured store. A store error other than an absent object is +/// the `Err`. +pub(super) async fn probe_manifest( + store: StoreCalls<'_>, + arn: &BackupArn, +) -> Result { let prefix = arn.store_prefix(); let manifest = match read_verified_manifest(store, &prefix).await? { Verified::Complete(manifest) => manifest, @@ -1662,13 +1794,37 @@ pub(super) async fn read_object( Ok(ReadObject::Bytes(bytes)) } +/// How long one FAILED-prefix or DELETED-prefix deletion, or one tombstone +/// removal, may run from the duty's read of the maintenance claim +/// ([`Duty::fenced`]), counted from before that read was sent: a read +/// answered after it starts nothing, the prefix deletion checks it inside +/// the store before each removal (the filesystem walk before each entry, +/// the S3 loop before each request: `delete_prefix_before`), and a call +/// still running at it is dropped. The duty then stops for this cycle, the +/// row left as it is for the next cycle to continue. It is the bound an S3 +/// store's calls run under ([`super::S3_CHECK_TIMEOUT`], the larger of the +/// two kinds' bounds), so a deletion of an S3 prefix stopped at it is +/// dropped before the SDK starts a retry of a request that stalled +/// ([`extenddb_backup_store::S3_ATTEMPT_TIMEOUT`]). A maintenance command +/// that removes FAILED or DELETED rows beside servers waits this bound plus +/// one store call's after it takes the claim +/// ([`super::store_maintenance::FORGET_WAIT`]), so what remains of a +/// deletion a duty began before the claim when the command removes a row is +/// the step in flight at the deadline (one removal, or one S3 request). +/// The bound assumes the duty's process keeps running: one paused past the +/// wait between its last check and a call already sent is outside it, as +/// it is for every lease in the design. +pub const DUTY_DELETION_BOUND: Duration = super::S3_CHECK_TIMEOUT; + /// What the duties that delete a prefix or remove a tombstone act through: -/// the catalog, the store's calls, and the deletion fence each runs right -/// before its deletion or removal. +/// the catalog, the store's calls, the deletion fence each runs right +/// before its deletion or removal, and the dead-owner window the +/// maintenance claim is read with. struct Duty<'a, S: ?Sized> { storage: &'a S, store: StoreCalls<'a>, fence: &'a StoreFence, + stale_job_after: Duration, } impl Clone for Duty<'_, S> { @@ -1679,19 +1835,142 @@ impl Clone for Duty<'_, S> { impl Copy for Duty<'_, S> {} +/// What [`Duty::fenced`] decided for one row. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Fenced { + /// Act, ending by this instant. + Until(tokio::time::Instant), + /// Act on nothing for this row; the duty goes on to the next. + Skip, + /// The claim's read answered after the deadline: the duty stops for + /// this cycle. + OutOfTime, +} + +/// What one deletion or removal of a duty came to. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Acted { + /// The prefix was deleted, or the row removed. + Done, + /// Nothing was done for this row; the duty goes on to the next. + Skipped, + /// The call did not end within [`DUTY_DELETION_BOUND`]: the duty stops + /// for this cycle. + OutOfTime, +} + impl Duty<'_, S> { - /// The fence before `row`'s deletion or removal: `true` when it passed; - /// a refusal is logged and acts on nothing. - async fn fenced(self, row: &BackupRow, status: &str, what: &str) -> bool { - match self.fence.check_catalog(self.storage).await { - Ok(()) => true, - Err(problem) => { + /// The fence before `row`'s deletion or removal, the row read once + /// more, then the maintenance claim read again: the instant the call + /// must end by ([`DUTY_DELETION_BOUND`] after the claim's read) when the + /// fence passed, the row is still of the status and the store the duty + /// listed it with, and no maintenance command holds the claim + /// ([`Fenced::Until`]); [`Fenced::OutOfTime`] when the claim's read + /// answered at or after that instant; [`Fenced::Skip`] otherwise, logged, + /// acting on nothing. The cycle reads the claim once, + /// at its start, so a duty that listed the row before a maintenance + /// command took the claim may reach its row after the command has + /// removed it (`backup sync --forget-backup`, `--forget-store`) or moved + /// it to another store (`--move-store`): the re-read skips a row that is + /// gone, of another status or of another store, and the claim's read + /// after it starts no deletion once a command holds the claim. A + /// deletion that read the claim free just before a command took it ends + /// by the bound ([`DUTY_DELETION_BOUND`], counted from before the read + /// was sent, so a read answered late starts nothing), and a command that + /// removes FAILED or DELETED rows waits the bound and one store call + /// before its first removal (`store_maintenance::FORGET_WAIT`). A failed + /// read is left for the next cycle. + async fn fenced(self, row: &BackupRow, status: &str, what: &str) -> Fenced { + if let Err(problem) = self.fence.check_catalog(self.storage).await { + tracing::error!( + "did not {what} of {status} backup {} (account {}): {problem}", + row.backup_arn, + row.account_id + ); + return Fenced::Skip; + } + let arn = match BackupArn::parse(&row.backup_arn) { + Ok(arn) => arn, + Err(e) => { + tracing::error!( + "did not {what} of {status} backup {}: its ARN does not parse: {e}", + row.backup_arn + ); + return Fenced::Skip; + } + }; + match self.storage.read_backup_row(&arn).await { + Ok(Some(now)) if now.status == row.status && now.store == row.store => {} + Ok(Some(now)) => { + tracing::info!( + "did not {what} of {status} backup {} (account {}): the row is now {} in \ + the {} store {} (store id {}); left to whoever changed it", + row.backup_arn, + row.account_id, + now.status, + now.store.store_kind, + now.store.store_root, + now.store.store_id + ); + return Fenced::Skip; + } + Ok(None) => { + tracing::info!( + "did not {what} of {status} backup {} (account {}): the row is gone (removed \ + since the duty listed it); its objects are left", + row.backup_arn, + row.account_id + ); + return Fenced::Skip; + } + Err(e) => { tracing::error!( - "did not {what} of {status} backup {} (account {}): {problem}", + "did not {what} of {status} backup {} (account {}): the row could not be read \ + again; the next cycle retries: {e}", row.backup_arn, row.account_id ); - false + return Fenced::Skip; + } + } + // The bound counts from before the claim's read: a command that + // takes the claim after the read was answered waits out the bound, + // and a read answered after it (a slow catalog) acts on nothing. + let deadline = tokio::time::Instant::now() + DUTY_DELETION_BOUND; + match self + .storage + .check_maintenance_claim(self.stale_job_after) + .await + { + Ok(false) if tokio::time::Instant::now() >= deadline => { + tracing::warn!( + "did not {what} of {status} backup {} (account {}): the maintenance claim's \ + read answered after the {} s bound; the duty stops for this cycle and the \ + next cycle continues", + row.backup_arn, + row.account_id, + DUTY_DELETION_BOUND.as_secs() + ); + Fenced::OutOfTime + } + Ok(false) => Fenced::Until(deadline), + Ok(true) => { + tracing::info!( + "did not {what} of {status} backup {} (account {}): a maintenance claim is \ + held now; the next cycle without one acts on it", + row.backup_arn, + row.account_id + ); + Fenced::Skip + } + Err(e) => { + tracing::error!( + "did not {what} of {status} backup {} (account {}): the maintenance claim \ + could not be read again; the next cycle retries: {e}", + row.backup_arn, + row.account_id + ); + Fenced::Skip } } } @@ -1716,8 +1995,10 @@ async fn delete_stale_failed_prefixes( .await?; let mut deleted = Vec::new(); for row in rows { - if delete_prefix_if_listed(duty, &row, "FAILED").await { - deleted.push(row.backup_arn); + match delete_prefix_if_listed(duty, &row, "FAILED").await { + Acted::Done => deleted.push(row.backup_arn), + Acted::Skipped => {} + Acted::OutOfTime => break, } } Ok(deleted) @@ -1736,21 +2017,24 @@ async fn sweep_deleted_prefixes( .await?; let mut swept = Vec::new(); for row in rows { - if delete_prefix_if_listed(duty, &row, "DELETED").await { - swept.push(row.backup_arn); + match delete_prefix_if_listed(duty, &row, "DELETED").await { + Acted::Done => swept.push(row.backup_arn), + Acted::Skipped => {} + Acted::OutOfTime => break, } } Ok(swept) } -/// Delete `row`'s prefix when it lists objects, once the fence passed; -/// returns whether a deletion ran. A store failure or the fence's refusal -/// is logged and left for the next cycle. +/// Delete `row`'s prefix when it lists objects, once [`Duty::fenced`] +/// passed, within [`DUTY_DELETION_BOUND`]. A store failure or a refusal is +/// logged and left for the next cycle; a deletion that outlasts the bound +/// is dropped ([`Acted::OutOfTime`]). async fn delete_prefix_if_listed( duty: Duty<'_, S>, row: &BackupRow, status: &str, -) -> bool { +) -> Acted { let store = duty.store; let prefix = match BackupArn::parse(&row.backup_arn) { Ok(arn) => arn.store_prefix(), @@ -1759,47 +2043,77 @@ async fn delete_prefix_if_listed( "{status} row {} carries an ARN this build cannot parse; left as it is: {e}", row.backup_arn ); - return false; + return Acted::Skipped; } }; match store.prefix_has_objects(&prefix).await { Ok(true) => {} - Ok(false) => return false, + Ok(false) => return Acted::Skipped, Err(e) => { tracing::error!( "could not list the prefix of {status} backup {}; the next cycle retries: {e}", row.backup_arn ); - return false; + return Acted::Skipped; } } - if !duty.fenced(row, status, "delete the prefix").await { - return false; - } - match store.delete_prefix(&prefix).await { - Ok(n) => { + let deadline = match duty.fenced(row, status, "delete the prefix").await { + Fenced::Until(deadline) => deadline, + Fenced::Skip => return Acted::Skipped, + Fenced::OutOfTime => return Acted::OutOfTime, + }; + // The store checks the deadline before each removal, so a call that + // resumes past it (a paused process) starts no other; the timeout ends + // a step that does not answer. + match tokio::time::timeout_at(deadline, store.delete_prefix_before(&prefix, deadline)).await { + Ok(Ok(done)) if done.finished => { tracing::info!( - "deleted {n} object(s) left under the prefix of {status} backup {} (account {})", + "deleted {} object(s) left under the prefix of {status} backup {} (account {})", + done.deleted, row.backup_arn, row.account_id ); - true + Acted::Done } - Err(e) => { + Ok(Ok(done)) => { + tracing::warn!( + "the deletion of the prefix of {status} backup {} (account {}) reached its {} s \ + bound after {} object(s) and stopped; the duty stops for this cycle and the \ + next cycle continues", + row.backup_arn, + row.account_id, + DUTY_DELETION_BOUND.as_secs(), + done.deleted + ); + Acted::OutOfTime + } + Ok(Err(e)) => { tracing::error!( "could not delete the prefix of {status} backup {} (account {}); the next cycle \ retries: {e}", row.backup_arn, row.account_id ); - false + Acted::Skipped + } + Err(_) => { + tracing::warn!( + "the deletion of the prefix of {status} backup {} (account {}) did not end \ + within {} s and was stopped; the duty stops for this cycle and the next cycle \ + continues", + row.backup_arn, + row.account_id, + DUTY_DELETION_BOUND.as_secs() + ); + Acted::OutOfTime } } } /// The tombstone duty (module doc): FAILED and DELETED rows whose /// `completed_at` is older than `retention` and whose prefix lists nothing -/// are removed. +/// are removed, each removal within [`DUTY_DELETION_BOUND`] of the claim's +/// read; one that outlasts it stops the duty for this cycle. async fn remove_tombstones( duty: Duty<'_, S>, identity: &StoreIdentity, @@ -1838,14 +2152,22 @@ async fn remove_tombstones( } // The empty listing is a fact about this catalog's root only // while the root is this catalog's. - if !duty + let deadline = match duty .fenced(&row, &status.to_string(), "remove the row") .await { - continue; - } - match storage.remove_backup_row(&row.key()).await { - Ok(true) => { + Fenced::Until(deadline) => deadline, + Fenced::Skip => continue, + Fenced::OutOfTime => return Ok(removed), + }; + // Conditioned on the store the duty listed the row with: a row + // `--move-store` moved since is the configured store's to + // finish. + let key = row.key(); + let removal = + storage.remove_backup_row(&key, row.store.store_kind, &row.store.store_root); + match tokio::time::timeout_at(deadline, removal).await { + Ok(Ok(true)) => { tracing::info!( "removed the {status} row of backup {} (account {}): its prefix is empty \ and its retention has passed", @@ -1854,13 +2176,24 @@ async fn remove_tombstones( ); removed.push(row.backup_arn); } - Ok(false) => {} - Err(e) => tracing::error!( + Ok(Ok(false)) => {} + Ok(Err(e)) => tracing::error!( "could not remove the {status} row of backup {} (account {}); the next cycle \ retries: {e}", row.backup_arn, row.account_id ), + Err(_) => { + tracing::warn!( + "the removal of the {status} row of backup {} (account {}) did not end \ + within {} s and was stopped; the duty stops for this cycle and the next \ + cycle continues", + row.backup_arn, + row.account_id, + DUTY_DELETION_BOUND.as_secs() + ); + return Ok(removed); + } } } } diff --git a/crates/engine/src/backup/restore.rs b/crates/engine/src/backup/restore.rs index 2615ab367..31cfedd58 100644 --- a/crates/engine/src/backup/restore.rs +++ b/crates/engine/src/backup/restore.rs @@ -23,8 +23,10 @@ //! the detached task ([`start`]) whose answer it awaits for at most //! [`RESTORE_REGISTRATION_TIMEOUT`]; past that it answers //! `InternalServerError` while the task goes on (the target then appears -//! through DescribeTable). The task registers the target and its RUNNING -//! record in one write (`claim_restore`, under the backup row's lock), sends +//! through DescribeTable). The task reads the maintenance claim (held, the +//! request is refused with `LimitExceededException` and nothing is +//! registered, so no record lands while a command holds the claim), then +//! registers the target and its RUNNING record in one write (`claim_restore`, under the backup row's lock), sends //! the job's first beat through the claim (the registration's own //! `heartbeat_at` stamp can be old: the design's 4(e) states each backend's //! clock, PostgreSQL's `now()` being the transaction's start, so a @@ -145,8 +147,9 @@ pub const RESTORE_REGISTRATION_TIMEOUT: Duration = Duration::from_secs(60); pub const TARGET_NOT_EMPTY: &str = "target_not_empty"; /// The reason the reconciler writes on a RUNNING record whose store is not -/// the configured one. -pub const OTHER_STORE: &str = "the restore reads a store other than the configured one; configure that store in [backup.store] and restore again"; +/// the configured one. `--move-store` re-points the backup's rows, not the +/// record, so the restore is run again after the move. +pub const OTHER_STORE: &str = "the restore reads a store other than the configured one; configure that store in [backup.store], or copy its tree under the configured root and move the backup's rows to the configured store with `extenddb backup sync --move-store `, and restore again"; /// What the request resolved before it spawned the task. pub(crate) struct RestoreRequestParts { @@ -452,6 +455,27 @@ pub(crate) async fn start( spool_host: spool_host.clone(), owner_generation: UNOWNED_GENERATION, }; + // The maintenance claim immediately before the registration: the + // request's check ran before the manifest reads and the permit, and a + // command that took the claim since gets no record of this restore. A + // claim taken in the gap between this read and the registration is met + // by the check after it, below, and by the command's wait. + let refused = match storage + .check_maintenance_claim(env.settings.stale_job_after) + .await + { + Ok(false) => None, + Ok(true) => Some(maintenance_held()), + Err(e) => Some(storage_err_to_dynamo(e)), + }; + if let Some(error) = refused { + drop(permit); + if let Some(registration) = registration { + registration.finish(Err(error.clone())); + } + let _ = reply.send(Err(error)); + return; + } let registered = storage .claim_restore(&account_id, input, ®istration_row) .await; diff --git a/crates/engine/src/backup/runtime.rs b/crates/engine/src/backup/runtime.rs index 7ee653fa6..3b10269e8 100644 --- a/crates/engine/src/backup/runtime.rs +++ b/crates/engine/src/backup/runtime.rs @@ -1084,6 +1084,51 @@ pub async fn read_identity_bounded( read } +/// The store's check as a command that writes nothing runs it after its +/// removals (`backup sync --forget-backup`): [`store_identity::read_identity`] +/// (no identity file is created, and `validate`, which writes a probe file, +/// does not run), the file's id compared with `known` when the command read +/// one when it opened the store (a root replaced since is +/// [`StoreProblem::IdMismatch`], as [`store_identity::recheck`] has it), then +/// the manifest probe, every store call under +/// [`ConfiguredStore::check_timeout`]. `Ok(None)` for a root with no file +/// and no rows for it, left as it is: a server's next check creates the +/// file. +/// +/// # Errors +/// +/// The problem that would make the store unhealthy +/// ([`store_identity`]'s module doc), or the bound. +pub async fn check_configured_store_read_only( + store: &ConfiguredStore, + catalog: &C, + known: Option<&StoreIdentity>, +) -> Result, StoreProblem> { + let via = store.calls("check"); + let outcome = async { + let Some(identity) = + store_identity::read_identity(&via, store.kind, &store.root, catalog).await? + else { + return Ok(None); + }; + if let Some(known) = known + && known.store_id != identity.store_id + { + return Err(StoreProblem::IdMismatch { + file_id: identity.store_id, + row_ids: vec![known.store_id.clone()], + }); + } + store_identity::manifest_probe(&via, &identity, catalog).await?; + Ok(Some(identity)) + } + .await; + if via.timed_out() { + return Err(via.did_not_finish_problem()); + } + outcome +} + /// The configured store as the checks, a command's identity read, the /// deletion fence and the deletions call it (module doc): each call runs under `bound`, and a call /// that does not finish is a transport failure naming `what` and the bound. diff --git a/crates/engine/src/backup/store_identity.rs b/crates/engine/src/backup/store_identity.rs index 45ca0d8d7..b73cda376 100644 --- a/crates/engine/src/backup/store_identity.rs +++ b/crates/engine/src/backup/store_identity.rs @@ -205,6 +205,17 @@ pub enum StoreProblem { } impl StoreProblem { + /// The row ids an identity mismatch is about (`IdMismatch`'s row ids, + /// `MultipleRowIds`' ids); empty for every other problem. + #[must_use] + pub fn mismatch_row_ids(&self) -> &[String] { + match self { + Self::IdMismatch { row_ids, .. } => row_ids, + Self::MultipleRowIds(ids) => ids, + _ => &[], + } + } + /// The problem's category, one or two words, for an API refusal. #[must_use] pub fn category(&self) -> &'static str { diff --git a/crates/engine/src/backup/store_maintenance.rs b/crates/engine/src/backup/store_maintenance.rs new file mode 100644 index 000000000..d7148f512 --- /dev/null +++ b/crates/engine/src/backup/store_maintenance.rs @@ -0,0 +1,1837 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! The options of `backup sync` that act on the rows of a root other than +//! the configured one: `--move-store ` ([`move_store`]), +//! `--forget-store ` ([`forget_store`]) and `--forget-backup ` +//! ([`forget_backups`]), and plain sync's report of the rows of the other +//! stores ([`other_stores`]). Each is written against the catalog's trait +//! and the configured store, so the engine's tests run them against a +//! double; the command takes the maintenance claim, waits for running jobs +//! and prints what they return. +//! +//! A root is given as its label, as the catalog stores it and as +//! `backup sync` prints it (the canonical directory of a filesystem store, +//! `s3:///` for S3 with its `?endpoint=` suffix when the +//! store has one), and matches byte for byte. [`locate_root`] finds the +//! kind of the rows under it (an S3 label starts with `s3://` and a +//! filesystem label is an absolute path, so the two never meet) and the +//! store ids the catalog records for it, which the command's wait reads +//! rows of besides the configured store's; a root with no rows is refused +//! with the stores the catalog knows ([`RootRefusal::NoRows`]), the help an +//! operator needs after a label that differs by a trailing `/` or by its +//! `?endpoint=` suffix. Both options refuse the configured root itself. +//! +//! [`move_store`] is the way a tree copied to a new root keeps its rows: +//! the operator copies the old root's tree under the configured root +//! first. It reads the old root's rows (each row's +//! `source_table_details_json` is dropped as it is read) and the configured +//! root's (account, backup id) pairs, so a failed read stops the move +//! before anything changed. The rows are grouped by (account, backup id), +//! and every old-root row of a group of more than one (with a row of the +//! configured root, or with another spelling of its ARN among the moved +//! rows, which only Cassandra holds) stops the move and is set aside before +//! any row is checked, so none of them is checked further: on Cassandra two +//! spellings of an ARN share one prefix, which a DELETING row's finished +//! deletion would otherwise empty. Every other row is checked against the +//! configured store, the store the copy is in, and every row that stops +//! the move is named in one pass: +//! - A CREATING row whose owner is dead is resolved AVAILABLE when the copy +//! holds its complete backup (`reconciler::resolve_dead_creating_row` +//! with `OnIncomplete::Leave`: the takeover on the catalog's clock, the +//! manifest probe of the configured store, the AVAILABLE write). Any +//! other probe result stops the move and writes no status: a copy without +//! the complete backup proves nothing about the old root's prefix (a +//! copy's order is arbitrary, so a copy without the manifest may come +//! from a complete prefix), and a FAILED row would have the reconciler +//! delete the copy, and any server still on the old root delete the +//! original. The row is left CREATING for a reconciler on the old root +//! to resolve there, or for `--forget-backup`. A row the backend does not +//! hand over has a live owner, or another taker, and stops the move. +//! - A DELETING row whose copied prefix holds a verified manifest +//! (`reconciler::verify_prefix`) stops the move. Otherwise its deletion +//! is finished in the configured store behind the deletion fence +//! (`finish_backup_delete`), which deletes what the copy brought of it +//! and sets the row DELETED: that is the deletion the row's DeleteBackup +//! asked for. +//! - A DELETED or FAILED row whose copied prefix holds a verified manifest +//! stops the move, naming `--forget-backup`: moving the tombstone would +//! have the reconciler delete a complete backup. Otherwise it moves, and +//! the reconciler of the configured store cleans its leftovers there. +//! - An AVAILABLE row is probed in the configured store +//! (`reconciler::probe_manifest`: the manifest and its sidecar, the +//! digest, the reader's acceptance, the row's full ARN, every data file +//! at its recorded size), and the manifest's size and item total must be +//! the row's; anything else stops the move with the reason. +//! +//! The AVAILABLE resolutions and the finished deletions are kept when the +//! move stops; each is idempotent, and the next run sees the statuses they +//! wrote. With no row stopping, the fence re-reads the configured root's +//! identity file once (no row is moved into a root another catalog adopted +//! since the check), then `rewrite_store` moves the rows in calls of +//! [`REWRITE_SLICE`] keys, or of one key on a backend that rewrites key by +//! key (Cassandra), each row under the condition that it is still of the +//! old kind and root and not CREATING. Before each call the signal and the +//! claim's local state ([`HeldClaim::lost`], no write) are read, and before +//! each [`REWRITE_SLICE`] keys the claim is confirmed with a beat; then the +//! old root is listed once more: a row still +//! there (a CreateBackup of a server still configured with the old root +//! whose row landed after the wait, a row `rewrite_store` did not count, a +//! rewrite that stopped part way, the keys after a stop or a lost claim) is +//! reported and the command exits 1; a row the old root no longer lists is +//! read by its ARN (one read per such key, [`MOVED_ROW_READS`] at a time, +//! not a listing of the configured store) and counts as moved when it +//! names the configured store. Every row moved, resolved or finished is +//! logged under the `extenddb::audit` target. No object is copied by the +//! move, and none of the old root is touched. +//! +//! [`forget_store`] and [`forget_backups`] remove rows outright, leaving no +//! DELETED tombstone; the removals delete no object and read no store, so +//! they run on a store in any state and with none configured. A deletion +//! duty starts no deletion once it reads the maintenance claim held +//! (`reconciler`'s `Duty::fenced`), each deletion it started before ends by +//! [`super::reconciler::DUTY_DELETION_BOUND`] from its read of the claim +//! (the store checks the deadline before each removal), and beside servers +//! the command waits [`FORGET_WAIT`] from the claim before its first +//! removal of a FAILED or DELETED row (on SQLite the serve lock excludes +//! every server, and nothing waits). What a duty can still do to a removed +//! row's bytes is the step in flight at its deadline (one removal, or one +//! S3 request), and only a server process paused past the wait between its +//! last check and that step runs it after the removal: the assumption every +//! lease in the design shares. The exception is a DELETING row of the +//! configured store: the command finishes its deletion behind the fence +//! (`finish_backup_delete`, the deletion its DeleteBackup asked for) before +//! it removes the row, which needs the store's check, run at the command's +//! opening and writing nothing, to have passed; with the store unhealthy or +//! its id unknown the row is refused ([`Forgotten::DeletionPending`]), and +//! when the check refused on an identity mismatch that the row's own store +//! id takes part in, the refusal says so and names the way out, the same +//! command with no store configured ([`Forgotten::DeletionBehindMismatch`]). +//! On SQLite the command's own dead-owner pass may finish it first, under the +//! claim, and the removal of its DELETED row says so. A DELETING row of +//! another store keeps its bytes unless a server configured with that store +//! finishes the deletion (its reconciler, under the claim: the deletion the +//! row's DeleteBackup asked for), and a sync of that root after its return +//! with the bytes adopts the backup again. While the configured root has no +//! `extenddb-store.json` and the catalog holds rows for it (an unmounted +//! volume), every row of that root is refused +//! ([`Forgotten::RootWithoutFile`]): a server's next check would write a +//! new identity file at the empty root once its last row is gone. Each +//! status has its removal ([`forget_row`]): AVAILABLE through +//! `forget_available_backup_row` (the row's lock, the status check and the +//! in-use rule in one step: a row a CREATING restore target reads is left); +//! DELETED and FAILED through `remove_backup_row`; DELETING through +//! `set_backup_deleted` then `remove_backup_row`, the DELETED state between +//! the two met by no duty that starts after the claim, and by none that +//! listed the row before it (the row was DELETING then); CREATING through +//! `remove_dead_creating_backup_row` on the catalog's clock, a row left +//! CREATING having a live owner. A FAILED row +//! whose heartbeat is within the owner window (a job that has not yet seen +//! its claim lost may still write under the prefix) is left as having a +//! live owner. A removal that finds another status re-reads the row once +//! and applies that status's removal. `--forget-backup` takes the ARN as +//! the row carries it: a row under another spelling of the ARN (the +//! server's region) is named and left, an earlier release's in-catalog +//! backup is reported as one, with nothing changed, and an ARN no row and +//! no in-catalog backup carries counts as done (a run given again after a +//! stop goes past the ARNs it removed). The command checks every ARN's +//! form ([`check_arns`]) before it takes the claim. After `--forget-store`'s +//! removals, the root's status entries are swept under the claim +//! (`remove_orphan_status_entries`, each verified against its row; only +//! Cassandra has them): an entry whose row is gone or names another store +//! (a deleter whose entry batch landed after a move of its row) is +//! deleted, and one whose row is of the root under another status (a +//! status change whose entry batch failed, or whose writer died between +//! its row and its batch) is repaired, each counted; the store ids the +//! catalog still records for the root are then read once more and +//! reported. +//! +//! Every option confirms the maintenance claim with a beat of its own +//! ([`HeldClaim::recheck`]) right before each step that cannot be undone, +//! as sync does before its insert: the move before a dead CREATING row's +//! resolution, before a DELETING row's finished deletion and before each +//! [`REWRITE_SLICE`] keys of the rewrite; the forget options before each row's removal, +//! before a DELETING row's DELETED write or finished deletion and again +//! before the removal of the DELETED row it leaves, and `--forget-store` before the +//! sweep of the status entries. A claim found lost there stops the run with +//! the row as the steps before it left it, and the stop names the step. On +//! MongoDB and Cassandra, which take a claim over on the heartbeat's age +//! alone, a command stopped (SIGSTOP, a paused VM) past +//! `stale_job_after_secs` can lose its claim to another command; it then +//! stops before its next such step. What remains is a stop after the +//! re-check confirmed the claim and before the step's own call reached the +//! catalog or the store, or with that call already sent: the call lands +//! when the process resumes. The move's rewrite reads the claim's local +//! state before each call as well, and a command resumed past +//! `stale_job_after_secs` reads it lost at once (no beat confirmed it +//! within two periods, and every stale window is at least three), so what +//! it can write after the loss is the one call it had sent or was about to +//! send when it stopped: one key on Cassandra (its few requests), one +//! statement of up to [`REWRITE_SLICE`] keys on MongoDB. PostgreSQL's claim +//! is the session's advisory lock and SQLite's the serve lock, which no +//! other command takes while the holder's process lives. +//! +//! [`other_stores`] is the report plain `backup sync` prints after its +//! listing: per store other than the configured one, the rows per status +//! (`summarize_store_roots`; on Cassandra a count of status entries, +//! checked against the root's listing, and what no row of its status +//! stands behind is reported as status entries), each CREATING and +//! DELETING row by ARN (the +//! rows 4(c) leaves alone), so the operator can move them with +//! `--move-store` or drop them with `--forget-store`. + +use std::collections::{HashMap, HashSet}; +use std::time::Duration; + +use extenddb_core::backup_format::BackupArn; +use extenddb_storage::backup_catalog::{ + BackupKey, BackupRow, BackupStatus, ForgetRow, StaleClock, StatusEntrySweep, StoreIdentity, + StoreKind, StoreRootCount, is_backup_not_found, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::{BackupCatalog, LegacyBackupReader}; +use futures::{StreamExt as _, TryStreamExt}; + +use super::ConfiguredStore; +use super::maintenance::HeldClaim; +use super::reconciler::{ + CreatingOutcome, OnIncomplete, Probe, StoreCalls, Verified, probe_manifest, + resolve_dead_creating_row, verify_prefix, +}; +use super::sync::Stop; + +/// A row named in a report: its status, ARN and account. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct NamedRow { + pub status: BackupStatus, + pub backup_arn: String, + pub account_id: String, +} + +impl NamedRow { + fn of(row: &BackupRow) -> Self { + Self { + status: row.status, + backup_arn: row.backup_arn.clone(), + account_id: row.account_id.clone(), + } + } +} + +// --- the report of other stores ------------------------------------------ + +/// The rows the catalog holds for one store other than the configured one. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OtherStore { + /// The store: kind, root and id. + pub store: StoreIdentity, + /// The rows per status, each status with at least one row, in + /// [`BackupStatus::ALL`]'s order. + pub counts: Vec<(BackupStatus, u64)>, + /// The store's CREATING and DELETING rows, in that order and each in + /// listing order. + pub unresolved: Vec, + /// Per status, the status entries no listed row of that status stands + /// behind, each status with at least one, in [`BackupStatus::ALL`]'s + /// order: on a backend whose summary counts status entries + /// (Cassandra), the summary's count less the store's rows of that + /// status in the root's listing (`list_backup_rows_by_root`, which + /// lists a row whose entry is under an earlier status under its own). + /// Empty elsewhere. + pub entries: Vec<(BackupStatus, u64)>, +} + +/// The counts of `summarize_store_roots` (in its order) grouped by store, +/// `except` left out; no row is named. +#[must_use] +pub fn group_counts(counts: &[StoreRootCount], except: Option<&StoreIdentity>) -> Vec { + let mut stores: Vec = Vec::new(); + for count in counts { + if except == Some(&count.store) { + continue; + } + match stores.last_mut() { + Some(last) if last.store == count.store => { + last.counts.push((count.status, count.rows)); + } + _ => stores.push(OtherStore { + store: count.store.clone(), + counts: vec![(count.status, count.rows)], + unresolved: Vec::new(), + entries: Vec::new(), + }), + } + } + stores +} + +/// Every store other than `configured` that the catalog holds rows for +/// (module doc), in [`StoreRootCount::sort`]'s order. A store of the +/// configured root under another id is another store too. On a backend +/// whose summary counts status entries (Cassandra), the store's rows are +/// read from its root's listing (`list_backup_rows_by_root`, kept to the +/// store's id), which lists each row under its own status whatever +/// partition its entry is in: `counts` holds those rows, and +/// [`OtherStore::entries`] the entries no row of their status stands +/// behind. +/// +/// # Errors +/// +/// The backend's error for the summary or a listing. +pub async fn other_stores( + storage: &S, + configured: &StoreIdentity, +) -> Result, StorageError> { + let counts = storage.summarize_store_roots().await?; + let mut stores = group_counts(&counts, Some(configured)); + if storage.summary_counts_status_entries() { + for other in &mut stores { + // The root's listing, which reads every status partition, so a + // row whose entry is under an earlier status counts under its + // own status; kept to the store's id. + let (rows, unresolved) = storage + .list_backup_rows_by_root(other.store.store_kind, &other.store.store_root, None) + .try_fold( + (Vec::<(BackupStatus, u64)>::new(), Vec::new()), + |(mut rows, mut unresolved), row| { + if row.store == other.store { + match rows.iter_mut().find(|(s, _)| *s == row.status) { + Some((_, n)) => *n += 1, + None => rows.push((row.status, 1)), + } + if matches!(row.status, BackupStatus::Creating | BackupStatus::Deleting) + { + unresolved.push(NamedRow::of(&row)); + } + } + std::future::ready(Ok((rows, unresolved))) + }, + ) + .await?; + for (status, entries) in &other.counts { + let listed = rows + .iter() + .find(|(s, _)| s == status) + .map_or(0, |(_, n)| *n); + if *entries > listed { + other.entries.push((*status, entries - listed)); + } + } + other.counts = BackupStatus::ALL + .iter() + .filter_map(|status| rows.iter().find(|(s, _)| s == status).copied()) + .collect(); + other.unresolved = [BackupStatus::Creating, BackupStatus::Deleting] + .iter() + .flat_map(|status| unresolved.iter().filter(|r| r.status == *status).cloned()) + .collect(); + } + return Ok(stores); + } + for other in &mut stores { + for status in [BackupStatus::Creating, BackupStatus::Deleting] { + if !other.counts.iter().any(|(s, n)| *s == status && *n > 0) { + continue; + } + let mut rows = storage.list_backup_rows(Some(status), &other.store); + while let Some(row) = rows.try_next().await? { + other.unresolved.push(NamedRow::of(&row)); + } + } + } + Ok(stores) +} + +// --- the root an option names -------------------------------------------- + +/// The root an option names, as the catalog records it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LocatedRoot { + pub kind: StoreKind, + pub root: String, + /// The store ids the catalog records for the root (`read_store_ids`). + pub store_ids: Vec, +} + +impl LocatedRoot { + /// The root's stores, one per id: what the command's wait reads rows + /// of besides the configured store's. + #[must_use] + pub fn identities(&self) -> Vec { + self.store_ids + .iter() + .map(|id| StoreIdentity { + store_kind: self.kind, + store_root: self.root.clone(), + store_id: id.clone(), + }) + .collect() + } +} + +/// Why [`locate_root`] refused a root. +#[derive(Debug)] +pub enum RootRefusal { + /// The root is the configured store's own. + ConfiguredRoot, + /// The catalog records no row under the root, of either kind; the + /// stores it does record rows for. + NoRows(Vec), + /// The catalog could not be read. + Catalog(StorageError), +} + +/// The kind and store ids of the rows the catalog records under `root` +/// (module doc). `configured_root` is the configured store's root label, +/// when a store is configured: that root is refused. +/// +/// # Errors +/// +/// [`RootRefusal`]. +pub async fn locate_root( + storage: &S, + root: &str, + configured_root: Option<&str>, +) -> Result { + if configured_root == Some(root) { + return Err(RootRefusal::ConfiguredRoot); + } + for kind in [StoreKind::Filesystem, StoreKind::S3] { + let store_ids = storage + .read_store_ids(kind, root) + .await + .map_err(RootRefusal::Catalog)?; + if !store_ids.is_empty() { + return Ok(LocatedRoot { + kind, + root: root.to_owned(), + store_ids, + }); + } + } + let known = storage + .summarize_store_roots() + .await + .map_err(RootRefusal::Catalog)?; + Err(RootRefusal::NoRows(known)) +} + +// --- the removals ---------------------------------------------------------- + +/// What [`forget_row`] did with one row. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Forgotten { + /// The row is gone; it had this status. No object was deleted. + Removed(BackupStatus), + /// A DELETING row of the configured store: this command finished its + /// deletion (its objects are deleted, the deletion its DeleteBackup + /// asked for), then removed the row. Also a DELETED row whose deletion + /// the command's own dead-owner pass finished (SQLite). + RemovedAfterDeletion, + /// Left: a DELETING row of the configured store whose deletion this + /// command cannot finish, since the store's check did not pass (or the + /// store's id is unknown). + DeletionPending, + /// Left: a DELETING row of the configured store that carries this + /// store id, one of the row ids the store's check refused on as an + /// identity mismatch (`IdMismatch`, `MultipleRowIds`), so the deletion + /// waits on a check the row itself takes part in. + DeletionBehindMismatch(String), + /// Left: a row of the configured root, whose `extenddb-store.json` the + /// command found missing while the catalog holds rows for the root (a + /// volume that is not mounted, say), or of the defaulted root found + /// missing with rows for it ([`ForgetRun::unmounted_root`]). + RootWithoutFile, + /// The row was gone before the removal. + Gone, + /// Left: a restore target registered from the backup is CREATING (its + /// name, and the row's account). + InUse(String, String), + /// Left: a CREATING row whose owner is alive, or a FAILED row whose + /// heartbeat is within the owner window (its job ended with the FAILED + /// write, and may have been writing under the prefix until then). + LiveOwner(BackupStatus), + /// Left: the row changed under the removal again after the re-read (its + /// status and store now). + Changed(BackupStatus, StoreIdentity), + /// Left: a catalog call failed (why). + Failed(String), + /// Not reached: the run stopped (why) during the wait before the + /// first FAILED or DELETED removal, or before a DELETING row's + /// finished deletion ([`FORGET_WAIT`]), the row as it was; or at the + /// claim's re-check before a step ([`ForgetRun::claim`]), the row as + /// the steps before it left it (the reason says which). + Stopped(String), +} + +impl Forgotten { + /// Whether the row is still in the catalog: the command exits 1. + #[must_use] + pub fn is_left(&self) -> bool { + !matches!( + self, + Self::Removed(_) | Self::RemovedAfterDeletion | Self::Gone + ) + } + + /// Whether this run removed the row. + #[must_use] + pub fn is_removed(&self) -> bool { + matches!(self, Self::Removed(_) | Self::RemovedAfterDeletion) + } +} + +impl std::fmt::Display for Forgotten { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Removed(status) => write!( + f, + "removed: the row ({status}) is gone, and no object was deleted" + ), + Self::RemovedAfterDeletion => f.write_str( + "removed: the row is gone, and this command deleted its objects first (the row \ + was DELETING in the configured store: the deletion its DeleteBackup asked for)", + ), + Self::DeletionPending => f.write_str( + "left: a DELETING row of the configured store, whose deletion this command \ + finishes before it removes the row, and the store's check did not pass; let a \ + server finish the DeleteBackup, or run again once the store's check passes", + ), + Self::DeletionBehindMismatch(id) => write!( + f, + "left: a DELETING row of the configured store carrying store id {id}, one of the \ + ids the store's identity check refuses, so neither this command nor a server \ + finishes its deletion while that check fails; remove the row by running this \ + command with [backup.store] kind = \"disabled\" (its bytes stay), then, with the \ + store configured again, let backup sync adopt the backup once the check passes \ + and DeleteBackup it again" + ), + Self::RootWithoutFile => write!( + f, + "left: the configured root has no {} while the catalog holds rows for it (a \ + volume that is not mounted?); mount it, or point [backup.store] at the root \ + that holds these backups, then run again", + super::STORE_IDENTITY_KEY + ), + Self::Gone => f.write_str("the row was already gone"), + Self::InUse(table, account) => write!( + f, + "left: the restore target {table} is CREATING from this backup: wait for its \ + restore to end, or, when that restore failed or its owner is dead, remove the \ + target with extenddb backup remove-target --account {account} {table}; then run \ + again" + ), + Self::LiveOwner(BackupStatus::Failed) => f.write_str( + "left: the FAILED row was written less than stale_job_after_secs ago (its \ + heartbeat is within the window); run the command again once its heartbeat is \ + older than stale_job_after_secs", + ), + Self::LiveOwner(status) => write!( + f, + "left: the {status} row has a live owner (its heartbeat is within \ + stale_job_after_secs); run the command again once its job has ended" + ), + Self::Changed(status, store) => write!( + f, + "left: the row changed under the removal; it is now {status} in the {} store {} \ + (store id {})", + store.store_kind, store.store_root, store.store_id + ), + Self::Failed(why) => write!(f, "left: {why}"), + Self::Stopped(why) => write!(f, "not reached: {why}"), + } + } +} + +/// How long `--forget-store` and `--forget-backup` beside servers +/// (PostgreSQL, MongoDB, Cassandra) wait after taking the maintenance claim +/// before their first removal of a FAILED or DELETED row (and before +/// finishing the deletion of a DELETING row of the configured store, whose +/// DELETED row is removed next): a deletion duty +/// that read the claim free just before the command took it starts no +/// removal after [`super::reconciler::DUTY_DELETION_BOUND`] from that read +/// (the store checks the deadline before each one), and its step in flight +/// at the deadline, one removal or one S3 request, is answered or abandoned +/// within one call's bound ([`super::S3_CHECK_TIMEOUT`], the larger of the +/// two kinds', since the servers' stores need not be the command's). The +/// wait assumes the servers' processes keep running: one paused past it +/// between its last check of the deadline and a step it has already sent +/// runs that step when it resumes, the assumption every lease in the design +/// shares. On SQLite the serve lock excludes every server, and the command +/// does not wait. +pub const FORGET_WAIT: Duration = + super::reconciler::DUTY_DELETION_BOUND.saturating_add(super::S3_CHECK_TIMEOUT); + +/// How often the wait before the first FAILED or DELETED removal asks the +/// run's stop. +const FORGET_WAIT_STEP: Duration = Duration::from_secs(1); + +/// What a run of `--forget-store` or `--forget-backup` removes rows under. +#[derive(Clone, Copy)] +pub struct ForgetRun<'a> { + /// The window a live owner is told apart by: `stale_job_after` beside + /// servers, zero on SQLite under the serve lock. + pub owner_window: Duration, + /// The instant before which no FAILED or DELETED row is removed: the + /// claim's taking plus [`FORGET_WAIT`] beside servers, `None` on + /// SQLite. + pub removals_from: Option, + /// Told how long the run waits, once, when it does. + pub on_wait: &'a (dyn Fn(Duration) + Sync), + /// Asked before each row or ARN, and during the wait. + pub stop: Stop<'a>, + /// The maintenance claim the command holds, re-checked with a beat + /// ([`HeldClaim::recheck`]) right before each removal, each + /// `set_backup_deleted`, each finished deletion and the sweep of the + /// root's status entries; a claim found lost there stops the run as + /// `stop` does ([`Forgotten::Stopped`]). `None` re-checks nothing. + pub claim: Option<&'a HeldClaim>, + /// The configured store as the command opened it, when one is + /// configured. + pub configured: Option>, + /// A filesystem root the command found missing as it opened the + /// store, with rows for it in the catalog: the defaulted root moved + /// away, or a symbolic link to a volume that is gone. Every row of it + /// is refused ([`Forgotten::RootWithoutFile`]), as the rows of a + /// configured root without its identity file are. + pub unmounted_root: Option<&'a str>, + /// The DELETING rows whose deletion the command's own dead-owner pass + /// finished before the removals (SQLite), by ARN: their DELETED rows' + /// removal says the command deleted the objects. + pub deletions_finished: &'a [String], +} + +/// The configured store as a forget option meets its rows. +#[derive(Clone, Copy)] +pub struct ForgetConfigured<'a> { + pub store: &'a ConfiguredStore, + /// Its identity when the command's check of the store, which writes + /// nothing, passed: a DELETING row of this store has its deletion + /// finished behind the fence before its removal. `None` otherwise, and + /// such a row is refused ([`Forgotten::DeletionPending`]). + pub healthy: Option<&'a StoreIdentity>, + /// The root's row ids when the opening's check failed on an identity + /// mismatch (`IdMismatch`, `MultipleRowIds`), empty otherwise: a + /// DELETING row carrying one of them is refused as + /// [`Forgotten::DeletionBehindMismatch`], which names the way out. + pub mismatch_row_ids: &'a [String], + /// Whether the opening found no identity file under the root while the + /// catalog holds rows for it: every row of the root is refused + /// ([`Forgotten::RootWithoutFile`]), since a server's next check + /// would write a new identity file at the root once its last row is + /// gone. + pub file_missing_with_rows: bool, +} + +impl ForgetConfigured<'_> { + /// Whether `row` is of the configured root (any id). + fn holds(&self, row: &BackupRow) -> bool { + row.store.store_kind == self.store.kind && row.store.store_root == self.store.root + } +} + +/// One run's removals: its [`ForgetRun`], the FAILED rows whose owner is +/// dead (read once per store), and whether the wait is over. +struct Removals<'a> { + run: ForgetRun<'a>, + failed_owners: FailedOwners, + waited: bool, +} + +impl<'a> Removals<'a> { + fn new(run: ForgetRun<'a>) -> Self { + Self { + run, + failed_owners: FailedOwners::new(run.owner_window), + waited: run.removals_from.is_none(), + } + } + + /// The wait before the first FAILED or DELETED removal ([`FORGET_WAIT`]); + /// `Err` is the stop that ended it. + async fn wait_for_duties(&mut self) -> Result<(), String> { + if self.waited { + return Ok(()); + } + let until = self + .run + .removals_from + .expect("an unfinished wait has an end"); + let now = tokio::time::Instant::now(); + if until > now { + (self.run.on_wait)(until - now); + } + loop { + if let Some(reason) = (self.run.stop)() { + return Err(reason); + } + let now = tokio::time::Instant::now(); + if now >= until { + break; + } + tokio::time::sleep((until - now).min(FORGET_WAIT_STEP)).await; + } + self.waited = true; + Ok(()) + } + + /// One beat of the run's claim right before `step`, a step that cannot + /// be undone; `Err` is the loss as the outcome that stops the run, the + /// row as the steps before it left it (`left`). + async fn recheck(&self, step: &str, left: &str) -> Result<(), Forgotten> { + match self.run.claim { + Some(claim) => claim + .recheck() + .await + .map_err(|reason| Forgotten::Stopped(format!("{reason} (before {step}; {left})"))), + None => Ok(()), + } + } +} + +/// The form of each ARN `--forget-backup` is given, before the command +/// takes the claim: the ARNs refused, each with why. +#[must_use] +pub fn check_arns(arns: &[String]) -> Vec<(String, String)> { + arns.iter() + .filter_map(|given| { + let why = match extenddb_storage::util::check_backup_arn(given) { + Err(e) => e.to_string(), + Ok(_) => match BackupArn::parse(given) { + Err(e) => e.to_string(), + Ok(_) => return None, + }, + }; + Some((given.clone(), why)) + }) + .collect() +} + +/// The FAILED rows whose owner is dead, per store, read once per store: +/// a FAILED row the backend lists as older than the owner window on the +/// heartbeat clock. +struct FailedOwners { + window: Duration, + dead: HashMap>, +} + +impl FailedOwners { + fn new(window: Duration) -> Self { + Self { + window, + dead: HashMap::new(), + } + } + + async fn is_dead( + &mut self, + storage: &S, + row: &BackupRow, + ) -> Result { + if !self.dead.contains_key(&row.store) { + let dead: HashSet = storage + .list_backup_rows_older_than( + BackupStatus::Failed, + &row.store, + StaleClock::Heartbeat, + self.window, + ) + .map_ok(|r| r.backup_arn) + .try_collect() + .await?; + self.dead.insert(row.store.clone(), dead); + } + Ok(self.dead[&row.store].contains(&row.backup_arn)) + } +} + +/// Remove `row` outright by its status's removal (module doc); a removal +/// that finds another status re-reads the row once and applies that +/// status's. +async fn forget_row( + storage: &S, + row: &BackupRow, + removals: &mut Removals<'_>, +) -> Forgotten { + let mut current = row.clone(); + for attempt in 0..2 { + if let Some(done) = remove_by_status(storage, ¤t, removals).await { + return done; + } + // The removal matched no row of that status: another writer moved + // it, or (CREATING) its owner is alive. Read it once more. + let arn = match BackupArn::parse(&row.backup_arn) { + Ok(arn) => arn, + Err(e) => return Forgotten::Failed(format!("the row's ARN does not parse: {e}")), + }; + let now = match storage.read_backup_row(&arn).await { + Ok(Some(now)) => lean(now), + Ok(None) => return Forgotten::Gone, + Err(e) => return Forgotten::Failed(format!("the row could not be read again: {e}")), + }; + if now.store != row.store { + return Forgotten::Changed(now.status, now.store); + } + if now.status == BackupStatus::Creating && current.status == BackupStatus::Creating { + return Forgotten::LiveOwner(BackupStatus::Creating); + } + current = now; + if attempt == 1 { + break; + } + } + Forgotten::Changed(current.status, current.store) +} + +/// The removal of `row`'s status (module doc): `None` when it matched no +/// row of that status. +async fn remove_by_status( + storage: &S, + row: &BackupRow, + removals: &mut Removals<'_>, +) -> Option { + let owner_window = removals.run.owner_window; + let key = row.key(); + let failed = |what: &str, e: StorageError| Some(Forgotten::Failed(format!("{what}: {e}"))); + let status = row.status; + let configured = removals + .run + .configured + .filter(|configured| configured.holds(row)); + let unmounted = removals.run.unmounted_root.is_some_and(|root| { + row.store.store_kind == StoreKind::Filesystem && row.store.store_root == root + }); + if unmounted || configured.is_some_and(|configured| configured.file_missing_with_rows) { + return Some(Forgotten::RootWithoutFile); + } + match status { + BackupStatus::Available => { + if let Err(stopped) = removals + .recheck("the AVAILABLE row's removal", "the row is unchanged") + .await + { + return Some(stopped); + } + match storage.forget_available_backup_row(&key).await { + Ok(ForgetRow::Removed) => Some(Forgotten::Removed(status)), + Ok(ForgetRow::InUse(table)) => { + Some(Forgotten::InUse(table, row.account_id.clone())) + } + Ok(ForgetRow::Status(_)) => None, + Err(e) if is_backup_not_found(&e) => Some(Forgotten::Gone), + Err(e) => failed("the row could not be removed", e), + } + } + BackupStatus::Creating => { + if let Err(stopped) = removals + .recheck("the CREATING row's removal", "the row is unchanged") + .await + { + return Some(stopped); + } + match storage + .remove_dead_creating_backup_row(&key, owner_window) + .await + { + Ok(true) => Some(Forgotten::Removed(status)), + Ok(false) => None, + Err(e) => failed("the row could not be removed", e), + } + } + BackupStatus::Failed | BackupStatus::Deleted => { + if status == BackupStatus::Failed { + match removals.failed_owners.is_dead(storage, row).await { + Ok(true) => {} + Ok(false) => return Some(Forgotten::LiveOwner(status)), + Err(e) => return failed("the row's owner could not be tested", e), + } + } + if let Err(reason) = removals.wait_for_duties().await { + return Some(Forgotten::Stopped(reason)); + } + if let Err(stopped) = removals + .recheck( + &format!("the {status} row's removal"), + "the row is unchanged", + ) + .await + { + return Some(stopped); + } + match storage + .remove_backup_row(&key, row.store.store_kind, &row.store.store_root) + .await + { + Ok(true) + if status == BackupStatus::Deleted + && removals.run.deletions_finished.contains(&row.backup_arn) => + { + Some(Forgotten::RemovedAfterDeletion) + } + Ok(true) => Some(Forgotten::Removed(status)), + Ok(false) => None, + Err(e) => failed("the row could not be removed", e), + } + } + BackupStatus::Deleting if configured.is_some() => { + let configured = configured.expect("tested above"); + // A passed check means every row of the configured root carries + // the file's id (`compare` refuses several ids and a mismatch), + // so with the store healthy this row's store is the healthy one; + // the comparison is kept so a row of another id is never + // finished in this store whatever the check let through. + if configured.healthy != Some(&row.store) { + if configured.mismatch_row_ids.contains(&row.store.store_id) { + return Some(Forgotten::DeletionBehindMismatch( + row.store.store_id.clone(), + )); + } + return Some(Forgotten::DeletionPending); + } + finish_then_remove(storage, configured.store, row, removals).await + } + BackupStatus::Deleting => { + if let Err(stopped) = removals + .recheck("the DELETING row's DELETED write", "the row is unchanged") + .await + { + return Some(stopped); + } + match storage.set_backup_deleted(&key).await { + Ok(true) => { + if let Err(stopped) = removals + .recheck( + "the DELETED row's removal", + "the row is DELETED, and a second run removes it", + ) + .await + { + return Some(stopped); + } + match storage + .remove_backup_row(&key, row.store.store_kind, &row.store.store_root) + .await + { + Ok(true) => Some(Forgotten::Removed(status)), + Ok(false) => None, + Err(e) => failed( + "the row was set DELETED and could not be removed (a second run \ + removes it)", + e, + ), + } + } + Ok(false) => None, + Err(e) => failed("the row could not be set DELETED", e), + } + } + } +} + +/// A DELETING row of the configured store (healthy, its id the row's): +/// after the wait for the duties, its deletion finished behind the fence +/// (`finish_backup_delete`, which deletes the prefix and sets the row +/// DELETED), then the row removed. The wait comes first, so a stop during +/// it leaves the row DELETING and its objects untouched. `None` when the +/// row was no longer DELETING. +async fn finish_then_remove( + storage: &S, + configured: &ConfiguredStore, + row: &BackupRow, + removals: &mut Removals<'_>, +) -> Option { + let arn = match BackupArn::parse(&row.backup_arn) { + Ok(arn) => arn, + Err(e) => { + return Some(Forgotten::Failed(format!( + "the row's ARN does not parse: {e}" + ))); + } + }; + if let Err(reason) = removals.wait_for_duties().await { + return Some(Forgotten::Stopped(reason)); + } + if let Err(stopped) = removals + .recheck( + "the DELETING row's deletion in the configured store", + "the row and its objects are unchanged", + ) + .await + { + return Some(stopped); + } + let fence = configured.fence(); + match super::finish_backup_delete(storage, &configured.calls("deletion"), &fence, &arn).await { + Ok(true) => { + tracing::info!( + target: "extenddb::audit", + action = "backup sync --forget-backup", + account_id = %row.account_id, + backup_arn = %row.backup_arn, + status = "DELETED", + store_root = %row.store.store_root, + "finished a DELETING row's deletion in the configured store before its removal" + ); + } + Ok(false) => return None, + Err(e) => { + return Some(Forgotten::Failed(format!( + "its deletion in the configured store did not finish ({e}); the row is left \ + DELETING" + ))); + } + } + if let Err(stopped) = removals + .recheck( + "the DELETED row's removal", + "its deletion is finished, the row is DELETED, and a second run removes it", + ) + .await + { + return Some(stopped); + } + match storage + .remove_backup_row(&row.key(), row.store.store_kind, &row.store.store_root) + .await + { + Ok(true) => Some(Forgotten::RemovedAfterDeletion), + Ok(false) => None, + Err(e) => Some(Forgotten::Failed(format!( + "the row's deletion is finished and the DELETED row could not be removed (a second \ + run removes it): {e}" + ))), + } +} + +/// The audit line of a removal. +fn audit_removal(action: &str, row: &BackupRow, outcome: &Forgotten) { + let (status, message) = match outcome { + Forgotten::Removed(status) => ( + *status, + "removed a backup row outright; no object was deleted", + ), + Forgotten::RemovedAfterDeletion => ( + row.status, + "removed a backup row after this command finished its deletion", + ), + _ => return, + }; + tracing::info!( + target: "extenddb::audit", + action = action, + account_id = %row.account_id, + backup_arn = %row.backup_arn, + status = %status, + store_kind = %row.store.store_kind, + store_root = %row.store.store_root, + store_id = %row.store.store_id, + "{message}" + ); +} + +/// What [`forget_store`] did. +#[derive(Debug, Default)] +pub struct ForgetStoreReport { + /// Each row of the root and what was done with it, in listing order. + pub rows: Vec<(NamedRow, Forgotten)>, + /// Why the run stopped before the remaining rows, when it did. + pub stopped: Option, + /// The rows the root still lists after the removals, named (each is + /// either a row above that was left, or one that landed after the first + /// listing, from a writer still configured with the root); `None` when + /// the listing failed (its error in `recheck_error`). + pub rows_left: Option>, + /// What the sweep of the root's status entries after the removals did + /// (Cassandra; zero elsewhere): the entries with no row of the root + /// behind them deleted, and those whose row is of the root under + /// another status repaired. `None` when it did not run (a stop, or a + /// listing that failed). + pub entries_swept: Option, + /// The store ids the catalog still records for the root after the + /// sweep, read when the root's listing after the removals returned no + /// row: status entries written after that listing (a writer still + /// configured with the root). + pub entries_left: Vec, + /// Why the read after the removals failed, when it did. + pub recheck_error: Option, +} + +impl ForgetStoreReport { + /// The rows removed. + #[must_use] + pub fn removed(&self) -> usize { + self.rows.iter().filter(|(_, f)| f.is_removed()).count() + } + + /// Whether every row is gone: no row left, no stop, the read after the + /// removals answered and listed nothing. Entries left behind do not + /// count: no row stands behind them. + #[must_use] + pub fn complete(&self) -> bool { + self.stopped.is_none() + && self.recheck_error.is_none() + && self.rows_left.as_ref().is_some_and(Vec::is_empty) + && !self.rows.iter().any(|(_, f)| f.is_left()) + } +} + +/// `--forget-store ` (module doc): every row of `root` (of kind +/// `kind`, from [`locate_root`]) removed outright under `run`. Its stop is +/// asked before each row (the claim lost, a signal). Each removal is logged +/// under the `extenddb::audit` target. +/// +/// # Errors +/// +/// The backend's error for the root's listing; nothing was removed. +pub async fn forget_store( + storage: &S, + kind: StoreKind, + root: &str, + run: ForgetRun<'_>, +) -> Result { + let rows: Vec = storage + .list_backup_rows_by_root(kind, root, None) + .map_ok(lean) + .try_collect() + .await?; + let mut report = ForgetStoreReport::default(); + let mut removals = Removals::new(run); + for row in &rows { + if let Some(reason) = (run.stop)() { + report.stopped = Some(reason); + return Ok(report); + } + let outcome = forget_row(storage, row, &mut removals).await; + if let Forgotten::Stopped(reason) = outcome { + report.stopped = Some(reason); + return Ok(report); + } + audit_removal("backup sync --forget-store", row, &outcome); + report.rows.push((NamedRow::of(row), outcome)); + } + match storage + .list_backup_rows_by_root(kind, root, None) + .map_ok(|row| NamedRow::of(&row)) + .try_collect::>() + .await + { + Ok(left) => report.rows_left = Some(left), + Err(e) => report.recheck_error = Some(format!("the root's rows could not be read: {e}")), + } + // The root's status entries (Cassandra), under the claim: each verified + // against its row as it is read, deleted or repaired. The claim is + // confirmed by a beat first, as before each removal. + if report.recheck_error.is_none() + && let Some(claim) = run.claim + && let Err(reason) = claim.recheck().await + { + report.stopped = Some(format!( + "{reason} (before the sweep of the root's status entries, which did not run)" + )); + return Ok(report); + } + if report.recheck_error.is_none() { + match storage.remove_orphan_status_entries(kind, root).await { + Ok(sweep) => report.entries_swept = Some(sweep), + Err(e) => { + report.recheck_error = + Some(format!("the root's status entries could not be swept: {e}")); + } + } + } + if report.rows_left.as_ref().is_some_and(Vec::is_empty) && report.recheck_error.is_none() { + match storage.read_store_ids(kind, root).await { + Ok(ids) => report.entries_left = ids, + Err(e) => { + report.recheck_error = Some(format!("the root's store ids could not be read: {e}")); + } + } + } + Ok(report) +} + +/// What `--forget-backup` did with one ARN. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ForgetBackup { + /// The ARN's row, and what [`forget_row`] did with it. + Row(BackupStatus, Forgotten), + /// No row carries the ARN as given; a row under this other spelling of + /// it does (its status), and was left. + OtherSpelling(String, BackupStatus), + /// The ARN is an earlier release's in-catalog backup: nothing changed. + Legacy, + /// No row and no in-catalog backup carries the ARN, under either + /// spelling: there is nothing to remove, and the ARN counts as done. + NoRow, + /// The ARN is not one (why). + Invalid(String), + /// A catalog call failed (why); nothing changed. + Failed(String), +} + +impl ForgetBackup { + /// Whether the ARN's row is gone (removed now, or already, or no row + /// carries it): the run goes on to the next ARN; anything else stops it + /// and exits 1. + #[must_use] + pub fn is_done(&self) -> bool { + matches!( + self, + Self::Row( + _, + Forgotten::Removed(_) | Forgotten::RemovedAfterDeletion | Forgotten::Gone + ) | Self::NoRow + ) + } +} + +impl std::fmt::Display for ForgetBackup { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Row(_, outcome) => write!(f, "{outcome}"), + Self::OtherSpelling(arn, status) => write!( + f, + "left: no row carries this ARN as written; the row of this backup is {arn} \ + ({status}): run --forget-backup with that ARN" + ), + Self::Legacy => f.write_str( + "an in-catalog backup; --forget-backup changes nothing; DeleteBackup removes it", + ), + Self::NoRow => f.write_str("no row carries this ARN; nothing to remove"), + Self::Invalid(why) => write!(f, "refused: {why}"), + Self::Failed(why) => write!(f, "left: {why}"), + } + } +} + +/// What [`forget_backups`] did. +#[derive(Debug, Default)] +pub struct ForgetBackupsReport { + /// Each ARN reached, as given, and its outcome, in the order given. + pub done: Vec<(String, ForgetBackup)>, + /// Why the run stopped before the remaining ARNs: a signal, the claim + /// lost. A refusal stops the run too and is the last of `done`. + pub stopped: Option, +} + +impl ForgetBackupsReport { + /// Whether every ARN's row is gone. + #[must_use] + pub fn complete(&self, asked: usize) -> bool { + self.stopped.is_none() + && self.done.len() == asked + && self.done.iter().all(|(_, outcome)| outcome.is_done()) + } +} + +/// `--forget-backup `, once per ARN in the order given under `run`, +/// stopping at the first ARN whose row is not removed (module doc). +/// `region` is the server's, for the other spelling of the ARN. The run's +/// stop is asked before each ARN. Each removal is logged under the +/// `extenddb::audit` target. +pub async fn forget_backups( + storage: &S, + arns: &[String], + region: &str, + run: ForgetRun<'_>, +) -> ForgetBackupsReport { + let mut report = ForgetBackupsReport::default(); + let mut removals = Removals::new(run); + for given in arns { + if let Some(reason) = (run.stop)() { + report.stopped = Some(reason); + return report; + } + let outcome = forget_backup(storage, given, region, &mut removals).await; + if let ForgetBackup::Row(_, Forgotten::Stopped(reason)) = &outcome { + report.stopped = Some(reason.clone()); + return report; + } + let done = outcome.is_done(); + report.done.push((given.clone(), outcome)); + if !done { + break; + } + } + report +} + +async fn forget_backup( + storage: &S, + given: &str, + region: &str, + removals: &mut Removals<'_>, +) -> ForgetBackup { + let parts = match extenddb_storage::util::check_backup_arn(given) { + Ok(parts) => parts, + Err(e) => return ForgetBackup::Invalid(e.to_string()), + }; + let arn = match BackupArn::parse(given) { + Ok(arn) => arn, + Err(e) => return ForgetBackup::Invalid(e.to_string()), + }; + match storage.read_backup_row(&arn).await { + Ok(Some(row)) => { + let row = lean(row); + let outcome = forget_row(storage, &row, removals).await; + audit_removal("backup sync --forget-backup", &row, &outcome); + return ForgetBackup::Row(row.status, outcome); + } + Ok(None) => {} + Err(e) => return ForgetBackup::Failed(format!("the row could not be read: {e}")), + } + let spellings = super::BackupArnSpellings::new(&parts, region); + for spelling in spellings.arns() { + if *spelling == arn { + continue; + } + match storage.read_backup_row(spelling).await { + Ok(Some(row)) => return ForgetBackup::OtherSpelling(row.backup_arn, row.status), + Ok(None) => {} + Err(e) => return ForgetBackup::Failed(format!("the row could not be read: {e}")), + } + } + let mut legacy = vec![spellings.canonical().to_owned()]; + if let Some(as_written) = &spellings.as_written { + legacy.push(as_written.clone()); + } + for spelling in legacy { + match storage.describe(parts.account_id, &spelling).await { + Ok(_) => return ForgetBackup::Legacy, + Err(e) if is_backup_not_found(&e) => {} + Err(e) => { + return ForgetBackup::Failed(format!( + "the in-catalog backups could not be read: {e}" + )); + } + } + } + ForgetBackup::NoRow +} + +/// A row with its parsed `source_table_details_json` dropped: the options +/// keep the rows of a whole root, and read no column of that one. +fn lean(mut row: BackupRow) -> BackupRow { + row.source_table_details_json = serde_json::Value::Null; + row +} + +// --- the move -------------------------------------------------------------- + +/// How many keys one `rewrite_store` call of the move covers, and how many +/// keys apart the move confirms its claim with a beat during the rewrite. +/// A backend that rewrites key by key (Cassandra, +/// `rewrite_store_runs_key_by_key`) is called with one key at a time. The +/// signal and the claim's local state are read before every call, so a +/// move whose claim another holder took during the rewrite (a command +/// stopped past `stale_job_after_secs` on MongoDB or Cassandra) writes no +/// call after the one it had sent or was about to send, with the keys not +/// yet rewritten left under the old root and listed there. One slice is +/// one statement on PostgreSQL and MongoDB, and one transaction on SQLite, +/// well inside PostgreSQL's batch and its statement bound. +pub const REWRITE_SLICE: usize = 500; + +/// How many rows the move reads by ARN at a time after the rewrite, to +/// tell a moved row (it names the configured store) from one another +/// writer removed or moved elsewhere: one read per key the old root no +/// longer lists, in place of a listing of the configured store, whose rows +/// every backend reads whole. +pub const MOVED_ROW_READS: usize = 16; + +/// What [`move_store`] moves rows from and to. +#[derive(Clone, Copy)] +pub struct MoveTarget<'a> { + /// The configured store, healthy: the copy is under its root. + pub configured: &'a ConfiguredStore, + /// Its identity, the one the rows get. + pub identity: &'a StoreIdentity, + /// The old root, from [`locate_root`]. + pub from_kind: StoreKind, + pub from_root: &'a str, + /// The window a live owner is told apart by (`stale_job_after` beside + /// servers, zero on SQLite under the serve lock). + pub owner_window: Duration, + /// Asked before each row, before the rewrite and before each call of + /// the rewrite; a stop there leaves the rows of the earlier calls moved + /// and the rest under the old root, as a claim lost there does. + pub stop: Stop<'a>, + /// The maintenance claim the command holds, re-checked with a beat + /// ([`HeldClaim::recheck`]) right before each dead CREATING row's + /// resolution, each DELETING row's finished deletion and each + /// [`REWRITE_SLICE`] keys of the rewrite, and read without a beat + /// ([`HeldClaim::lost`]) before every other call of the rewrite; a claim + /// found lost there stops the move as `stop` does + /// ([`MoveReport::stopped`]), with the rows of the calls before it + /// moved. `None` re-checks nothing. + pub claim: Option<&'a HeldClaim>, +} + +/// What [`move_store`] did. +#[derive(Debug, Default)] +pub struct MoveReport { + /// How many rows the old root listed. + pub rows: usize, + /// The CREATING rows resolved, with the status written (AVAILABLE: the + /// move writes no other). + pub resolved: Vec<(String, BackupStatus)>, + /// The DELETING rows whose deletion was finished in the configured + /// store (now DELETED). + pub finished: Vec, + /// Every row that stops the move, with the reason; non-empty means no + /// row was moved. + pub stopping: Vec<(NamedRow, String)>, + /// Why the move stopped before the rewrite, or between two of its + /// calls, otherwise: a signal, the claim lost, the fence's refusal. + /// A stop between calls leaves the earlier calls' rows moved + /// ([`Self::moved`]) and the rest under the old root ([`Self::left`]). + pub stopped: Option, + /// The rows moved, by ARN: rows the old root no longer lists and whose + /// row, read by its ARN, now names the configured store (every key + /// counted, when the old root's listing after the rewrite failed). + pub moved: Vec, + /// The rows neither root lists after the rewrite: removed, or moved to + /// another store, by another writer during the move (a tombstone duty + /// of a server on the old root, a `--move-store` of another catalog's + /// command); nothing was moved for them. + pub gone: Vec, + /// What `rewrite_store` counted. + pub rewritten: u64, + /// The rewrite's error, when it failed (it may have moved some rows). + pub rewrite_error: Option, + /// The rows the old root still lists after the rewrite. + pub left: Vec, + /// Why the listing after the rewrite failed, when it did. + pub relist_error: Option, +} + +impl MoveReport { + /// Whether every row of the old root was moved. + #[must_use] + pub fn complete(&self) -> bool { + self.stopping.is_empty() + && self.stopped.is_none() + && self.rewrite_error.is_none() + && self.relist_error.is_none() + && self.left.is_empty() + } +} + +/// `--move-store ` (module doc). The caller holds the maintenance +/// claim and has waited for the jobs of the configured store and of the old +/// root's store ids. +/// +/// # Errors +/// +/// The backend's error for the listing of the old root or of the +/// configured root, both read before any row is checked; nothing was +/// changed. +pub async fn move_store( + storage: &S, + target: MoveTarget<'_>, +) -> Result { + let mut rows: Vec = storage + .list_backup_rows_by_root(target.from_kind, target.from_root, None) + .map_ok(lean) + .try_collect() + .await?; + let mut report = MoveReport { + rows: rows.len(), + ..MoveReport::default() + }; + // One backup id per account after the move, checked first, so no row of + // a group that stops for it gets a resolution or a deletion (on + // Cassandra two spellings of an ARN share one prefix). The rows are + // grouped by (account, backup id) over the configured root's rows and + // the old root's, and every old-root row of a group of more than one is + // set aside before any row is checked. + let mut groups: HashMap<(String, String), Vec> = HashMap::new(); + let mut configured_rows = storage.list_backup_rows_by_root( + target.identity.store_kind, + &target.identity.store_root, + None, + ); + while let Some(row) = configured_rows.try_next().await? { + groups + .entry((row.account_id, row.backup_id)) + .or_default() + .push(row.backup_arn); + } + drop(configured_rows); + for row in &rows { + groups + .entry((row.account_id.clone(), row.backup_id.clone())) + .or_default() + .push(row.backup_arn.clone()); + } + let mut two_rows: HashSet = HashSet::new(); + for row in &rows { + let group = &groups[&(row.account_id.clone(), row.backup_id.clone())]; + if group.len() < 2 { + continue; + } + two_rows.insert(row.backup_arn.clone()); + let others: Vec<&str> = group + .iter() + .filter(|arn| **arn != row.backup_arn) + .map(String::as_str) + .collect(); + report.stopping.push(( + NamedRow::of(row), + format!( + "the move would give account {} two rows of backup id {}: {} holds it as well; \ + forget one of them with --forget-backup", + row.account_id, + row.backup_id, + others.join(", ") + ), + )); + } + let calls = StoreCalls { + store: &*target.configured.store, + metrics: None, + }; + for row in &mut rows { + if two_rows.contains(&row.backup_arn) { + continue; + } + if let Some(reason) = (target.stop)() { + report.stopped = Some(reason); + return Ok(report); + } + if let Err(why) = check_row(storage, target, calls, row, &mut report).await { + report.stopping.push((NamedRow::of(row), why)); + } + if report.stopped.is_some() { + return Ok(report); + } + } + if !report.stopping.is_empty() { + return Ok(report); + } + if let Some(reason) = (target.stop)() { + report.stopped = Some(reason); + return Ok(report); + } + if let Err(problem) = target.configured.fence().check_catalog(storage).await { + report.stopped = Some(format!( + "the configured store's fence refused the move: {problem}" + )); + return Ok(report); + } + // The rewrite in calls of REWRITE_SLICE keys, or of one key on a + // backend that rewrites key by key (Cassandra). Before each call the + // signal and the claim's local state are read, which writes nothing, + // and before each REWRITE_SLICE keys the claim is confirmed with a + // beat: a claim another holder took during the rewrite (a command + // stopped past `stale_job_after_secs` on MongoDB or Cassandra) stops + // it before the next call, and the keys not yet rewritten stay under + // the old root, where the listing below reports them. + let keys: Vec = rows.iter().map(BackupRow::key).collect(); + let per_call = if storage.rewrite_store_runs_key_by_key() { + 1 + } else { + REWRITE_SLICE + }; + let mut sent = 0_usize; + for call in keys.chunks(per_call) { + let mut reason = (target.stop)().or_else(|| target.claim.and_then(HeldClaim::lost)); + if reason.is_none() + && sent.is_multiple_of(REWRITE_SLICE) + && let Some(claim) = target.claim + { + reason = claim.recheck().await.err(); + } + if let Some(reason) = reason { + report.stopped = Some(if sent == 0 { + format!("{reason} (before the rewrite; no row was moved)") + } else { + format!( + "{reason} (before the rewrite of row {} of {}; the {} row(s) rewritten \ + before it are moved, and the rest are left under the old root)", + sent + 1, + keys.len(), + report.rewritten + ) + }); + break; + } + match storage + .rewrite_store(target.from_kind, target.from_root, call, target.identity) + .await + { + Ok(count) => report.rewritten += count, + Err(e) => { + report.rewrite_error = Some(e.to_string()); + break; + } + } + sent += call.len(); + } + let left: Result, StorageError> = storage + .list_backup_rows_by_root(target.from_kind, target.from_root, None) + .map_ok(|row| NamedRow::of(&row)) + .try_collect() + .await; + match left { + Ok(left) => { + let still: HashSet<&str> = left.iter().map(|r| r.backup_arn.as_str()).collect(); + // A key the old root no longer lists is moved when its row now + // names the configured store: one read per such key, up to + // MOVED_ROW_READS at a time and taken in key order, so the + // configured store's rows are not read whole here. + let mut reads = futures::stream::iter( + keys.iter() + .filter(|k| !still.contains(k.backup_arn.as_str())) + .map(|key| async move { + let now = match BackupArn::parse(&key.backup_arn) { + Ok(arn) => storage.read_backup_row(&arn).await, + Err(_) => Ok(None), + }; + (key, now) + }), + ) + .buffered(MOVED_ROW_READS); + while let Some((key, now)) = reads.next().await { + if report.relist_error.is_some() { + report.moved.push(key.backup_arn.clone()); + continue; + } + match now { + Ok(Some(row)) if row.store == *target.identity => { + report.moved.push(key.backup_arn.clone()); + } + Ok(_) => report.gone.push(key.backup_arn.clone()), + Err(e) => { + // The count is all there is for this key and the + // rest: each is taken as moved. + report.relist_error = Some(format!( + "the row of {} could not be read after the rewrite: {e}", + key.backup_arn + )); + report.moved.push(key.backup_arn.clone()); + } + } + } + drop(reads); + report.left = left; + } + Err(e) => { + report.relist_error = Some(format!( + "the old root's rows could not be read after the rewrite: {e}" + )); + // The count is all there is; with every key counted, every + // row moved. + if report.rewrite_error.is_none() && report.rewritten == keys.len() as u64 { + report.moved = keys.iter().map(|k| k.backup_arn.clone()).collect(); + } + } + } + let moved: HashSet<&str> = report.moved.iter().map(String::as_str).collect(); + for row in rows + .iter() + .filter(|r| moved.contains(r.backup_arn.as_str())) + { + tracing::info!( + target: "extenddb::audit", + action = "backup sync --move-store", + account_id = %row.account_id, + backup_arn = %row.backup_arn, + status = %row.status, + old_store_root = %target.from_root, + old_store_id = %row.store.store_id, + store_kind = %target.identity.store_kind, + store_root = %target.identity.store_root, + store_id = %target.identity.store_id, + "moved a backup row to the configured store; no object was copied or deleted" + ); + } + Ok(report) +} + +/// Why a CREATING row the backend does not hand over stops the move. +const LIVE_CREATING: &str = "CREATING with a live owner (or another taker): wait for its \ + CreateBackup to end and run the move again"; + +/// One beat of the move's claim ([`MoveTarget::claim`]) right before +/// `step`, a step that cannot be undone: `false` when the claim is lost, +/// with the reason in `report.stopped`, which stops the move. +async fn claim_held(target: MoveTarget<'_>, step: &str, report: &mut MoveReport) -> bool { + let Some(claim) = target.claim else { + return true; + }; + match claim.recheck().await { + Ok(()) => true, + Err(reason) => { + report.stopped = Some(format!("{reason} (before {step}; no row was moved)")); + false + } + } +} + +/// One row of the old root against the configured store (module doc): +/// `Err` is why it stops the move. A CREATING row resolved, and a DELETING +/// row finished, carry their new status in `row` from here on. +async fn check_row( + storage: &S, + target: MoveTarget<'_>, + calls: StoreCalls<'_>, + row: &mut BackupRow, + report: &mut MoveReport, +) -> Result<(), String> { + let arn = BackupArn::parse(&row.backup_arn) + .map_err(|e| format!("the row's ARN does not parse: {e}"))?; + let prefix = arn.store_prefix(); + if row.status == BackupStatus::Creating { + if !claim_held( + target, + &format!("the resolution of the CREATING row {}", row.backup_arn), + report, + ) + .await + { + return Ok(()); + } + return match resolve_dead_creating_row( + storage, + calls, + row, + target.owner_window, + OnIncomplete::Leave, + ) + .await + { + CreatingOutcome::Resolved(status) => { + tracing::info!( + target: "extenddb::audit", + action = "backup sync --move-store", + account_id = %row.account_id, + backup_arn = %row.backup_arn, + status = %status, + store_root = %row.store.store_root, + "resolved a CREATING row whose owner is dead against the configured store" + ); + report.resolved.push((row.backup_arn.clone(), status)); + row.status = status; + // The probe that resolved it is the AVAILABLE check. + Ok(()) + } + CreatingOutcome::Incomplete(why) => Err(format!( + "CREATING, its owner dead, and the copy under the configured root holds no \ + complete backup ({why}); the move writes no status for it, since the old root \ + may hold the complete backup. Run a server configured with the old root (or, \ + on SQLite, plain backup sync there), whose reconciler resolves the row against \ + the old root, then copy the prefix again; or remove the row with \ + --forget-backup" + )), + CreatingOutcome::NotHandedOver => Err(LIVE_CREATING.to_owned()), + CreatingOutcome::LeftToOwner(why) => Err(format!( + "CREATING, taken over by another owner during the move ({why})" + )), + CreatingOutcome::Failed(why) => Err(format!("CREATING: {why}")), + }; + } + // A DELETING or DELETED row's backup was deleted by a client: adopting + // its copied prefix would bring it back, so the remedy offered is the + // prefix's removal. A FAILED row's prefix may be what the operator + // wants kept. + let verified = |status: BackupStatus| { + let remedy = match status { + BackupStatus::Failed => { + "Remove that prefix from the configured root, or remove the row with \ + --forget-backup and let backup sync adopt the prefix" + } + _ => { + "The backup was deleted: remove that prefix from the configured root and run \ + the move again (removing the row with --forget-backup instead would have \ + backup sync adopt the prefix, bringing back AVAILABLE a backup a client \ + deleted, to be deleted again)" + } + }; + format!( + "{status} whose prefix under the configured root holds a verified manifest: moving \ + it would have the reconciler delete a complete backup. {remedy}" + ) + }; + match row.status { + BackupStatus::Creating => unreachable!("returned above"), + BackupStatus::Deleting => { + match verify_prefix(calls, &prefix, None).await { + Ok(Verified::Complete(_)) => return Err(verified(BackupStatus::Deleting)), + Ok(Verified::Incomplete(_)) => {} + Err(e) => return Err(format!("the configured store could not be read: {e}")), + } + if !claim_held( + target, + &format!( + "the deletion of the DELETING row {} in the configured store", + row.backup_arn + ), + report, + ) + .await + { + return Ok(()); + } + let fence = target.configured.fence(); + match super::finish_backup_delete( + storage, + &target.configured.calls("deletion"), + &fence, + &arn, + ) + .await + { + Ok(true) => { + tracing::info!( + target: "extenddb::audit", + action = "backup sync --move-store", + account_id = %row.account_id, + backup_arn = %row.backup_arn, + status = "DELETED", + store_root = %target.identity.store_root, + "finished a DELETING row's deletion in the configured store" + ); + report.finished.push(row.backup_arn.clone()); + row.status = BackupStatus::Deleted; + Ok(()) + } + Ok(false) => Err( + "DELETING: the row was no longer DELETING when its objects were gone" + .to_owned(), + ), + Err(e) => Err(format!( + "DELETING: its deletion in the configured store did not finish: {e}" + )), + } + } + BackupStatus::Deleted | BackupStatus::Failed => { + match verify_prefix(calls, &prefix, None).await { + Ok(Verified::Complete(_)) => Err(verified(row.status)), + Ok(Verified::Incomplete(_)) => Ok(()), + Err(e) => Err(format!("the configured store could not be read: {e}")), + } + } + BackupStatus::Available => match probe_manifest(calls, &arn).await { + Ok(Probe::Complete(fields)) => { + if fields.size_bytes == Some(row.size_bytes) + && fields.item_count == Some(row.item_count) + { + Ok(()) + } else { + Err(format!( + "the manifest under the configured root records {} bytes and {} items \ + and the row {} bytes and {} items: another backup under the same ARN", + fields.size_bytes.unwrap_or_default(), + fields.item_count.unwrap_or_default(), + row.size_bytes, + row.item_count + )) + } + } + Ok(Probe::Incomplete(why)) => Err(format!( + "not a complete copy under the configured root: {why}; copy the prefix again \ + (or remove the row with --forget-backup)" + )), + Err(e) => Err(format!("the configured store could not be read: {e}")), + }, + } +} diff --git a/crates/engine/src/backup/sync.rs b/crates/engine/src/backup/sync.rs index 0d8e8cb30..a671fa9eb 100644 --- a/crates/engine/src/backup/sync.rs +++ b/crates/engine/src/backup/sync.rs @@ -93,6 +93,16 @@ //! them. A later sync where the bytes are adopts them again. Without //! `--prune-missing` such rows stay AVAILABLE. //! +//! [`sync`] neither lists nor changes the rows of the other stores (another +//! kind or root, or the configured root under another store id): a prefix +//! whose backup id such a row holds meets it at the read under the ARN's +//! spellings or at the insert, and is skipped naming that row's status, +//! kind, root and store id. The command reports +//! those rows as a whole once a listing has reached its end +//! ([`super::store_maintenance::other_stores`]: the rows per status of +//! each other store, and each CREATING and DELETING row by ARN), with the +//! two options that act on them, `--move-store` and `--forget-store`. +//! //! The command takes the maintenance claim for every sync on PostgreSQL, //! MongoDB and Cassandra, where it runs beside servers (design 4(f)), and //! runs under the serve lock on SQLite; so no purge runs between its row diff --git a/crates/engine/src/backup/tests.rs b/crates/engine/src/backup/tests.rs index 8dc9c9f1a..06fd7587c 100644 --- a/crates/engine/src/backup/tests.rs +++ b/crates/engine/src/backup/tests.rs @@ -23,8 +23,8 @@ use extenddb_core::types::{ use extenddb_storage::backup_catalog::{ AbandonedLegacyBackup, BackupKey, BackupRow, BackupStatus, DeleteStart, ForgetRow, Heartbeat, JobClaim, JobRow, JobStatus, RestoreJobRow, RestoreKind, RestoreRegistrationRow, - RestoreTargetKey, StaleClock, StatusFields, StoreIdentity, StoreKind, TimeBounds, - UNOWNED_GENERATION, WriteOutcome, new_generation, + RestoreTargetKey, StaleClock, StatusEntrySweep, StatusFields, StoreIdentity, StoreKind, + StoreRootCount, TimeBounds, UNOWNED_GENERATION, WriteOutcome, new_generation, }; use extenddb_storage::backup_definition::{BACKUP_DEFINITION_VERSION, BackupTableDefinition}; use extenddb_storage::error::StorageError; @@ -329,7 +329,87 @@ struct Fake { /// ARN, and `read_backup_row` finds a row by its ARN; otherwise by /// backup id, as on PostgreSQL, SQLite and MongoDB. unique_by_arn: std::sync::atomic::AtomicBool, -} + /// Rows `rewrite_store` lands just before it decides, by ARN: a + /// CreateBackup of a server still on the old root whose row lands after + /// the move's checks, or another writer's change to a row the move + /// checked. + lands_before_rewrite: Mutex>, + /// When set, `rewrite_store` rewrites at most this many rows and then + /// fails, leaving the rest of the keys as they were: a move that stops + /// part way (MongoDB between chunks, Cassandra between keys). + fail_rewrite_after: Mutex>, + /// Every `rewrite_store` call: the old kind and root, the keys and the + /// new store. + rewrites_seen: Mutex>, + /// When set, `rewrite_store_runs_key_by_key` answers `true`, as + /// Cassandra's does. + rewrites_key_by_key: std::sync::atomic::AtomicBool, + /// When set, the `rewrite_store` call of this number (1 for the first) + /// sleeps for the duration after its rows are rewritten and before it + /// answers: a command stopped inside the rewrite. + rewrite_sleep: Mutex>, + /// When set, run (once) inside `list_backup_rows_older_than` of this + /// status and clock after it builds its answer: a maintenance command's + /// removal or move of a row landing between a duty's listing and its + /// deletion. + after_older_than: Mutex>, + /// When set, run (once) right after `read_backup_row` of this ARN + /// builds its answer: a maintenance command acting between a duty's + /// re-read of the row and what follows it. + after_row_read: Mutex>, + /// When set, run (once) right after the `n`th `check_maintenance_claim` + /// (counted over `maintenance_checks`) builds its answer: a command + /// taking the claim just after a reader found it free. + after_maintenance_check: Mutex>, + /// When set, `claim_dead_backup` hands over a CREATING row whose + /// `heartbeat_at` is older than its `stale_after` on the catalog's + /// clock, as the backends judge it, instead of the rows in `dead`. + dead_by_heartbeat: std::sync::atomic::AtomicBool, + /// When set, `list_backup_rows_by_root` fails once `rewrite_store` has + /// been called: the move's re-list of the old root failing. + by_root_error_after_rewrite: std::sync::atomic::AtomicBool, + /// When set, `read_backup_row` fails once `rewrite_store` has been + /// called: the move's read of a row the old root no longer lists + /// failing. + read_row_error_after_rewrite: std::sync::atomic::AtomicBool, + /// Rows `rewrite_store` removes just before it decides, by ARN: a + /// tombstone duty of a server on the old root removing a row the move + /// checked. + removed_before_rewrite: Mutex>, + /// When set, `remove_backup_row` waits this long before it removes + /// anything (a catalog write held on a lock); a call dropped during the + /// wait removes nothing. + remove_delay: Mutex>, + /// Status entries with no row behind them, as Cassandra can hold + /// (store and status): `read_store_ids` and `summarize_store_roots` + /// count them, and `remove_orphan_status_entries` deletes a root's. + orphan_entries: Mutex>, + /// What `summary_counts_status_entries` answers (Cassandra's `true`). + counts_entries: std::sync::atomic::AtomicBool, + /// Status entries a writer still configured with a root writes after + /// `remove_orphan_status_entries` read that root: counted by + /// `read_store_ids`, not seen by the sweep. + late_entries: Mutex>, + /// Rows whose one status entry is under an earlier status, by ARN, as + /// Cassandra holds after a status change whose entry batch failed (or + /// whose writer died before it): the entry's status, and the row's + /// status and store when the knob was set. While the row keeps that + /// status and store, `list_backup_rows` does not list it (its readers + /// read one status partition and skip the entry) and + /// `summarize_store_roots` counts it under the entry's status; + /// `list_backup_rows_by_root` lists it under its own, and + /// `remove_orphan_status_entries` of its root repairs the entry (or + /// removes it, once the row is gone or names another store). + stale_entries: Mutex>, + /// When set, `list_backup_rows_by_root` of this root fails. + by_root_error_for: Mutex>, + /// When set, `list_backup_rows_older_than` fails with it (every call). + older_than_error: Mutex>, +} + +/// One `rewrite_store` call the `Fake` received: the old kind and root, the +/// keys and the new store. +type RewriteCall = (StoreKind, String, Vec, StoreIdentity); /// A closure the `Fake` runs inside `sweep_restore_progress`. type SweepHook = Box; @@ -484,6 +564,27 @@ impl JobClaim for FakeRestoreClaim { } impl Fake { + /// The status of `row`'s stale status entry (`stale_entries`), while + /// the row keeps the status and store it had when the entry was set. + fn stale_entry(&self, row: &BackupRow) -> Option { + self.stale_entries + .lock() + .unwrap() + .get(&row.backup_arn) + .filter(|(_, status, store)| *status == row.status && *store == row.store) + .map(|(entry, _, _)| *entry) + } + + /// Leave the seeded row of `arn` with its one status entry under + /// `entry` (`stale_entries`). + fn set_stale_entry(&self, arn: &str, entry: BackupStatus) { + let row = self.rows.lock().unwrap()[arn].clone(); + self.stale_entries + .lock() + .unwrap() + .insert(arn.to_owned(), (entry, row.status, row.store)); + } + fn restore_claim_on(&self, table_id: &str, generation: u64) -> Box { Box::new(FakeRestoreClaim { records: Arc::clone(&self.restore_jobs), @@ -890,6 +991,11 @@ struct MemStore { panic_body_of_get: std::sync::atomic::AtomicUsize, /// Every `get`'s key, in order, failed ones included. gets: Mutex>, + /// When set, `delete_prefix_before` deletes nothing and answers that it + /// reached its deadline (a walk stopped at its next entry). + deadline_reached: std::sync::atomic::AtomicBool, + /// Every `delete_prefix_before`'s prefix and deadline, in order. + deadlines_seen: Mutex>, /// Called with the key of every `put` that stored its object, so a test /// can act at a point of the job (after the sidecar's put, before the /// claim re-check). @@ -1450,6 +1556,41 @@ impl extenddb_backup_store::BackupStore for MemStore { }) } + /// The trait's rule (the deadline checked once, then the whole + /// deletion); `deadline_reached` answers unfinished with nothing + /// deleted. + fn delete_prefix_before<'a>( + &'a self, + prefix: &'a str, + deadline: tokio::time::Instant, + ) -> BoxedFuture< + 'a, + Result, + > { + self.deadlines_seen + .lock() + .unwrap() + .push((prefix.to_owned(), deadline)); + Box::pin(async move { + if self + .deadline_reached + .load(std::sync::atomic::Ordering::Acquire) + || tokio::time::Instant::now() >= deadline + { + return Ok(extenddb_backup_store::PrefixDeletion { + deleted: 0, + finished: false, + }); + } + self.delete_prefix(prefix) + .await + .map(|deleted| extenddb_backup_store::PrefixDeletion { + deleted, + finished: true, + }) + }) + } + fn validate(&self) -> BoxedFuture<'_, Result<(), extenddb_backup_store::StoreError>> { Box::pin(async move { self.stall().await; @@ -1888,10 +2029,20 @@ impl BackupCatalog for Fake { fn claim_dead_backup<'a>( &'a self, key: &'a BackupKey, - _stale_after: Duration, + stale_after: Duration, ) -> BoxedFuture<'a, Result>, StorageError>> { - let dead = self.dead.lock().unwrap().contains(&key.backup_arn); let mut rows = self.rows.lock().unwrap(); + let dead = if self + .dead_by_heartbeat + .load(std::sync::atomic::Ordering::Acquire) + { + let cutoff = seconds_since_epoch() + *self.catalog_clock_offset_secs.lock().unwrap() + - stale_after.as_secs_f64(); + rows.get(&key.backup_arn) + .is_some_and(|row| row.heartbeat_at < cutoff) + } else { + self.dead.lock().unwrap().contains(&key.backup_arn) + }; let claim = match rows.get_mut(&key.backup_arn) { Some(row) if dead && row.status == BackupStatus::Creating => { row.owner_generation = new_generation(); @@ -1937,6 +2088,57 @@ impl BackupCatalog for Fake { .filter(|r| r.store == *store) .filter(|r| status.is_none_or(|s| r.status == s)) .filter(|r| !self.unlisted.lock().unwrap().contains(&r.backup_arn)) + .filter(|r| self.stale_entry(r).is_none()) + .cloned() + .collect(); + rows.sort_by(|a, b| { + a.created_at + .total_cmp(&b.created_at) + .then_with(|| a.backup_arn.cmp(&b.backup_arn)) + }); + Box::pin(futures::stream::iter(rows.into_iter().map(Ok))) + } + + /// The backend's listing of a root: rows of the kind and root, whatever + /// their id, of the status when given, in the order `list_backup_rows` + /// uses; `unlisted` rows and `list_error` as there. + fn list_backup_rows_by_root<'a>( + &'a self, + store_kind: StoreKind, + store_root: &'a str, + status: Option, + ) -> BoxStream<'a, Result> { + if self + .by_root_error_after_rewrite + .load(std::sync::atomic::Ordering::Acquire) + && !self.rewrites_seen.lock().unwrap().is_empty() + { + return Box::pin(futures::stream::once(async { + Err(StorageError::Internal( + "the root's listing failed".to_owned(), + )) + })); + } + if let Some(e) = self.list_error.lock().unwrap().clone() { + return Box::pin(futures::stream::once(async move { + Err(StorageError::Internal(e)) + })); + } + if self.by_root_error_for.lock().unwrap().as_deref() == Some(store_root) { + return Box::pin(futures::stream::once(async { + Err(StorageError::Internal( + "the root's listing failed".to_owned(), + )) + })); + } + let mut rows: Vec = self + .rows + .lock() + .unwrap() + .values() + .filter(|r| r.store.store_kind == store_kind && r.store.store_root == store_root) + .filter(|r| status.is_none_or(|s| r.status == s)) + .filter(|r| !self.unlisted.lock().unwrap().contains(&r.backup_arn)) .cloned() .collect(); rows.sort_by(|a, b| { @@ -1947,6 +2149,129 @@ impl BackupCatalog for Fake { Box::pin(futures::stream::iter(rows.into_iter().map(Ok))) } + /// The backends' rewrite: `lands_before_rewrite` lands first; then each + /// key's row that is still of the old kind and root and not CREATING + /// gets the new store, counted, until `fail_rewrite_after` rows are + /// rewritten, when the call fails with the rest left. + fn rewrite_store<'a>( + &'a self, + from_kind: StoreKind, + from_root: &'a str, + keys: &'a [BackupKey], + to: &'a StoreIdentity, + ) -> BoxedFuture<'a, Result> { + let call = { + let mut seen = self.rewrites_seen.lock().unwrap(); + seen.push((from_kind, from_root.to_owned(), keys.to_vec(), to.clone())); + seen.len() + }; + let sleep = self + .rewrite_sleep + .lock() + .unwrap() + .filter(|(n, _)| *n == call) + .map(|(_, d)| d); + let mut rows = self.rows.lock().unwrap(); + for landed in self.lands_before_rewrite.lock().unwrap().drain(..) { + rows.insert(landed.backup_arn.clone(), landed); + } + for removed in self.removed_before_rewrite.lock().unwrap().drain(..) { + rows.remove(&removed); + } + let limit = *self.fail_rewrite_after.lock().unwrap(); + let mut rewritten = 0_usize; + for key in keys { + let Some(row) = rows.get_mut(&key.backup_arn) else { + continue; + }; + if row.account_id != key.account_id + || row.store.store_kind != from_kind + || row.store.store_root != from_root + || row.status == BackupStatus::Creating + { + continue; + } + if limit.is_some_and(|limit| rewritten >= limit) { + return Box::pin(async move { + Err(StorageError::Internal(format!( + "the rewrite stopped after {rewritten} rows" + ))) + }); + } + row.store = to.clone(); + rewritten += 1; + } + Box::pin(async move { + if let Some(d) = sleep { + tokio::time::sleep(d).await; + } + Ok(rewritten as u64) + }) + } + + fn rewrite_store_runs_key_by_key(&self) -> bool { + self.rewrites_key_by_key + .load(std::sync::atomic::Ordering::Acquire) + } + + /// One count per store and status of the rows, sorted as the backends + /// sort them. + fn summarize_store_roots(&self) -> BoxedFuture<'_, Result, StorageError>> { + let mut seen: Vec<(StoreIdentity, BackupStatus)> = self + .rows + .lock() + .unwrap() + .values() + .map(|r| (r.store.clone(), self.stale_entry(r).unwrap_or(r.status))) + .collect(); + seen.extend(self.orphan_entries.lock().unwrap().iter().cloned()); + let counts = StoreRootCount::tally(seen); + Box::pin(async move { Ok(counts) }) + } + + fn summary_counts_status_entries(&self) -> bool { + self.counts_entries + .load(std::sync::atomic::Ordering::Acquire) + } + + /// Delete the root's entries of `orphan_entries`, and settle its + /// `stale_entries`: repaired while the row is of the root under + /// another status, removed once it is gone or names another store. + fn remove_orphan_status_entries<'a>( + &'a self, + store_kind: StoreKind, + store_root: &'a str, + ) -> BoxedFuture<'a, Result> { + let at_root = |store: &StoreIdentity| { + store.store_kind == store_kind && store.store_root == store_root + }; + let mut sweep = StatusEntrySweep::default(); + let mut entries = self.orphan_entries.lock().unwrap(); + let before = entries.len(); + entries.retain(|(store, _)| !at_root(store)); + sweep.removed = (before - entries.len()) as u64; + drop(entries); + let rows = self.rows.lock().unwrap(); + self.stale_entries + .lock() + .unwrap() + .retain(|arn, (entry, _, store)| { + if !at_root(store) { + return true; + } + match rows.get(arn) { + Some(row) if at_root(&row.store) => { + if row.status != *entry { + sweep.repaired += 1; + } + } + _ => sweep.removed += 1, + } + false + }); + Box::pin(async move { Ok(sweep) }) + } + /// The last AVAILABLE row of the store in the listing's order, read as /// one row; `list_error` fails it as it fails the listing. fn newest_available_backup_row<'a>( @@ -2003,6 +2328,13 @@ impl BackupCatalog for Fake { if let Some(e) = self.read_row_error.lock().unwrap().clone() { return Box::pin(async move { Err(StorageError::Internal(e)) }); } + if self + .read_row_error_after_rewrite + .load(std::sync::atomic::Ordering::Acquire) + && !self.rewrites_seen.lock().unwrap().is_empty() + { + return Box::pin(async { Err(StorageError::Internal("read timed out".to_owned())) }); + } let by_arn = self .unique_by_arn .load(std::sync::atomic::Ordering::Acquire); @@ -2021,6 +2353,15 @@ impl BackupCatalog for Fake { }) .filter(|r| r.table_name == arn.table_name() && r.backup_arn == arn.to_arn_string()) .cloned(); + let mut hook = self.after_row_read.lock().unwrap(); + if hook + .as_ref() + .is_some_and(|(a, _)| *a == arn.to_arn_string()) + { + let (_, run) = hook.take().expect("checked"); + drop(hook); + run(self); + } Box::pin(async move { Ok(found) }) } @@ -2077,18 +2418,32 @@ impl BackupCatalog for Fake { Box::pin(async move { Ok(moved) }) } - /// Remove a DELETED or FAILED tombstone, as the backends do. + /// Remove a DELETED or FAILED tombstone still of the store given, as + /// the backends do. fn remove_backup_row<'a>( &'a self, key: &'a BackupKey, + store_kind: StoreKind, + store_root: &'a str, ) -> BoxedFuture<'a, Result> { self.remove_attempts .lock() .unwrap() .push(key.backup_arn.clone()); + if let Some(delay) = *self.remove_delay.lock().unwrap() { + return Box::pin(async move { + tokio::time::sleep(delay).await; + *self.remove_delay.lock().unwrap() = None; + self.remove_backup_row(key, store_kind, store_root).await + }); + } let mut rows = self.rows.lock().unwrap(); let removed = match rows.get(&key.backup_arn) { - Some(row) if matches!(row.status, BackupStatus::Deleted | BackupStatus::Failed) => { + Some(row) + if matches!(row.status, BackupStatus::Deleted | BackupStatus::Failed) + && row.store.store_kind == store_kind + && row.store.store_root == store_root => + { rows.remove(&key.backup_arn); true } @@ -2170,6 +2525,11 @@ impl BackupCatalog for Fake { clock: StaleClock, age: Duration, ) -> BoxStream<'a, Result> { + if let Some(e) = self.older_than_error.lock().unwrap().clone() { + return Box::pin(futures::stream::once(async move { + Err(StorageError::Internal(e)) + })); + } let cutoff = seconds_since_epoch() + *self.catalog_clock_offset_secs.lock().unwrap() - age.as_secs_f64(); let mut rows: Vec = self @@ -2189,6 +2549,15 @@ impl BackupCatalog for Fake { .total_cmp(&b.created_at) .then_with(|| a.backup_arn.cmp(&b.backup_arn)) }); + let mut hook = self.after_older_than.lock().unwrap(); + if hook + .as_ref() + .is_some_and(|(s, c, _)| *s == status && *c == clock) + { + let (_, _, run) = hook.take().expect("checked"); + drop(hook); + run(self); + } Box::pin(futures::stream::iter(rows.into_iter().map(Ok))) } @@ -2376,6 +2745,31 @@ impl BackupCatalog for Fake { .filter(|row| row.store.store_kind == store_kind && row.store.store_root == store_root) .map(|row| row.store.store_id.clone()) .collect(); + ids.extend( + self.orphan_entries + .lock() + .unwrap() + .iter() + .filter(|(store, _)| { + store.store_kind == store_kind && store.store_root == store_root + }) + .map(|(store, _)| store.store_id.clone()), + ); + ids.extend( + self.late_entries + .lock() + .unwrap() + .iter() + .chain( + self.stale_entries + .lock() + .unwrap() + .values() + .map(|(_, _, s)| s), + ) + .filter(|store| store.store_kind == store_kind && store.store_root == store_root) + .map(|store| store.store_id.clone()), + ); ids.sort(); ids.dedup(); Box::pin(async move { Ok(ids) }) @@ -2414,12 +2808,22 @@ impl BackupCatalog for Fake { &self, stale_after: Duration, ) -> BoxedFuture<'_, Result> { - self.maintenance_checks.lock().unwrap().push(stale_after); + let checks = { + let mut checks = self.maintenance_checks.lock().unwrap(); + checks.push(stale_after); + checks.len() + }; let held = match *self.maintenance_beat_age.lock().unwrap() { Some(age) => age < stale_after, None => *self.maintenance_held.lock().unwrap(), }; let error = self.maintenance_error.lock().unwrap().clone(); + let mut hook = self.after_maintenance_check.lock().unwrap(); + if hook.as_ref().is_some_and(|(n, _)| *n == checks) { + let (_, run) = hook.take().expect("checked"); + drop(hook); + run(self); + } let delay = *self.maintenance_delay.lock().unwrap(); Box::pin(async move { if let Some(delay) = delay { @@ -8235,7 +8639,7 @@ async fn the_wait_reports_each_outcome_as_itself() { let (_tx, mut quiet) = Interrupt::new(); assert_eq!( - wait_for_quiet(&fake, &store(), &claim, wait, &mut quiet, announce).await, + wait_for_quiet(&fake, &[store()], &claim, wait, &mut quiet, announce).await, WaitOutcome::Quiet ); // A FAILED record whose target is still CREATING is listed by the @@ -8251,7 +8655,7 @@ async fn the_wait_reports_each_outcome_as_itself() { .insert(arn_n(3), row(&arn_n(3), BackupStatus::Failed)); let started = tokio::time::Instant::now(); assert_eq!( - wait_for_quiet(&fake, &store(), &claim, wait, &mut quiet, announce).await, + wait_for_quiet(&fake, &[store()], &claim, wait, &mut quiet, announce).await, WaitOutcome::Quiet, "a FAILED restore record and a FAILED row with a dead owner are not waited on" ); @@ -8272,7 +8676,7 @@ async fn the_wait_reports_each_outcome_as_itself() { .unwrap() .push(dead_job(RestoreKind::Legacy, "Target")); let started = tokio::time::Instant::now(); - let busy = wait_for_quiet(&fake, &store(), &claim, wait, &mut quiet, announce).await; + let busy = wait_for_quiet(&fake, &[store()], &claim, wait, &mut quiet, announce).await; let expected = vec![ format!( "CreateBackup {} has a live owner (heartbeat within stale_job_after_secs)", @@ -8296,7 +8700,7 @@ async fn the_wait_reports_each_outcome_as_itself() { }); let started = tokio::time::Instant::now(); assert_eq!( - wait_for_quiet(&fake, &store(), &claim, wait, &mut interrupted, announce).await, + wait_for_quiet(&fake, &[store()], &claim, wait, &mut interrupted, announce).await, WaitOutcome::Interrupted("SIGTERM") ); assert!( @@ -8307,14 +8711,14 @@ async fn the_wait_reports_each_outcome_as_itself() { // One already received ends it before the first read. assert_eq!(interrupted.signalled(), Some("SIGTERM")); assert_eq!( - wait_for_quiet(&fake, &store(), &claim, wait, &mut interrupted, announce).await, + wait_for_quiet(&fake, &[store()], &claim, wait, &mut interrupted, announce).await, WaitOutcome::Interrupted("SIGTERM") ); // The catalog unreadable. *fake.list_error.lock().unwrap() = Some("pool exhausted".to_owned()); assert_eq!( - wait_for_quiet(&fake, &store(), &claim, wait, &mut quiet, announce).await, + wait_for_quiet(&fake, &[store()], &claim, wait, &mut quiet, announce).await, WaitOutcome::CatalogError( "cannot list the catalog's backup rows: Internal error: pool exhausted".to_owned() ) @@ -8325,7 +8729,7 @@ async fn the_wait_reports_each_outcome_as_itself() { *holder.lost.lock().unwrap() = true; tokio::time::sleep(Duration::from_millis(1_500)).await; assert_eq!( - wait_for_quiet(&fake, &store(), &claim, wait, &mut quiet, announce).await, + wait_for_quiet(&fake, &[store()], &claim, wait, &mut quiet, announce).await, WaitOutcome::Lost( "the maintenance claim was lost (another holder took it over)".to_owned() ) @@ -8568,7 +8972,7 @@ async fn the_wait_reads_a_live_owner_on_the_catalogs_clock() { r }); assert_eq!( - wait_for_quiet(&fake, &store(), &claim, wait, &mut quiet, |_| {}).await, + wait_for_quiet(&fake, &[store()], &claim, wait, &mut quiet, |_| {}).await, WaitOutcome::Busy(vec![format!( "CreateBackup {} has a live owner (heartbeat within stale_job_after_secs)", arn_n(1) @@ -8584,7 +8988,7 @@ async fn the_wait_reads_a_live_owner_on_the_catalogs_clock() { .unwrap() .heartbeat_at = seconds_since_epoch(); assert_eq!( - wait_for_quiet(&fake, &store(), &claim, wait, &mut quiet, |_| {}).await, + wait_for_quiet(&fake, &[store()], &claim, wait, &mut quiet, |_| {}).await, WaitOutcome::Quiet, "an owner whose heartbeat is old on the catalog's clock is dead" ); @@ -8666,8 +9070,9 @@ async fn a_cycle_run_by_the_claim_holder_takes_the_held_branch_whatever_its_wind assert!(!report.maintenance_held); assert_eq!( *fake.maintenance_checks.lock().unwrap(), - [Duration::ZERO], - "a default state reads the claim through the window it is given" + [Duration::ZERO, Duration::ZERO], + "a default state reads the claim through the window it is given, at the cycle's start \ + and again before the FAILED row's deletion" ); assert_eq!( report.failed_prefixes_deleted, @@ -10710,7 +11115,7 @@ mod review_round_2 { poll: Duration::from_secs(2), }; let (_tx, mut quiet) = Interrupt::new(); - let outcome = wait_for_quiet(&fake, &store(), &claim, wait, &mut quiet, |_| {}).await; + let outcome = wait_for_quiet(&fake, &[store()], &claim, wait, &mut quiet, |_| {}).await; assert_eq!( outcome, WaitOutcome::Busy(vec![format!( @@ -10719,7 +11124,7 @@ mod review_round_2 { ); fake.running_jobs.lock().unwrap().clear(); assert_eq!( - wait_for_quiet(&fake, &store(), &claim, wait, &mut quiet, |_| {}).await, + wait_for_quiet(&fake, &[store()], &claim, wait, &mut quiet, |_| {}).await, WaitOutcome::Quiet ); claim.release().await.unwrap(); @@ -11351,8 +11756,9 @@ async fn a_store_backed_create_answers_creating_and_its_job_writes_the_backup() ); assert_eq!( fake.maintenance_checks.lock().unwrap().as_slice(), - [Duration::from_secs(300), Duration::from_secs(300)], - "the maintenance claim is read before the permit, by the refused request too" + [Duration::from_secs(300); 3], + "the maintenance claim is read before the permit, by the refused request too, and by \ + the job again before its insert" ); // Three heartbeat periods inside the stream, so the timer beats before @@ -12760,8 +13166,9 @@ async fn a_stopped_heartbeat_timer_starts_no_put() { ); } -/// The maintenance check runs before the permit and the row, so a slow -/// check delays the request and not the job's first put: with a period of +/// The maintenance checks (the request's before the permit, the job's right +/// before its insert) run before the row, so a slow check delays the +/// request and not the job's first put: with a period of /// 100 ms, `stale_job_after` of 300 ms (the floor's ratio) and a check of /// 400 ms, the request takes at least the check, the row is inserted after /// it (its heartbeat the insert's, within two periods of the first put), and @@ -12785,7 +13192,11 @@ async fn a_slow_maintenance_check_delays_the_request_and_not_the_first_put() { asked.elapsed() >= 4 * period, "the request waited for the check" ); - assert_eq!(fake.maintenance_checks.lock().unwrap().len(), 1); + assert_eq!( + fake.maintenance_checks.lock().unwrap().len(), + 2, + "the request's check and the job's before its insert" + ); wait_for_jobs(&runtime).await; let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); assert_eq!( @@ -13801,8 +14212,8 @@ async fn a_store_backed_restore_answers_creating_and_its_job_loads_the_backup() assert_eq!(record.spool_host, None, "a filesystem store has no spool"); assert_eq!( fake.maintenance_checks.lock().unwrap().as_slice(), - [job_settings().stale_job_after], - "the maintenance claim was read once, after the registration" + [job_settings().stale_job_after; 2], + "the maintenance claim was read before the registration and after it" ); wait_for_jobs(&runtime).await; @@ -14026,12 +14437,15 @@ async fn the_manifest_request_refuses_in_order_before_the_registration() { assert_eq!(runtime.running_jobs(), 0); } -/// The maintenance claim is checked after the registration, not before: a -/// held claim leaves a record that was FAILED `maintenance` from RUNNING -/// under the job's claim, its target removed, and the request answered -/// `LimitExceededException`; the registration ran (the row is what the -/// command's wait sees). A claim that cannot be read fails the record with -/// the read's error and answers it. +/// The maintenance claim is read immediately before the registration and +/// again after it. Held before, the request is answered +/// `LimitExceededException` and nothing is registered, so no record lands +/// while a command holds the claim. Taken between the read before and the +/// registration, the record is FAILED `maintenance` from RUNNING under the +/// job's claim, its target removed, and the request answered +/// `LimitExceededException` (the record is what the command's wait sees). +/// A claim that cannot be read after the registration fails the record +/// with the read's error and answers it. #[tokio::test] async fn a_manifest_restore_under_a_maintenance_claim_fails_its_record_after_registering() { let mem = Arc::new(MemStore::default()); @@ -14045,6 +14459,14 @@ async fn a_manifest_restore_under_a_maintenance_claim_fails_its_record_after_reg true, ); let runtime = job_runtime(&mem); + // Held before the registration. + let checks = |fake: &Fake| fake.maintenance_checks.lock().unwrap().len(); + let take_claim_after = |fake: &Fake, n: usize| { + *fake.after_maintenance_check.lock().unwrap() = Some(( + n, + Box::new(|fake: &Fake| *fake.maintenance_held.lock().unwrap() = true), + )); + }; *fake.maintenance_held.lock().unwrap() = true; let err = restore_on(&fake, &runtime, &restore_body(json!({}))) .await @@ -14054,6 +14476,24 @@ async fn a_manifest_restore_under_a_maintenance_claim_fails_its_record_after_reg "{err:?}" ); wait_for_jobs(&runtime).await; + assert!( + fake.restore_claims.lock().unwrap().is_empty(), + "nothing is registered while the claim is held" + ); + assert!(restore_writes(&fake).is_empty()); + assert_eq!(runtime.free_slots(), 4); + + // Taken after the read before the registration. + *fake.maintenance_held.lock().unwrap() = false; + take_claim_after(&fake, checks(&fake) + 1); + let err = restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::LimitExceededException(ref m) if m.contains("maintenance claim")), + "{err:?}" + ); + wait_for_jobs(&runtime).await; assert_eq!( fake.restore_claims.lock().unwrap().len(), 1, @@ -14100,7 +14540,13 @@ async fn a_manifest_restore_under_a_maintenance_claim_fails_its_record_after_reg ); *fake.maintenance_held.lock().unwrap() = false; - *fake.maintenance_error.lock().unwrap() = Some("pool exhausted".to_owned()); + let fail_after = checks(&fake) + 1; + *fake.after_maintenance_check.lock().unwrap() = Some(( + fail_after, + Box::new(|fake: &Fake| { + *fake.maintenance_error.lock().unwrap() = Some("pool exhausted".to_owned()); + }), + )); let err = restore_on(&fake, &runtime, &restore_body(json!({}))) .await .unwrap_err(); @@ -23866,3 +24312,3777 @@ async fn adopt_store_rechecks_the_claim_before_the_rewrite() { assert_eq!(beats_of(&beats), 1); assert_eq!(mem.puts.lock().unwrap().as_slice(), [STORE_IDENTITY_KEY]); } + +// --- backup sync --move-store, --forget-store, --forget-backup --------------- + +/// The root the rows were written under before the store moved. +fn old_store() -> StoreIdentity { + StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/srv/old".to_owned(), + store_id: "store-0".to_owned(), + } +} + +/// A row of `arn` under the old root. +fn old_row(arn: &str, status: BackupStatus) -> BackupRow { + let mut row = row(arn, status); + row.store = old_store(); + row +} + +fn seed_rows(fake: &Fake, rows: &[BackupRow]) { + let mut held = fake.rows.lock().unwrap(); + for row in rows { + held.insert(row.backup_arn.clone(), row.clone()); + } +} + +/// The stop of a run nothing stops. +fn never() -> Option { + None +} + +/// What a run told of its wait ignores. +fn no_wait(_: Duration) {} + +/// A forget run with `owner_window` and `stop` that waits for nothing +/// before its FAILED and DELETED removals (as on SQLite). +fn forget_run<'a>( + owner_window: Duration, + stop: super::sync::Stop<'a>, +) -> super::store_maintenance::ForgetRun<'a> { + super::store_maintenance::ForgetRun { + owner_window, + removals_from: None, + on_wait: &no_wait, + stop, + claim: None, + configured: None, + unmounted_root: None, + deletions_finished: &[], + } +} + +/// `--move-store /srv/old` into the store `runtime` is configured with, +/// with the servers' 300 s owner window. +async fn move_old(fake: &Fake, runtime: &BackupRuntime) -> super::store_maintenance::MoveReport { + let configured = runtime.configured_store().unwrap(); + let identity = configured.identity().unwrap(); + super::store_maintenance::move_store( + fake, + super::store_maintenance::MoveTarget { + configured, + identity: &identity, + from_kind: StoreKind::Filesystem, + from_root: "/srv/old", + owner_window: Duration::from_secs(300), + stop: &never, + claim: None, + }, + ) + .await + .unwrap() +} + +fn sorted(mut arns: Vec) -> Vec { + arns.sort(); + arns +} + +/// `--move-store` over a copied tree, one row of each status the move +/// resolves or carries: a dead CREATING row whose copied prefix is complete +/// is resolved AVAILABLE against the configured store; a DELETING row whose copy holds no verified manifest has its +/// deletion finished in the configured store (the copied leftover deleted) +/// and moves DELETED; DELETED and FAILED rows without a verified manifest +/// and a complete AVAILABLE row move. Every row then names the configured +/// store, which serves it (`is_store_of`), the rewrite is one call from the +/// old root, and the old root lists nothing. A reconciler cycle afterwards +/// (no claim) cleans the moved tombstones' leftovers in the configured +/// store and removes the tombstones past their retention. +#[tokio::test] +async fn move_store_resolves_checks_and_moves_each_status_of_the_old_root() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + + let resolved_available = seed_dead_creating_with_manifest(&fake, &mem, true); + fake.rows + .lock() + .unwrap() + .get_mut(&resolved_available) + .unwrap() + .store = old_store(); + let deleting = sync_arn(2); + let deleting_leftover = format!("{}data/000001.json.gz", prefix_of(&deleting)); + mem.insert(&deleting_leftover, b"x"); + let deleted = sync_arn(3); + let deleted_leftover = format!("{}data/000001.json.gz", prefix_of(&deleted)); + mem.insert(&deleted_leftover, b"x"); + let failed = sync_arn(4); + let failed_leftover = format!("{}data/000001.json.gz", prefix_of(&failed)); + mem.insert(&failed_leftover, b"x"); + let available = sync_arn(5); + put_backup(&mem, &available); + let mut rows = vec![ + old_row(&deleting, BackupStatus::Deleting), + old_row(&deleted, BackupStatus::Deleted), + old_row(&failed, BackupStatus::Failed), + old_row(&available, BackupStatus::Available), + ]; + for row in &mut rows[1..3] { + row.completed_at = Some(1.0); + } + seed_rows(&fake, &rows); + + let report = move_old(&fake, &runtime).await; + assert!(report.complete(), "{report:?}"); + assert_eq!(report.rows, 5); + assert_eq!( + report.resolved, + [(resolved_available.clone(), BackupStatus::Available)] + ); + assert_eq!(report.finished, std::slice::from_ref(&deleting)); + assert_eq!(report.rewritten, 5); + assert!(report.left.is_empty()); + assert!(report.gone.is_empty()); + let all = sorted(vec![ + resolved_available.clone(), + deleting.clone(), + deleted.clone(), + failed.clone(), + available.clone(), + ]); + assert_eq!(sorted(report.moved.clone()), all); + let configured = runtime.configured_store().unwrap(); + { + let held = fake.rows.lock().unwrap(); + for arn in &all { + assert_eq!(held[arn].store, store(), "{arn}"); + assert!(configured.is_store_of(&held[arn].store), "{arn}"); + } + assert_eq!(held[&resolved_available].status, BackupStatus::Available); + assert_eq!(held[&deleting].status, BackupStatus::Deleted); + assert_eq!(held[&available].status, BackupStatus::Available); + } + let calls = fake.rewrites_seen.lock().unwrap().clone(); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].0, StoreKind::Filesystem); + assert_eq!(calls[0].1, "/srv/old"); + assert_eq!(calls[0].3, store()); + assert!( + !mem.keys().contains(&deleting_leftover), + "the deletion was finished" + ); + assert!(mem.keys().contains(&deleted_leftover)); + assert!(mem.keys().contains(&failed_leftover)); + + // The configured store's reconciler cleans the moved tombstones. + let cycle = cycle_on(&fake, &runtime).await; + assert_eq!(cycle.store_healthy, Some(true)); + assert!(!mem.keys().contains(&deleted_leftover), "{cycle:?}"); + assert!(!mem.keys().contains(&failed_leftover), "{cycle:?}"); + let held = fake.rows.lock().unwrap(); + assert!( + !held.contains_key(&deleted), + "the DELETED tombstone is removed" + ); + assert!( + !held.contains_key(&failed), + "the FAILED tombstone is removed" + ); + assert_eq!(held[&available].status, BackupStatus::Available); +} + +/// Every row that stops the move is named in one pass and no row is +/// moved: a CREATING row with a live owner; a DELETING, a DELETED and a +/// FAILED row whose copied prefix holds a verified manifest (the +/// DELETING row's copy is not deleted); AVAILABLE rows with no copy, a data +/// file missing, a data file of another size, no sidecar, a manifest naming +/// another backup, and a manifest whose size is not the row's. The one +/// complete AVAILABLE row is not moved either, and `rewrite_store` is never +/// called. +#[tokio::test] +async fn move_store_names_every_row_that_stops_it_and_moves_none() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + + let live = sync_arn(1); + let deleting = sync_arn(2); + let deleted = sync_arn(3); + let failed = sync_arn(4); + for arn in [&deleting, &deleted, &failed] { + put_backup(&mem, arn); + } + let no_copy = sync_arn(5); + let missing_file = sync_arn(6); + let mut objects = backup_objects(&missing_file); + let dropped = objects + .iter() + .position(|(k, _)| k.contains("/data/")) + .unwrap(); + objects.remove(dropped); + for (key, bytes) in objects { + mem.insert(&key, &bytes); + } + let short_file = sync_arn(7); + for (key, mut bytes) in backup_objects(&short_file) { + if key.contains("/data/") && bytes.len() > 1 { + bytes.pop(); + } + mem.insert(&key, &bytes); + } + let no_sidecar = sync_arn(8); + for (key, bytes) in backup_objects(&no_sidecar) { + if !key.ends_with("extenddb-manifest.sha256") { + mem.insert(&key, &bytes); + } + } + let other_backup = sync_arn(9); + for (key, bytes) in backup_objects(&sync_arn(19)) { + mem.insert( + &key.replacen(&prefix_of(&sync_arn(19)), &prefix_of(&other_backup), 1), + &bytes, + ); + } + let other_size = sync_arn(10); + put_backup(&mem, &other_size); + let complete = sync_arn(11); + put_backup(&mem, &complete); + + let mut sized = old_row(&other_size, BackupStatus::Available); + sized.size_bytes += 1; + let rows = vec![ + old_row(&live, BackupStatus::Creating), + old_row(&deleting, BackupStatus::Deleting), + old_row(&deleted, BackupStatus::Deleted), + old_row(&failed, BackupStatus::Failed), + old_row(&no_copy, BackupStatus::Available), + old_row(&missing_file, BackupStatus::Available), + old_row(&short_file, BackupStatus::Available), + old_row(&no_sidecar, BackupStatus::Available), + old_row(&other_backup, BackupStatus::Available), + sized, + old_row(&complete, BackupStatus::Available), + ]; + seed_rows(&fake, &rows); + let keys_before = mem.keys(); + + let report = move_old(&fake, &runtime).await; + assert!(!report.complete()); + let why: HashMap = report + .stopping + .iter() + .map(|(row, why)| (row.backup_arn.clone(), why.clone())) + .collect(); + assert_eq!(report.stopping.len(), 10, "{why:#?}"); + assert!(why[&live].contains("live owner"), "{}", why[&live]); + for (arn, status) in [ + (&deleting, "DELETING"), + (&deleted, "DELETED"), + (&failed, "FAILED"), + ] { + assert!( + why[arn].starts_with(&format!( + "{status} whose prefix under the configured root holds a verified manifest" + )), + "{}", + why[arn] + ); + assert!(why[arn].contains("--forget-backup"), "{}", why[arn]); + // A deleted backup's remedy is the prefix's removal; the adoption + // route is named only as what brings it back. + let deleted_by_a_client = status != "FAILED"; + assert_eq!( + why[arn].contains("The backup was deleted: remove that prefix"), + deleted_by_a_client, + "{}", + why[arn] + ); + assert_eq!( + why[arn].contains("bringing back AVAILABLE a backup a client deleted"), + deleted_by_a_client, + "{}", + why[arn] + ); + assert_eq!( + why[arn].contains("let backup sync adopt the prefix"), + !deleted_by_a_client, + "{}", + why[arn] + ); + } + assert!(why[&no_copy].contains("no manifest"), "{}", why[&no_copy]); + assert!( + why[&missing_file].contains("is not in the store"), + "{}", + why[&missing_file] + ); + assert!( + why[&short_file].contains("bytes in the store"), + "{}", + why[&short_file] + ); + assert!( + why[&no_sidecar].contains("without its sidecar"), + "{}", + why[&no_sidecar] + ); + assert!( + why[&other_backup].contains("not the row's backup"), + "{}", + why[&other_backup] + ); + assert!( + why[&other_size].contains("another backup under the same ARN"), + "{}", + why[&other_size] + ); + assert!(!why.contains_key(&complete)); + assert!(report.moved.is_empty()); + assert!( + fake.rewrites_seen.lock().unwrap().is_empty(), + "nothing is rewritten after a stop" + ); + let held = fake.rows.lock().unwrap(); + for row in &rows { + assert_eq!(held[&row.backup_arn], *row, "{}", row.backup_arn); + } + assert_eq!(mem.keys(), keys_before, "no object was deleted"); +} + +/// The rewrite moves only what is still the old root's: a CreateBackup of a +/// server still on the old root whose CREATING row lands after the checks +/// is left there and reported (the move is not complete), and a rewrite +/// that stops part way leaves the rest under the old root, listed, with +/// the rows moved before the stop counted as moved. +#[tokio::test] +async fn rows_the_rewrite_does_not_move_are_listed_as_left() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let rows: Vec = (1..=3) + .map(|n| { + let arn = sync_arn(n); + put_backup(&mem, &arn); + old_row(&arn, BackupStatus::Available) + }) + .collect(); + seed_rows(&fake, &rows); + let landed = old_row(&sync_arn(4), BackupStatus::Creating); + fake.lands_before_rewrite + .lock() + .unwrap() + .push(landed.clone()); + + let report = move_old(&fake, &runtime).await; + assert!(report.stopping.is_empty(), "{:?}", report.stopping); + assert_eq!(report.rewritten, 3); + assert_eq!(report.moved.len(), 3); + assert_eq!( + report.left, + [super::store_maintenance::NamedRow { + status: BackupStatus::Creating, + backup_arn: landed.backup_arn.clone(), + account_id: ACCOUNT.to_owned(), + }] + ); + assert!(!report.complete()); + assert_eq!( + fake.rows.lock().unwrap()[&landed.backup_arn].store, + old_store() + ); + + // A rewrite that stops after one row. + let fake = Arc::new(Fake::default()); + seed_rows(&fake, &rows); + *fake.fail_rewrite_after.lock().unwrap() = Some(1); + let report = move_old(&fake, &runtime).await; + assert!(report.rewrite_error.is_some()); + assert_eq!(report.moved.len(), 1); + assert_eq!(report.left.len(), 2); + assert!(!report.complete()); +} + +/// The fence runs once before the rewrite: a root another catalog adopted +/// after the store's check gets no row, though every row passed its check. +#[tokio::test] +async fn move_store_on_a_root_another_catalog_adopted_moves_nothing() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let arn = sync_arn(1); + put_backup(&mem, &arn); + seed_rows(&fake, &[old_row(&arn, BackupStatus::Available)]); + mem.insert(STORE_IDENTITY_KEY, &adopted_file()); + + let report = move_old(&fake, &runtime).await; + assert!(report.stopping.is_empty(), "{:?}", report.stopping); + let stopped = report.stopped.clone().unwrap(); + assert!(stopped.contains("fence refused"), "{stopped}"); + assert!(fake.rewrites_seen.lock().unwrap().is_empty()); + assert_eq!(fake.rows.lock().unwrap()[&arn].store, old_store()); + assert_fenced(&runtime); +} + +/// Where rows are unique by ARN alone (Cassandra), a moved row whose +/// account and backup id a row of the configured root holds under another +/// spelling stops the move, naming that row: no account holds two rows of +/// one backup id. +#[tokio::test] +async fn move_store_refuses_a_second_row_of_one_backup_id() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + fake.unique_by_arn + .store(true, std::sync::atomic::Ordering::Release); + let runtime = runtime_on(&mem, store()); + let ours = sync_arn(1); + let other_spelling = ours.replace(REGION, "eu-west-1"); + put_backup(&mem, &other_spelling); + seed_rows( + &fake, + &[ + row(&ours, BackupStatus::Available), + old_row(&other_spelling, BackupStatus::Available), + ], + ); + + let report = move_old(&fake, &runtime).await; + assert_eq!(report.stopping.len(), 1, "{:?}", report.stopping); + assert_eq!(report.stopping[0].0.backup_arn, other_spelling); + assert!( + report.stopping[0].1.contains("two rows of backup id") + && report.stopping[0].1.contains(&ours), + "{}", + report.stopping[0].1 + ); + assert!(fake.rewrites_seen.lock().unwrap().is_empty()); +} + +/// The wait of `--move-store` and `--forget-store` covers the old root's +/// store ids as well as the configured store's: a live CreateBackup of a +/// server still configured with the old root is waited on. +#[tokio::test(start_paused = true)] +async fn the_wait_covers_every_store_it_is_given() { + use maintenance::{Interrupt, Wait, WaitOutcome, wait_for_quiet}; + let fake = Fake::default(); + let mut live = old_row(&sync_arn(1), BackupStatus::Creating); + live.heartbeat_at = seconds_since_epoch(); + seed_rows(&fake, &[live.clone()]); + let holder = Arc::new(FakeHolder::default()); + let claim = held(&holder); + let wait = Wait { + owner_window: Duration::from_secs(300), + limit: Duration::ZERO, + poll: Duration::from_secs(1), + }; + let (_tx, mut quiet) = Interrupt::new(); + assert_eq!( + wait_for_quiet(&fake, &[store()], &claim, wait, &mut quiet, |_| {}).await, + WaitOutcome::Quiet + ); + assert_eq!( + wait_for_quiet( + &fake, + &[store(), old_store()], + &claim, + wait, + &mut quiet, + |_| {} + ) + .await, + WaitOutcome::Busy(vec![format!( + "CreateBackup {} has a live owner (heartbeat within stale_job_after_secs)", + live.backup_arn + )]) + ); + assert_eq!( + wait_for_quiet(&fake, &[], &claim, wait, &mut quiet, |_| {}).await, + WaitOutcome::Quiet + ); +} + +/// The root an option names: the configured root is refused; a root with +/// no row, of either kind, is refused with the stores the catalog knows, +/// a root that differs from one by a trailing `/` or by the `?endpoint=` +/// suffix among them; an S3 label's rows are found as S3 rows, with their +/// store ids. +#[tokio::test] +async fn locate_root_finds_the_kind_and_ids_and_refuses_the_rest() { + use super::store_maintenance::{RootRefusal, locate_root}; + let fake = Fake::default(); + let s3_root = "s3://bucket/prefix?endpoint=http://127.0.0.1:9000".to_owned(); + let mut s3_row = row(&sync_arn(1), BackupStatus::Available); + s3_row.store = StoreIdentity { + store_kind: StoreKind::S3, + store_root: s3_root.clone(), + store_id: "s3-id".to_owned(), + }; + seed_rows( + &fake, + &[ + s3_row, + old_row(&sync_arn(2), BackupStatus::Failed), + row(&sync_arn(3), BackupStatus::Available), + ], + ); + let configured = store().store_root; + assert!(matches!( + locate_root(&fake, &configured, Some(&configured)).await, + Err(RootRefusal::ConfiguredRoot) + )); + let found = locate_root(&fake, &s3_root, Some(&configured)) + .await + .unwrap(); + assert_eq!(found.kind, StoreKind::S3); + assert_eq!(found.store_ids, ["s3-id"]); + let found = locate_root(&fake, "/srv/old", None).await.unwrap(); + assert_eq!(found.kind, StoreKind::Filesystem); + assert_eq!(found.identities(), [old_store()]); + for absent in ["/srv/old/", "s3://bucket/prefix", "/srv/none"] { + match locate_root(&fake, absent, Some(&configured)).await { + Err(RootRefusal::NoRows(known)) => { + let roots: Vec<&str> = known.iter().map(|c| c.store.store_root.as_str()).collect(); + assert_eq!( + roots, + ["/srv/old", "/var/lib/extenddb/backups", s3_root.as_str()], + "{absent}" + ); + } + other => panic!("{absent}: {other:?}"), + } + } +} + +/// `--forget-store`: every row of the root is removed outright, each status +/// by its removal (no DELETED tombstone is left, and no object is deleted), +/// except a row a restore is reading, a FAILED row and a CREATING row whose +/// owner is alive, which are left and reported; the configured store's row +/// is untouched. Once the root is configured again, a reconciler cycle +/// deletes none of its bytes: no row is left for a duty to act on. +#[tokio::test] +async fn forget_store_removes_the_roots_rows_outright_and_reports_what_it_left() { + use super::store_maintenance::{Forgotten, forget_store}; + let fake = Arc::new(Fake::default()); + let now = seconds_since_epoch(); + let available = sync_arn(1); + let in_use = sync_arn(2); + let deleting = sync_arn(3); + let deleted = sync_arn(4); + let failed = sync_arn(5); + let failed_live = sync_arn(6); + let creating = sync_arn(7); + let creating_live = sync_arn(8); + let ours = sync_arn(9); + let mut rows = vec![ + old_row(&available, BackupStatus::Available), + old_row(&in_use, BackupStatus::Available), + old_row(&deleting, BackupStatus::Deleting), + old_row(&deleted, BackupStatus::Deleted), + old_row(&failed, BackupStatus::Failed), + old_row(&failed_live, BackupStatus::Failed), + old_row(&creating, BackupStatus::Creating), + old_row(&creating_live, BackupStatus::Creating), + ]; + rows[5].heartbeat_at = now; + rows[7].heartbeat_at = now; + for row in &mut rows { + row.completed_at = Some(1.0); + } + seed_rows(&fake, &rows); + seed_rows(&fake, &[row(&ours, BackupStatus::Available)]); + fake.forget_answers + .lock() + .unwrap() + .insert(in_use.clone(), ForgetRow::InUse("Target".to_owned())); + let old_mem = Arc::new(MemStore::default()); + for arn in [&available, &in_use, &deleting, &deleted, &failed, &creating] { + put_backup(&old_mem, arn); + } + + let report = forget_store( + &*fake, + StoreKind::Filesystem, + "/srv/old", + forget_run(Duration::from_secs(300), &never), + ) + .await + .unwrap(); + let outcome: HashMap = report + .rows + .iter() + .map(|(row, f)| (row.backup_arn.clone(), f.clone())) + .collect(); + assert_eq!( + outcome[&available], + Forgotten::Removed(BackupStatus::Available) + ); + assert_eq!( + outcome[&deleting], + Forgotten::Removed(BackupStatus::Deleting) + ); + assert_eq!(outcome[&deleted], Forgotten::Removed(BackupStatus::Deleted)); + assert_eq!(outcome[&failed], Forgotten::Removed(BackupStatus::Failed)); + assert_eq!( + outcome[&creating], + Forgotten::Removed(BackupStatus::Creating) + ); + assert_eq!( + outcome[&in_use], + Forgotten::InUse("Target".to_owned(), ACCOUNT.to_owned()) + ); + assert_eq!( + outcome[&failed_live], + Forgotten::LiveOwner(BackupStatus::Failed) + ); + assert_eq!( + outcome[&creating_live], + Forgotten::LiveOwner(BackupStatus::Creating) + ); + assert_eq!(report.removed(), 5); + assert_eq!( + report + .rows_left + .as_ref() + .map(|left| sorted(left.iter().map(|r| r.backup_arn.clone()).collect())), + Some(sorted(vec![ + in_use.clone(), + failed_live.clone(), + creating_live.clone() + ])) + ); + assert!(!report.complete()); + { + let held = fake.rows.lock().unwrap(); + for arn in [&available, &deleting, &deleted, &failed, &creating] { + assert!(!held.contains_key(arn), "{arn}: no tombstone is left"); + } + assert_eq!(held[&ours].store, store()); + assert_eq!(held[&ours].status, BackupStatus::Available); + } + + // The store's return: no duty acts on its bytes. + let runtime = runtime_on(&old_mem, old_store()); + let keys_before = old_mem.keys(); + let cycle = cycle_on(&fake, &runtime).await; + assert_eq!(cycle.store_healthy, Some(true), "{cycle:?}"); + assert_eq!(old_mem.keys(), keys_before); +} + +/// `--forget-backup`, in the order given, up to the first ARN whose row is +/// not removed: an AVAILABLE, a DELETING, a dead FAILED and a dead CREATING +/// row are removed outright; a CREATING row with a live owner stops the +/// run, and the ARN after it is not reached. Alone, each refusal leaves its +/// row: a backup a restore target is reading, a FAILED row whose owner is +/// alive, a row under another spelling of the ARN (named), an earlier +/// release's in-catalog backup (said so, nothing changed), an ARN with no +/// row and a string that is no ARN. +#[tokio::test] +async fn forget_backup_removes_each_row_in_order_and_stops_at_the_first_refusal() { + use super::store_maintenance::{ForgetBackup, Forgotten, forget_backups}; + let fake = Arc::new(Fake::default()); + let now = seconds_since_epoch(); + let [available, deleting, failed, creating, live, after] = [1, 2, 3, 4, 5, 6].map(sync_arn); + let mut live_row = old_row(&live, BackupStatus::Creating); + live_row.heartbeat_at = now; + seed_rows( + &fake, + &[ + row(&available, BackupStatus::Available), + old_row(&deleting, BackupStatus::Deleting), + row(&failed, BackupStatus::Failed), + old_row(&creating, BackupStatus::Creating), + live_row, + row(&after, BackupStatus::Available), + ], + ); + let asked = [&available, &deleting, &failed, &creating, &live, &after].map(|a| a.clone()); + let report = forget_backups( + &*fake, + &asked, + REGION, + forget_run(Duration::from_secs(300), &never), + ) + .await; + assert_eq!( + report.done, + [ + ( + available.clone(), + ForgetBackup::Row( + BackupStatus::Available, + Forgotten::Removed(BackupStatus::Available) + ) + ), + ( + deleting.clone(), + ForgetBackup::Row( + BackupStatus::Deleting, + Forgotten::Removed(BackupStatus::Deleting) + ) + ), + ( + failed.clone(), + ForgetBackup::Row( + BackupStatus::Failed, + Forgotten::Removed(BackupStatus::Failed) + ) + ), + ( + creating.clone(), + ForgetBackup::Row( + BackupStatus::Creating, + Forgotten::Removed(BackupStatus::Creating) + ) + ), + ( + live.clone(), + ForgetBackup::Row( + BackupStatus::Creating, + Forgotten::LiveOwner(BackupStatus::Creating) + ) + ), + ] + ); + assert!(!report.complete(asked.len())); + { + let held = fake.rows.lock().unwrap(); + for arn in [&available, &deleting, &failed, &creating] { + assert!(!held.contains_key(arn), "{arn}"); + } + assert!(held.contains_key(&live)); + assert!(held.contains_key(&after), "not reached"); + } + + let one = |arn: &str| vec![arn.to_owned()]; + let in_use = sync_arn(7); + let failed_live = sync_arn(8); + let stored = sync_arn(9); + let legacy = sync_arn(10); + let mut live_failed = row(&failed_live, BackupStatus::Failed); + live_failed.heartbeat_at = now; + seed_rows( + &fake, + &[ + row(&in_use, BackupStatus::Available), + live_failed, + row(&stored, BackupStatus::Available), + ], + ); + fake.forget_answers + .lock() + .unwrap() + .insert(in_use.clone(), ForgetRow::InUse("Target".to_owned())); + fake.descriptions.lock().unwrap().insert( + legacy.clone(), + backup_description("AVAILABLE", "PAY_PER_REQUEST"), + ); + let written = stored.replace(REGION, "eu-west-1"); + for (asked, expected) in [ + ( + in_use.clone(), + ForgetBackup::Row( + BackupStatus::Available, + Forgotten::InUse("Target".to_owned(), ACCOUNT.to_owned()), + ), + ), + ( + failed_live.clone(), + ForgetBackup::Row( + BackupStatus::Failed, + Forgotten::LiveOwner(BackupStatus::Failed), + ), + ), + ( + written.clone(), + ForgetBackup::OtherSpelling(stored.clone(), BackupStatus::Available), + ), + (legacy.clone(), ForgetBackup::Legacy), + ] { + let report = forget_backups( + &*fake, + &one(&asked), + REGION, + forget_run(Duration::from_secs(300), &never), + ) + .await; + assert_eq!(report.done, [(asked.clone(), expected)], "{asked}"); + assert!(!report.complete(1)); + } + let report = forget_backups( + &*fake, + &one("not-an-arn"), + REGION, + forget_run(Duration::from_secs(300), &never), + ) + .await; + assert!( + matches!(&report.done[0].1, ForgetBackup::Invalid(_)), + "{:?}", + report.done + ); + let none = forget_backups( + &*fake, + &one(&sync_arn(11)), + REGION, + forget_run(Duration::from_secs(300), &never), + ) + .await; + assert_eq!(none.done, [(sync_arn(11), ForgetBackup::NoRow)]); + assert!(none.complete(1), "an ARN no row carries counts as done"); + let held = fake.rows.lock().unwrap(); + for arn in [&in_use, &failed_live, &stored] { + assert!(held.contains_key(arn), "{arn}"); + } + assert!(fake.descriptions.lock().unwrap().contains_key(&legacy)); +} + +/// A stop asked before the first ARN or row changes nothing. +#[tokio::test] +async fn a_stop_before_the_first_row_forgets_nothing() { + use super::store_maintenance::{forget_backups, forget_store}; + let fake = Arc::new(Fake::default()); + let arn = sync_arn(1); + seed_rows(&fake, &[old_row(&arn, BackupStatus::Available)]); + let stop = || Some("SIGINT received".to_owned()); + let report = forget_backups( + &*fake, + std::slice::from_ref(&arn), + REGION, + forget_run(Duration::ZERO, &stop), + ) + .await; + assert_eq!(report.stopped.as_deref(), Some("SIGINT received")); + assert!(report.done.is_empty()); + let report = forget_store( + &*fake, + StoreKind::Filesystem, + "/srv/old", + forget_run(Duration::ZERO, &stop), + ) + .await + .unwrap(); + assert_eq!(report.stopped.as_deref(), Some("SIGINT received")); + assert!(report.rows.is_empty()); + assert!(fake.rows.lock().unwrap().contains_key(&arn)); + assert!(fake.forgets_seen.lock().unwrap().is_empty()); +} + +/// A restored catalog whose newest AVAILABLE row's backup was deleted after +/// the dump: the store's check fails naming it ("manifest probe failed"); +/// `--forget-backup` of that ARN removes the row, and the next check passes +/// on the next-newest row, whose manifest is there. +#[tokio::test] +async fn forget_backup_of_the_newest_rows_deleted_backup_heals_the_store() { + use super::store_maintenance::forget_backups; + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let kept = sync_arn(1); + let gone = sync_arn(2); + put_backup(&mem, &kept); + seed_rows( + &fake, + &[ + row(&kept, BackupStatus::Available), + row(&gone, BackupStatus::Available), + ], + ); + let runtime = runtime_on(&mem, store()); + let configured = runtime.configured_store().unwrap(); + let check = super::check_configured_store(configured, &*fake, "test").await; + assert!( + matches!(&check, Err(StoreProblem::ManifestMissing { backup_arn }) if *backup_arn == gone), + "{check:?}" + ); + configured.record_check(check); + assert!(!configured.is_healthy()); + + let report = forget_backups( + &*fake, + std::slice::from_ref(&gone), + REGION, + forget_run(Duration::from_secs(300), &never), + ) + .await; + assert!(report.complete(1), "{:?}", report.done); + let check = super::check_configured_store(configured, &*fake, "test").await; + assert_eq!(check.unwrap(), store()); + assert!(fake.rows.lock().unwrap().contains_key(&kept)); +} + +/// A deletion duty that listed a row before a maintenance command removed +/// it, or moved it to another store, acts on nothing (the row is read once +/// more after the fence): the FAILED-prefix duty keeps the prefix of a row +/// `--forget-backup` removed after the listing, and the tombstone duty +/// keeps a row `--move-store` moved after the listing. +#[tokio::test] +async fn a_duty_whose_row_is_forgotten_or_moved_after_its_listing_acts_on_nothing() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let forgotten = sync_arn(1); + let moved = sync_arn(2); + let leftover = format!("{}data/000001.json.gz", prefix_of(&forgotten)); + mem.insert(&leftover, b"x"); + seed_rows(&fake, &[row(&forgotten, BackupStatus::Failed)]); + let gone = forgotten.clone(); + *fake.after_older_than.lock().unwrap() = Some(( + BackupStatus::Failed, + StaleClock::Heartbeat, + Box::new(move |fake: &Fake| { + fake.rows.lock().unwrap().remove(&gone); + }), + )); + let cycle = cycle_on(&fake, &runtime).await; + assert!(cycle.failed_prefixes_deleted.is_empty(), "{cycle:?}"); + assert!( + mem.keys().contains(&leftover), + "the forgotten row's bytes are kept" + ); + + let mut tombstone = row(&moved, BackupStatus::Deleted); + tombstone.completed_at = Some(1.0); + seed_rows(&fake, &[tombstone]); + let elsewhere = moved.clone(); + *fake.after_older_than.lock().unwrap() = Some(( + BackupStatus::Deleted, + StaleClock::Completed, + Box::new(move |fake: &Fake| { + fake.rows.lock().unwrap().get_mut(&elsewhere).unwrap().store = old_store(); + }), + )); + let cycle = cycle_on(&fake, &runtime).await; + assert!(cycle.tombstones_removed.is_empty(), "{cycle:?}"); + assert_eq!(fake.rows.lock().unwrap()[&moved].store, old_store()); + assert!( + fake.after_older_than.lock().unwrap().is_none(), + "the hook ran" + ); +} + +/// Plain sync's report of other stores: per store other than the +/// configured one (another root, another kind, the configured root under +/// another id), the rows per status and each CREATING and DELETING row by +/// ARN; the configured store's rows are not in it. A reconciler cycle of +/// the configured store deletes none of the other stores' prefixes. +#[tokio::test] +async fn the_report_of_other_stores_counts_and_names_their_rows_and_nothing_acts_on_them() { + use super::store_maintenance::{NamedRow, other_stores}; + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let s3 = StoreIdentity { + store_kind: StoreKind::S3, + store_root: "s3://bucket/p".to_owned(), + store_id: "s3-id".to_owned(), + }; + let other_id = StoreIdentity { + store_id: "store-9".to_owned(), + ..store() + }; + let mut s3_failed = row(&sync_arn(5), BackupStatus::Failed); + s3_failed.store = s3.clone(); + let mut other_id_row = row(&sync_arn(6), BackupStatus::Available); + other_id_row.store = other_id.clone(); + let mut old_failed = old_row(&sync_arn(3), BackupStatus::Failed); + old_failed.completed_at = Some(1.0); + let mut old_deleted = old_row(&sync_arn(4), BackupStatus::Deleted); + old_deleted.completed_at = Some(1.0); + seed_rows( + &fake, + &[ + row(&sync_arn(10), BackupStatus::Available), + row(&sync_arn(11), BackupStatus::Creating), + old_row(&sync_arn(1), BackupStatus::Creating), + old_row(&sync_arn(2), BackupStatus::Deleting), + old_failed, + old_deleted, + old_row(&sync_arn(7), BackupStatus::Available), + old_row(&sync_arn(8), BackupStatus::Available), + s3_failed, + other_id_row, + ], + ); + for n in [3, 4, 5] { + mem.insert( + &format!("{}data/000001.json.gz", prefix_of(&sync_arn(n))), + b"x", + ); + } + put_backup(&mem, &sync_arn(10)); + + let others = other_stores(&*fake, &store()).await.unwrap(); + let stores: Vec<&StoreIdentity> = others.iter().map(|o| &o.store).collect(); + assert_eq!(stores, [&old_store(), &other_id, &s3]); + assert_eq!( + others[0].counts, + [ + (BackupStatus::Creating, 1), + (BackupStatus::Available, 2), + (BackupStatus::Deleting, 1), + (BackupStatus::Deleted, 1), + (BackupStatus::Failed, 1), + ] + ); + let named = |status, n| NamedRow { + status, + backup_arn: sync_arn(n), + account_id: ACCOUNT.to_owned(), + }; + assert_eq!( + others[0].unresolved, + [ + named(BackupStatus::Creating, 1), + named(BackupStatus::Deleting, 2) + ] + ); + assert_eq!(others[1].counts, [(BackupStatus::Available, 1)]); + assert!(others[1].unresolved.is_empty()); + assert_eq!(others[2].counts, [(BackupStatus::Failed, 1)]); + + // The configured root under a second id would make the store unhealthy; + // the cycle is about the other roots' prefixes. + fake.rows.lock().unwrap().remove(&sync_arn(6)); + let keys_before = mem.keys(); + let cycle = cycle_on(&fake, &runtime).await; + assert_eq!(cycle.store_healthy, Some(true)); + assert_eq!( + mem.keys(), + keys_before, + "no prefix of another store is deleted" + ); +} + +/// A dead CREATING row of the old root whose copy under the configured root +/// holds no complete backup stops the move, and the move writes no status +/// for it: the copy proves nothing about the old root's prefix, and a FAILED +/// row would have the reconciler delete the copy (and a server still on the +/// old root the original). Each shape of copy: none at all, a data file +/// missing, one short, one long, no sidecar, a manifest of another backup. +/// Each row is named with the reason and the two ways out, keeps its +/// status, heartbeat and store, and nothing is rewritten or deleted. +#[tokio::test] +async fn a_dead_creating_row_whose_copy_is_incomplete_stops_the_move_unwritten() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let no_copy = sync_arn(1); + let missing = sync_arn(2); + let short = sync_arn(3); + let long = sync_arn(4); + let no_sidecar = sync_arn(5); + let other = sync_arn(6); + let mut objects = backup_objects(&missing); + let dropped = objects + .iter() + .position(|(k, _)| k.contains("/data/")) + .unwrap(); + objects.remove(dropped); + for (key, bytes) in objects { + mem.insert(&key, &bytes); + } + for (key, mut bytes) in backup_objects(&short) { + if key.contains("/data/") && bytes.len() > 1 { + bytes.pop(); + } + mem.insert(&key, &bytes); + } + for (key, mut bytes) in backup_objects(&long) { + if key.contains("/data/") { + bytes.push(0); + } + mem.insert(&key, &bytes); + } + for (key, bytes) in backup_objects(&no_sidecar) { + if !key.ends_with("extenddb-manifest.sha256") { + mem.insert(&key, &bytes); + } + } + for (key, bytes) in backup_objects(&sync_arn(19)) { + mem.insert( + &key.replacen(&prefix_of(&sync_arn(19)), &prefix_of(&other), 1), + &bytes, + ); + } + let arns = [&no_copy, &missing, &short, &long, &no_sidecar, &other]; + let rows: Vec = arns + .iter() + .map(|arn| { + let mut row = old_row(arn, BackupStatus::Creating); + row.heartbeat_at = 1.0; + fake.dead.lock().unwrap().insert((*arn).clone()); + row + }) + .collect(); + seed_rows(&fake, &rows); + let keys_before = mem.keys(); + + let report = move_old(&fake, &runtime).await; + assert!(!report.complete()); + assert!(report.resolved.is_empty(), "{:?}", report.resolved); + let why: HashMap = report + .stopping + .iter() + .map(|(row, why)| (row.backup_arn.clone(), why.clone())) + .collect(); + assert_eq!(report.stopping.len(), 6, "{why:#?}"); + for (arn, detail) in [ + (&no_copy, "no manifest"), + (&missing, "is not in the store"), + (&short, "bytes in the store"), + (&long, "bytes in the store"), + (&no_sidecar, "without its sidecar"), + (&other, "not the row's backup"), + ] { + let why = &why[arn]; + assert!( + why.starts_with("CREATING, its owner dead, and the copy under the configured root") + && why.contains(detail) + && why.contains("the move writes no status for it") + && why.contains("configured with the old root") + && why.contains("--forget-backup"), + "{arn}: {why}" + ); + } + assert!(fake.rewrites_seen.lock().unwrap().is_empty()); + let held = fake.rows.lock().unwrap(); + for row in &rows { + let now = &held[&row.backup_arn]; + assert_eq!( + ( + now.status, + now.heartbeat_at, + &now.store, + &now.failure_reason + ), + (BackupStatus::Creating, 1.0, &old_store(), &None), + "{}", + row.backup_arn + ); + } + drop(held); + assert_eq!(mem.keys(), keys_before, "no object was deleted"); +} + +/// The owner window the move is given is the one the takeover reads: a +/// CREATING row whose last beat is 100 s old has a live owner under the +/// servers' 300 s window and stops the move as such, and is resolved under +/// SQLite's zero window (the serve lock held, no job of another process can +/// be alive), its complete copy making it AVAILABLE. +#[tokio::test] +async fn the_moves_owner_window_is_the_one_the_takeover_reads() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + fake.dead_by_heartbeat + .store(true, std::sync::atomic::Ordering::Release); + let runtime = runtime_on(&mem, store()); + let arn = seed_dead_creating_with_manifest(&fake, &mem, true); + { + let mut rows = fake.rows.lock().unwrap(); + let row = rows.get_mut(&arn).unwrap(); + row.store = old_store(); + row.heartbeat_at = seconds_since_epoch() - 100.0; + } + let report = move_old(&fake, &runtime).await; + assert_eq!(report.stopping.len(), 1, "{report:?}"); + assert!(report.stopping[0].1.contains("live owner"), "{report:?}"); + assert_eq!( + fake.rows.lock().unwrap()[&arn].status, + BackupStatus::Creating + ); + + let configured = runtime.configured_store().unwrap(); + let identity = configured.identity().unwrap(); + let report = super::store_maintenance::move_store( + &*fake, + super::store_maintenance::MoveTarget { + configured, + identity: &identity, + from_kind: StoreKind::Filesystem, + from_root: "/srv/old", + owner_window: Duration::ZERO, + stop: &never, + claim: None, + }, + ) + .await + .unwrap(); + assert!(report.complete(), "{report:?}"); + assert_eq!(report.resolved, [(arn.clone(), BackupStatus::Available)]); + let row = fake.rows.lock().unwrap()[&arn].clone(); + assert_eq!((row.status, row.store), (BackupStatus::Available, store())); +} + +/// The manifest's item total must be the row's too: a copy whose manifest +/// records the row's size and another item total is another backup under +/// the same ARN, and stops the move. +#[tokio::test] +async fn an_available_row_whose_manifest_counts_other_items_stops_the_move() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let arn = sync_arn(1); + put_backup(&mem, &arn); + let mut row = old_row(&arn, BackupStatus::Available); + row.item_count += 1; + seed_rows(&fake, &[row]); + let report = move_old(&fake, &runtime).await; + assert_eq!(report.stopping.len(), 1, "{report:?}"); + assert!( + report.stopping[0] + .1 + .contains("another backup under the same ARN"), + "{}", + report.stopping[0].1 + ); + assert!(fake.rewrites_seen.lock().unwrap().is_empty()); +} + +/// Where rows are unique by ARN alone (Cassandra), two rows of one backup +/// id among the old root's rows (two spellings of the ARN, two FAILED rows +/// with no copy, each of which passes its own check) stop the move: each +/// is named with the other. +#[tokio::test] +async fn two_rows_of_one_backup_id_among_the_moved_stop_the_move() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + fake.unique_by_arn + .store(true, std::sync::atomic::Ordering::Release); + let runtime = runtime_on(&mem, store()); + let ours = sync_arn(1); + let other_spelling = ours.replace(REGION, "eu-west-1"); + seed_rows( + &fake, + &[ + old_row(&ours, BackupStatus::Failed), + old_row(&other_spelling, BackupStatus::Failed), + ], + ); + let report = move_old(&fake, &runtime).await; + assert_eq!(report.stopping.len(), 2, "{:?}", report.stopping); + let both = sorted(vec![ours.clone(), other_spelling.clone()]); + for ((row, why), (arn, other)) in report + .stopping + .iter() + .zip([(&both[0], &both[1]), (&both[1], &both[0])]) + { + assert_eq!(&row.backup_arn, arn); + assert!( + why.contains("two rows of backup id") && why.contains(other.as_str()), + "{why}" + ); + } + assert!(fake.rewrites_seen.lock().unwrap().is_empty()); +} + +/// The stop is asked before each row: a stop that fires after the first +/// row's check leaves the second unchecked (its dead CREATING row not taken +/// over, its DELETING row's deletion not finished) and nothing rewritten. +#[tokio::test] +async fn a_stop_during_the_checks_leaves_the_later_rows_unchecked() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let first = sync_arn(1); + let deleting = sync_arn(2); + put_backup(&mem, &first); + let leftover = format!("{}data/000001.json.gz", prefix_of(&deleting)); + mem.insert(&leftover, b"x"); + let mut first_row = old_row(&first, BackupStatus::Available); + first_row.created_at = 1.0; + let mut deleting_row = old_row(&deleting, BackupStatus::Deleting); + deleting_row.created_at = 2.0; + seed_rows(&fake, &[first_row, deleting_row]); + let asked = std::sync::atomic::AtomicUsize::new(0); + let stop = || { + (asked.fetch_add(1, std::sync::atomic::Ordering::AcqRel) >= 1) + .then(|| "SIGTERM received".to_owned()) + }; + let configured = runtime.configured_store().unwrap(); + let identity = configured.identity().unwrap(); + let report = super::store_maintenance::move_store( + &*fake, + super::store_maintenance::MoveTarget { + configured, + identity: &identity, + from_kind: StoreKind::Filesystem, + from_root: "/srv/old", + owner_window: Duration::from_secs(300), + stop: &stop, + claim: None, + }, + ) + .await + .unwrap(); + assert_eq!(report.stopped.as_deref(), Some("SIGTERM received")); + assert!(report.finished.is_empty(), "{report:?}"); + assert!( + mem.keys().contains(&leftover), + "the second row was not checked" + ); + assert_eq!( + fake.rows.lock().unwrap()[&deleting].status, + BackupStatus::Deleting + ); + assert!(fake.rewrites_seen.lock().unwrap().is_empty()); +} + +/// A key the old root no longer lists counts as moved only when its row, +/// read by its ARN, names the configured store: a row another writer +/// removed during the move (a tombstone duty of a server on the old root) +/// or moved to a third store is gone, not moved, and gets no audit line. A +/// re-list of the old root that fails after a rewrite that counted every +/// key takes every key as moved, and so does a read of a key's row that +/// fails, for that key and the rest; either leaves the move incomplete. +#[tokio::test] +async fn a_key_counts_as_moved_only_when_its_row_names_the_configured_store() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let kept = sync_arn(1); + let removed = sync_arn(2); + put_backup(&mem, &kept); + seed_rows( + &fake, + &[ + old_row(&kept, BackupStatus::Available), + old_row(&removed, BackupStatus::Failed), + ], + ); + fake.removed_before_rewrite + .lock() + .unwrap() + .push(removed.clone()); + let report = move_old(&fake, &runtime).await; + assert_eq!(report.rewritten, 1); + assert_eq!(report.moved, std::slice::from_ref(&kept)); + assert_eq!(report.gone, std::slice::from_ref(&removed)); + assert!(report.complete(), "{report:?}"); + + // Moved to a third store by another writer before the rewrite: read by + // its ARN, the row names that store, so it is gone, not moved. + let fake = Arc::new(Fake::default()); + seed_rows( + &fake, + &[ + old_row(&kept, BackupStatus::Available), + old_row(&removed, BackupStatus::Failed), + ], + ); + let third = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/srv/third".to_owned(), + store_id: "third-1".to_owned(), + }; + fake.lands_before_rewrite.lock().unwrap().push(BackupRow { + store: third.clone(), + ..old_row(&removed, BackupStatus::Failed) + }); + let report = move_old(&fake, &runtime).await; + assert_eq!(report.rewritten, 1); + assert_eq!(report.moved, std::slice::from_ref(&kept)); + assert_eq!(report.gone, std::slice::from_ref(&removed)); + assert!(report.complete(), "{report:?}"); + assert_eq!(fake.rows.lock().unwrap()[&removed].store, third); + + // A read of a moved key's row that fails: the count is all there is, + // so that key and the rest are taken as moved, and the move is not + // complete. + let fake = Arc::new(Fake::default()); + seed_rows( + &fake, + &[ + old_row(&kept, BackupStatus::Available), + old_row(&removed, BackupStatus::Failed), + ], + ); + fake.read_row_error_after_rewrite + .store(true, std::sync::atomic::Ordering::Release); + let report = move_old(&fake, &runtime).await; + assert_eq!(report.rewritten, 2); + let relist = report.relist_error.clone().expect("the read's error"); + assert!( + relist.contains("could not be read after the rewrite: ") + && relist.contains("read timed out"), + "{relist}" + ); + assert_eq!( + sorted(report.moved.clone()), + sorted(vec![kept.clone(), removed.clone()]) + ); + assert!(report.gone.is_empty(), "{report:?}"); + assert!(!report.complete()); + + let fake = Arc::new(Fake::default()); + seed_rows( + &fake, + &[ + old_row(&kept, BackupStatus::Available), + old_row(&removed, BackupStatus::Failed), + ], + ); + fake.by_root_error_after_rewrite + .store(true, std::sync::atomic::Ordering::Release); + let report = move_old(&fake, &runtime).await; + assert_eq!(report.rewritten, 2); + assert!(report.relist_error.is_some(), "{report:?}"); + assert_eq!( + sorted(report.moved.clone()), + sorted(vec![kept.clone(), removed.clone()]) + ); + assert!(!report.complete()); + + // With a key the rewrite did not count, a failed re-list takes none as + // moved. + let fake = Arc::new(Fake::default()); + seed_rows( + &fake, + &[ + old_row(&kept, BackupStatus::Available), + old_row(&removed, BackupStatus::Failed), + ], + ); + fake.removed_before_rewrite + .lock() + .unwrap() + .push(removed.clone()); + fake.by_root_error_after_rewrite + .store(true, std::sync::atomic::Ordering::Release); + let report = move_old(&fake, &runtime).await; + assert_eq!(report.rewritten, 1); + assert!(report.moved.is_empty(), "{report:?}"); + assert!(!report.complete()); +} + +/// A row left for its heartbeat: a CREATING row has a live owner to wait +/// for; a FAILED row's job ended with its FAILED write, so its text gives +/// the condition to run again on, the heartbeat's age, and no job to wait +/// for. +#[test] +fn a_failed_row_left_for_its_heartbeat_names_the_heartbeats_age() { + use super::store_maintenance::Forgotten; + assert_eq!( + Forgotten::LiveOwner(BackupStatus::Failed).to_string(), + "left: the FAILED row was written less than stale_job_after_secs ago (its heartbeat is \ + within the window); run the command again once its heartbeat is older than \ + stale_job_after_secs" + ); + assert_eq!( + Forgotten::LiveOwner(BackupStatus::Creating).to_string(), + "left: the CREATING row has a live owner (its heartbeat is within \ + stale_job_after_secs); run the command again once its job has ended" + ); +} + +/// A removal that meets another status reads the row once more and applies +/// that status's removal: an AVAILABLE row a DeleteBackup moves to DELETING +/// under `--forget-backup` is removed as DELETING. A row moved to another +/// store under the removal is left and named as changed. +#[tokio::test] +async fn a_removal_that_meets_another_status_or_store_reads_the_row_again() { + use super::store_maintenance::{ForgetBackup, Forgotten, forget_backups}; + let fake = Arc::new(Fake::default()); + let arn = sync_arn(1); + seed_rows(&fake, &[row(&arn, BackupStatus::Available)]); + let deleting = arn.clone(); + *fake.after_row_read.lock().unwrap() = Some(( + arn.clone(), + Box::new(move |fake: &Fake| { + fake.rows.lock().unwrap().get_mut(&deleting).unwrap().status = BackupStatus::Deleting; + }), + )); + let report = forget_backups( + &*fake, + std::slice::from_ref(&arn), + REGION, + forget_run(Duration::from_secs(300), &never), + ) + .await; + assert_eq!( + report.done, + [( + arn.clone(), + ForgetBackup::Row( + BackupStatus::Available, + Forgotten::Removed(BackupStatus::Deleting) + ) + )] + ); + assert!(!fake.rows.lock().unwrap().contains_key(&arn)); + + let moved = sync_arn(2); + seed_rows(&fake, &[row(&moved, BackupStatus::Available)]); + let elsewhere = moved.clone(); + *fake.after_row_read.lock().unwrap() = Some(( + moved.clone(), + Box::new(move |fake: &Fake| { + let mut rows = fake.rows.lock().unwrap(); + let row = rows.get_mut(&elsewhere).unwrap(); + row.status = BackupStatus::Deleting; + row.store = old_store(); + }), + )); + let report = forget_backups( + &*fake, + std::slice::from_ref(&moved), + REGION, + forget_run(Duration::from_secs(300), &never), + ) + .await; + assert_eq!( + report.done, + [( + moved.clone(), + ForgetBackup::Row( + BackupStatus::Available, + Forgotten::Changed(BackupStatus::Deleting, old_store()) + ) + )] + ); + assert!(fake.rows.lock().unwrap().contains_key(&moved)); +} + +/// `--forget-backup` given again after a stop: an ARN no row and no +/// in-catalog backup carries counts as done ("nothing to remove") and the +/// run goes on to the next ARN, whose row is removed. +#[tokio::test] +async fn forget_backup_goes_past_an_arn_no_row_carries() { + use super::store_maintenance::{ForgetBackup, Forgotten, forget_backups}; + let fake = Arc::new(Fake::default()); + let gone = sync_arn(1); + let kept = sync_arn(2); + seed_rows(&fake, &[row(&kept, BackupStatus::Available)]); + let asked = [gone.clone(), kept.clone()]; + let report = forget_backups( + &*fake, + &asked, + REGION, + forget_run(Duration::from_secs(300), &never), + ) + .await; + assert_eq!( + report.done, + [ + (gone.clone(), ForgetBackup::NoRow), + ( + kept.clone(), + ForgetBackup::Row( + BackupStatus::Available, + Forgotten::Removed(BackupStatus::Available) + ) + ) + ] + ); + assert!(report.complete(2)); + assert_eq!( + ForgetBackup::NoRow.to_string(), + "no row carries this ARN; nothing to remove" + ); + assert!(!fake.rows.lock().unwrap().contains_key(&kept)); +} + +/// The form of every ARN is checked before the command takes the claim: +/// a string that is no ARN and an ARN whose backup id is no id are refused, +/// each with why, said once; a well-formed ARN passes. +#[test] +fn the_arns_are_checked_before_the_claim() { + use super::store_maintenance::check_arns; + let bad_id = sync_arn(1).replace("/backup/", "/backup/x"); + let refused = check_arns(&[ + "not-an-arn".to_owned(), + String::new(), + sync_arn(2), + bad_id.clone(), + ]); + assert_eq!( + refused, + [ + ("not-an-arn".to_owned(), "not a backup ARN".to_owned()), + (String::new(), "not a backup ARN".to_owned()), + (bad_id, "not a backup id".to_owned()), + ] + ); + assert_eq!( + super::store_maintenance::ForgetBackup::Invalid("not a backup ARN".to_owned()).to_string(), + "refused: not a backup ARN" + ); +} + +/// A deletion duty that listed a FAILED row and re-read it before a +/// maintenance command took the claim reads the claim once more after the +/// re-read, finds it held, and deletes nothing; so does a tombstone duty +/// whose listing comes after the command took it. The claim's read is the +/// last step before the call. +#[tokio::test] +async fn a_duty_that_finds_the_claim_held_after_its_reread_acts_on_nothing() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let failed = sync_arn(1); + put_backup(&mem, &failed); + let mut failed_row = row(&failed, BackupStatus::Failed); + failed_row.heartbeat_at = 1.0; + seed_rows(&fake, &[failed_row]); + *fake.after_row_read.lock().unwrap() = Some(( + failed.clone(), + Box::new(|fake: &Fake| *fake.maintenance_held.lock().unwrap() = true), + )); + let cycle = cycle_on(&fake, &runtime).await; + assert!(!cycle.maintenance_held, "the cycle began without the claim"); + assert!(cycle.failed_prefixes_deleted.is_empty(), "{cycle:?}"); + assert!( + objects_under(&mem, &failed).len() > 1, + "the row's bytes are kept" + ); + assert!( + fake.after_row_read.lock().unwrap().is_none(), + "the hook ran" + ); + + *fake.maintenance_held.lock().unwrap() = false; + fake.rows.lock().unwrap().clear(); + let tombstone = sync_arn(2); + let mut tombstone_row = row(&tombstone, BackupStatus::Deleted); + tombstone_row.completed_at = Some(1.0); + seed_rows(&fake, &[tombstone_row]); + *fake.after_older_than.lock().unwrap() = Some(( + BackupStatus::Deleted, + StaleClock::Completed, + Box::new(|fake: &Fake| *fake.maintenance_held.lock().unwrap() = true), + )); + let cycle = cycle_on(&fake, &runtime).await; + assert!(cycle.tombstones_removed.is_empty(), "{cycle:?}"); + assert!(fake.rows.lock().unwrap().contains_key(&tombstone)); +} + +/// The status half of the duty's re-read: a FAILED row forgotten and its +/// complete prefix adopted again (an AVAILABLE row under the same ARN) +/// between the FAILED-prefix duty's listing and its deletion keeps its +/// prefix. +#[tokio::test] +async fn a_duty_keeps_the_prefix_of_a_row_that_turned_available_after_its_listing() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let arn = sync_arn(1); + put_backup(&mem, &arn); + let mut failed = row(&arn, BackupStatus::Failed); + failed.heartbeat_at = 1.0; + seed_rows(&fake, &[failed]); + let adopted = arn.clone(); + *fake.after_older_than.lock().unwrap() = Some(( + BackupStatus::Failed, + StaleClock::Heartbeat, + Box::new(move |fake: &Fake| { + let mut rows = fake.rows.lock().unwrap(); + let row = rows.get_mut(&adopted).unwrap(); + row.status = BackupStatus::Available; + row.owner_generation = UNOWNED_GENERATION; + }), + )); + let whole = objects_under(&mem, &arn).len(); + let cycle = cycle_on(&fake, &runtime).await; + assert!(cycle.failed_prefixes_deleted.is_empty(), "{cycle:?}"); + assert_eq!( + objects_under(&mem, &arn).len(), + whole, + "the backup is whole" + ); +} + +/// A deletion that read the claim free just before a command took it runs +/// at most `DUTY_DELETION_BOUND`: a prefix deletion that would take longer +/// is dropped at the bound, nothing of the prefix is deleted, and the duty +/// stops for this cycle (the next FAILED row's prefix is not even listed); +/// the next cycle continues. +#[tokio::test(start_paused = true)] +async fn a_duty_deletion_that_outlasts_the_bound_stops_the_duty_for_the_cycle() { + use super::reconciler::DUTY_DELETION_BOUND; + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let first = sync_arn(1); + let second = sync_arn(2); + for (n, arn) in [&first, &second].into_iter().enumerate() { + put_backup(&mem, arn); + let mut failed = row(arn, BackupStatus::Failed); + failed.heartbeat_at = 1.0; + failed.created_at = n as f64; + seed_rows(&fake, &[failed]); + } + *mem.delete_delay.lock().unwrap() = Some(DUTY_DELETION_BOUND + Duration::from_secs(10)); + // The cycle's own read is the first; the first row's after its re-read + // the second: a command takes the claim right after it. + *fake.after_maintenance_check.lock().unwrap() = Some(( + 2, + Box::new(|fake: &Fake| *fake.maintenance_held.lock().unwrap() = true), + )); + let whole = objects_under(&mem, &first).len(); + let started = tokio::time::Instant::now(); + let cycle = cycle_on(&fake, &runtime).await; + let took = started.elapsed(); + assert!( + took >= DUTY_DELETION_BOUND && took < DUTY_DELETION_BOUND + Duration::from_secs(10), + "{took:?}" + ); + assert!(cycle.failed_prefixes_deleted.is_empty(), "{cycle:?}"); + assert_eq!( + objects_under(&mem, &first).len(), + whole, + "nothing was deleted" + ); + assert_eq!(objects_under(&mem, &second).len(), whole); + assert!( + !mem.lists.lock().unwrap().contains(&prefix_of(&second)), + "the duty stopped before the next row" + ); + + // The next cycle, without the claim and with a store that answers, + // continues. + *fake.maintenance_held.lock().unwrap() = false; + *mem.delete_delay.lock().unwrap() = None; + let cycle = cycle_on(&fake, &runtime).await; + assert_eq!( + sorted(cycle.failed_prefixes_deleted.clone()), + sorted(vec![first.clone(), second.clone()]) + ); +} + +/// Beside servers the forget options wait `FORGET_WAIT` from the claim +/// before their first FAILED or DELETED removal, told once; an AVAILABLE +/// row's removal does not wait; a stop during the wait leaves the FAILED +/// row and ends the run; with no wait (SQLite) nothing waits. The wait is +/// the duty's bound and one store call's. +#[tokio::test(start_paused = true)] +async fn the_forget_options_wait_out_the_duties_before_a_failed_removal() { + use super::reconciler::DUTY_DELETION_BOUND; + use super::store_maintenance::{FORGET_WAIT, ForgetRun, Forgotten, forget_store}; + assert_eq!(FORGET_WAIT, DUTY_DELETION_BOUND + super::S3_CHECK_TIMEOUT); + assert!(DUTY_DELETION_BOUND < extenddb_backup_store::S3_ATTEMPT_TIMEOUT); + let told = Mutex::new(Vec::new()); + let tell = |remaining: Duration| told.lock().unwrap().push(remaining); + let seed = |fake: &Fake| { + let mut available = old_row(&sync_arn(1), BackupStatus::Available); + available.created_at = 1.0; + let mut failed = old_row(&sync_arn(2), BackupStatus::Failed); + failed.created_at = 2.0; + seed_rows(fake, &[available, failed]); + }; + + let fake = Arc::new(Fake::default()); + seed(&fake); + let claimed = tokio::time::Instant::now(); + let run = ForgetRun { + owner_window: Duration::ZERO, + removals_from: Some(claimed + FORGET_WAIT), + on_wait: &tell, + stop: &never, + claim: None, + configured: None, + unmounted_root: None, + deletions_finished: &[], + }; + let report = forget_store(&*fake, StoreKind::Filesystem, "/srv/old", run) + .await + .unwrap(); + assert!(report.complete(), "{report:?}"); + assert!(claimed.elapsed() >= FORGET_WAIT); + assert_eq!(*told.lock().unwrap(), [FORGET_WAIT]); + + // Only an AVAILABLE row: no wait. + told.lock().unwrap().clear(); + let fake = Arc::new(Fake::default()); + seed_rows(&fake, &[old_row(&sync_arn(1), BackupStatus::Available)]); + let claimed = tokio::time::Instant::now(); + let run = ForgetRun { + removals_from: Some(claimed + FORGET_WAIT), + ..run + }; + let report = forget_store(&*fake, StoreKind::Filesystem, "/srv/old", run) + .await + .unwrap(); + assert!(report.complete()); + assert!(claimed.elapsed() < FORGET_WAIT); + assert!(told.lock().unwrap().is_empty()); + + // A stop during the wait. + let fake = Arc::new(Fake::default()); + seed(&fake); + let claimed = tokio::time::Instant::now(); + let stop = + || (claimed.elapsed() >= Duration::from_secs(5)).then(|| "SIGINT received".to_owned()); + let run = ForgetRun { + removals_from: Some(claimed + FORGET_WAIT), + stop: &stop, + ..run + }; + let report = forget_store(&*fake, StoreKind::Filesystem, "/srv/old", run) + .await + .unwrap(); + assert_eq!(report.stopped.as_deref(), Some("SIGINT received")); + assert_eq!( + report + .rows + .iter() + .map(|(_, f)| f.clone()) + .collect::>(), + [Forgotten::Removed(BackupStatus::Available)] + ); + assert!(fake.rows.lock().unwrap().contains_key(&sync_arn(2))); + assert!(claimed.elapsed() < FORGET_WAIT); + + // No wait: SQLite, under the serve lock. + let fake = Arc::new(Fake::default()); + seed(&fake); + let claimed = tokio::time::Instant::now(); + let report = forget_store( + &*fake, + StoreKind::Filesystem, + "/srv/old", + forget_run(Duration::ZERO, &never), + ) + .await + .unwrap(); + assert!(report.complete()); + assert_eq!(claimed.elapsed(), Duration::ZERO); +} + +/// The store-backed create reads the maintenance claim again immediately +/// before its insert: a claim a command took after the request's check +/// (while the job opened the table, say) refuses the request as a held +/// claim does, with no row inserted and every slot free. +#[tokio::test] +async fn a_claim_taken_after_the_requests_check_refuses_the_create_before_its_row() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(3), true, None); + *fake.after_maintenance_check.lock().unwrap() = Some(( + 1, + Box::new(|fake: &Fake| *fake.maintenance_held.lock().unwrap() = true), + )); + let runtime = job_runtime(&mem); + let err = create_music_backup(&fake, &runtime).await.unwrap_err(); + assert!( + matches!(&err, DynamoDbError::LimitExceededException(m) + if m.contains("A backup maintenance command holds")), + "{err:?}" + ); + wait_for_jobs(&runtime).await; + assert_eq!(fake.maintenance_checks.lock().unwrap().len(), 2); + assert!(fake.rows.lock().unwrap().is_empty(), "no row is inserted"); + assert!(fake.claims.lock().unwrap().is_empty()); + assert!(mem.puts.lock().unwrap().is_empty()); + assert_eq!(runtime.free_slots(), 4); +} + +/// The read-only check after `--forget-backup` writes nothing: on a root +/// with no identity file and no rows it creates no file (`Ok(None)`); on +/// a root whose file holds another id than the one the command opened it +/// with it is an identity mismatch; otherwise it passes, the manifest +/// probe included. +#[tokio::test] +async fn the_read_only_check_writes_nothing_to_the_root() { + let fake = Arc::new(Fake::default()); + let mem = Arc::new(MemStore::default()); + let configured = ConfiguredStore::new( + store().store_kind, + store().store_root, + Arc::clone(&mem) as Arc, + None, + Ok(store()), + ); + let check = super::check_configured_store_read_only(&configured, &*fake, None).await; + assert_eq!(check, Ok(None)); + assert!(mem.keys().is_empty(), "no identity file is created"); + assert!(mem.puts.lock().unwrap().is_empty()); + + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let check = super::check_configured_store_read_only(&configured, &*fake, Some(&store())).await; + assert_eq!(check, Ok(Some(store()))); + let other = StoreIdentity { + store_id: "store-9".to_owned(), + ..store() + }; + let check = super::check_configured_store_read_only(&configured, &*fake, Some(&other)).await; + assert!( + matches!(&check, Err(StoreProblem::IdMismatch { file_id, .. }) if file_id == "store-1"), + "{check:?}" + ); + let gone = sync_arn(1); + seed_rows(&fake, &[row(&gone, BackupStatus::Available)]); + let check = super::check_configured_store_read_only(&configured, &*fake, Some(&store())).await; + assert!( + matches!(&check, Err(StoreProblem::ManifestMissing { backup_arn }) if *backup_arn == gone), + "{check:?}" + ); + assert!(mem.puts.lock().unwrap().is_empty()); +} + +/// The DELETED-prefix deletion and the tombstone removal run under the same +/// bound: a deletion of a DELETED row's prefix that would outlast it is +/// dropped with nothing deleted and the DELETED-prefix duty stops for the +/// cycle (the next DELETED row's prefix is not listed); a tombstone +/// removal held past it is dropped with the row kept and the tombstone duty +/// stops (the next tombstone is not tried). +#[tokio::test(start_paused = true)] +async fn the_deleted_prefix_and_tombstone_calls_stop_their_duty_at_the_bound() { + use super::reconciler::DUTY_DELETION_BOUND; + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let swept = [sync_arn(1), sync_arn(2)]; + for (n, arn) in swept.iter().enumerate() { + put_backup(&mem, arn); + let mut deleted = row(arn, BackupStatus::Deleted); + deleted.created_at = n as f64; + seed_rows(&fake, &[deleted]); + } + *mem.delete_delay.lock().unwrap() = Some(DUTY_DELETION_BOUND + Duration::from_secs(10)); + let whole = objects_under(&mem, &swept[0]).len(); + let cycle = cycle_on(&fake, &runtime).await; + assert!(cycle.deleted_prefixes_swept.is_empty(), "{cycle:?}"); + assert_eq!(objects_under(&mem, &swept[0]).len(), whole); + assert!( + !mem.lists.lock().unwrap().contains(&prefix_of(&swept[1])), + "the duty stopped before the next row" + ); + + *mem.delete_delay.lock().unwrap() = None; + fake.rows.lock().unwrap().clear(); + let tombstones = [sync_arn(3), sync_arn(4)]; + for (n, arn) in tombstones.iter().enumerate() { + let mut failed = row(arn, BackupStatus::Failed); + failed.completed_at = Some(1.0); + failed.heartbeat_at = seconds_since_epoch(); + failed.created_at = n as f64; + seed_rows(&fake, &[failed]); + } + *fake.remove_delay.lock().unwrap() = Some(DUTY_DELETION_BOUND + Duration::from_secs(10)); + let cycle = cycle_on(&fake, &runtime).await; + assert!(cycle.tombstones_removed.is_empty(), "{cycle:?}"); + let held = fake.rows.lock().unwrap(); + assert!(held.contains_key(&tombstones[0]) && held.contains_key(&tombstones[1])); + drop(held); + assert_eq!( + *fake.remove_attempts.lock().unwrap(), + [tombstones[0].clone()], + "the duty stopped before the next tombstone" + ); +} + +// --- the duties' deadline, the move's order, the forget's configured store --- + +/// A duty's claim read that answers "free" after its deadline (the query +/// ran before a command took the claim; its answer came two minutes later, +/// after the command had waited and removed the row) starts no prefix +/// deletion and no tombstone removal: the duty stops for this cycle. +#[tokio::test(start_paused = true)] +async fn a_claim_read_answered_after_the_bound_starts_no_deletion() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let arn = sync_arn(1); + put_backup(&mem, &arn); + let mut failed = row(&arn, BackupStatus::Failed); + failed.heartbeat_at = 1.0; + seed_rows(&fake, &[failed]); + let gone = arn.clone(); + // The cycle's own read is the first; the duty's, after its re-read, the + // second. + *fake.after_maintenance_check.lock().unwrap() = Some(( + 2, + Box::new(move |fake: &Fake| { + fake.rows.lock().unwrap().remove(&gone); + *fake.maintenance_delay.lock().unwrap() = Some(Duration::from_secs(120)); + }), + )); + let whole = objects_under(&mem, &arn).len(); + let cycle = cycle_on(&fake, &runtime).await; + assert!(cycle.failed_prefixes_deleted.is_empty(), "{cycle:?}"); + assert_eq!( + objects_under(&mem, &arn).len(), + whole, + "a deletion whose claim read answered after the bound must not start" + ); + + *fake.maintenance_delay.lock().unwrap() = None; + let tombstone = sync_arn(2); + let mut tombstone_row = row(&tombstone, BackupStatus::Deleted); + tombstone_row.completed_at = Some(1.0); + seed_rows(&fake, &[tombstone_row]); + let checks = fake.maintenance_checks.lock().unwrap().len(); + *fake.after_maintenance_check.lock().unwrap() = Some(( + checks + 2, + Box::new(|fake: &Fake| { + *fake.maintenance_delay.lock().unwrap() = Some(Duration::from_secs(120)); + }), + )); + let cycle = cycle_on(&fake, &runtime).await; + assert!(cycle.tombstones_removed.is_empty(), "{cycle:?}"); + assert!(fake.rows.lock().unwrap().contains_key(&tombstone)); +} + +/// On a catalog keyed by ARN (Cassandra), an old-root DELETING row under +/// another spelling of a configured-root row's ARN shares that row's +/// prefix: the two-rows check stops it before any check, so the configured +/// row's prefix, incomplete, keeps every object. A configured root whose +/// rows cannot be read stops the move before anything changed: the dead +/// CREATING row of the old root is not resolved, and the DELETING row not +/// finished. +#[tokio::test] +async fn the_two_rows_check_comes_before_any_row_of_the_old_root_is_checked() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + fake.unique_by_arn + .store(true, std::sync::atomic::Ordering::Release); + let runtime = runtime_on(&mem, store()); + let ours = sync_arn(1); + let other_spelling = ours.replace(REGION, "eu-west-1"); + let mut objects = backup_objects(&ours); + let dropped = objects + .iter() + .position(|(k, _)| k.contains("/data/")) + .unwrap(); + objects.remove(dropped); + for (key, bytes) in objects { + mem.insert(&key, &bytes); + } + let before = objects_under(&mem, &ours).len(); + seed_rows( + &fake, + &[ + row(&ours, BackupStatus::Available), + old_row(&other_spelling, BackupStatus::Deleting), + ], + ); + let report = move_old(&fake, &runtime).await; + assert!(report.finished.is_empty(), "{report:?}"); + assert_eq!(report.stopping.len(), 1, "{report:?}"); + assert!( + report.stopping[0].1.contains("two rows of backup id"), + "{report:?}" + ); + assert_eq!(objects_under(&mem, &ours).len(), before); + assert_eq!( + fake.rows.lock().unwrap()[&other_spelling].status, + BackupStatus::Deleting + ); + + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let creating = sync_arn(2); + let deleting = sync_arn(3); + put_backup(&mem, &creating); + let leftover = format!("{}data/000001.json.gz", prefix_of(&deleting)); + mem.insert(&leftover, b"x"); + let mut creating_row = old_row(&creating, BackupStatus::Creating); + creating_row.heartbeat_at = 1.0; + fake.dead_by_heartbeat + .store(true, std::sync::atomic::Ordering::Release); + seed_rows( + &fake, + &[creating_row, old_row(&deleting, BackupStatus::Deleting)], + ); + *fake.by_root_error_for.lock().unwrap() = Some(store().store_root); + let configured = runtime.configured_store().unwrap(); + let identity = configured.identity().unwrap(); + let outcome = super::store_maintenance::move_store( + &*fake, + super::store_maintenance::MoveTarget { + configured, + identity: &identity, + from_kind: StoreKind::Filesystem, + from_root: "/srv/old", + owner_window: Duration::from_secs(300), + stop: &never, + claim: None, + }, + ) + .await; + assert!(outcome.is_err(), "{outcome:?}"); + let held = fake.rows.lock().unwrap(); + assert_eq!(held[&creating].status, BackupStatus::Creating); + assert_eq!(held[&deleting].status, BackupStatus::Deleting); + assert!(mem.keys().contains(&leftover)); +} + +/// The stop asked after the last row's check and before the fence ends the +/// move with nothing rewritten (the claim lost, or a signal, during the +/// configured root's listing or the last check). +#[tokio::test] +async fn a_stop_after_the_last_check_rewrites_nothing() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let arn = sync_arn(1); + put_backup(&mem, &arn); + seed_rows(&fake, &[old_row(&arn, BackupStatus::Available)]); + let asked = std::sync::atomic::AtomicUsize::new(0); + // Asked before the one row, then before the fence. + let stop = || { + (asked.fetch_add(1, std::sync::atomic::Ordering::AcqRel) >= 1) + .then(|| "the maintenance claim was lost".to_owned()) + }; + let configured = runtime.configured_store().unwrap(); + let identity = configured.identity().unwrap(); + let report = super::store_maintenance::move_store( + &*fake, + super::store_maintenance::MoveTarget { + configured, + identity: &identity, + from_kind: StoreKind::Filesystem, + from_root: "/srv/old", + owner_window: Duration::from_secs(300), + stop: &stop, + claim: None, + }, + ) + .await + .unwrap(); + assert_eq!(asked.load(std::sync::atomic::Ordering::Acquire), 2); + assert_eq!( + report.stopped.as_deref(), + Some("the maintenance claim was lost") + ); + assert!(report.stopping.is_empty(), "{report:?}"); + assert!(fake.rewrites_seen.lock().unwrap().is_empty()); + assert_eq!(fake.rows.lock().unwrap()[&arn].store.store_root, "/srv/old"); +} + +/// The wait before the first FAILED or DELETED removal covers a DELETED +/// row as it covers a FAILED one. +#[tokio::test(start_paused = true)] +async fn the_forget_options_wait_out_the_duties_before_a_deleted_removal() { + use super::store_maintenance::{FORGET_WAIT, ForgetRun, forget_store}; + let told = Mutex::new(Vec::new()); + let tell = |remaining: Duration| told.lock().unwrap().push(remaining); + let fake = Arc::new(Fake::default()); + let mut available = old_row(&sync_arn(1), BackupStatus::Available); + available.created_at = 1.0; + let mut deleted = old_row(&sync_arn(2), BackupStatus::Deleted); + deleted.created_at = 2.0; + seed_rows(&fake, &[available, deleted]); + let claimed = tokio::time::Instant::now(); + let run = ForgetRun { + owner_window: Duration::ZERO, + removals_from: Some(claimed + FORGET_WAIT), + on_wait: &tell, + stop: &never, + claim: None, + configured: None, + unmounted_root: None, + deletions_finished: &[], + }; + let report = forget_store(&*fake, StoreKind::Filesystem, "/srv/old", run) + .await + .unwrap(); + assert!(report.complete(), "{report:?}"); + assert!(claimed.elapsed() >= FORGET_WAIT); + assert_eq!(*told.lock().unwrap(), [FORGET_WAIT]); +} + +/// A failed read in the duty's fence is left for the next cycle: the claim +/// read failing, or the row's re-read failing, deletes nothing. +#[tokio::test] +async fn a_duty_whose_claim_read_or_reread_fails_deletes_nothing() { + for cell in ["the claim read fails", "the re-read fails"] { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let arn = sync_arn(1); + put_backup(&mem, &arn); + let mut failed = row(&arn, BackupStatus::Failed); + failed.heartbeat_at = 1.0; + seed_rows(&fake, &[failed]); + if cell == "the claim read fails" { + *fake.after_row_read.lock().unwrap() = Some(( + arn.clone(), + Box::new(|fake: &Fake| { + *fake.maintenance_error.lock().unwrap() = Some("timed out".to_owned()); + }), + )); + } else { + *fake.after_older_than.lock().unwrap() = Some(( + BackupStatus::Failed, + StaleClock::Heartbeat, + Box::new(|fake: &Fake| { + *fake.read_row_error.lock().unwrap() = Some("timed out".to_owned()); + }), + )); + } + let whole = objects_under(&mem, &arn).len(); + let cycle = cycle_on(&fake, &runtime).await; + assert!( + cycle.failed_prefixes_deleted.is_empty(), + "{cell}: {cycle:?}" + ); + assert_eq!(objects_under(&mem, &arn).len(), whole, "{cell}"); + } +} + +/// `--forget-backup` of an earlier release's in-catalog backup given under +/// a spelling of its ARN that is not the server's region reads the legacy +/// backup under the ARN as written, and changes nothing. +#[tokio::test] +async fn forget_backup_reads_a_legacy_backup_under_the_arn_as_written() { + use super::store_maintenance::{ForgetBackup, forget_backups}; + let fake = Arc::new(Fake::default()); + let written = sync_arn(12).replace(REGION, "eu-west-1"); + fake.descriptions.lock().unwrap().insert( + written.clone(), + backup_description("AVAILABLE", "PAY_PER_REQUEST"), + ); + let report = forget_backups( + &*fake, + std::slice::from_ref(&written), + REGION, + forget_run(Duration::from_secs(300), &never), + ) + .await; + assert_eq!(report.done, [(written.clone(), ForgetBackup::Legacy)]); + assert!(!report.complete(1)); + assert!(fake.descriptions.lock().unwrap().contains_key(&written)); +} + +/// The FAILED live-owner rule reads each store's dead rows: dead FAILED +/// rows of two stores in one run are both removed. +#[tokio::test] +async fn forget_backup_tells_dead_failed_rows_of_two_stores_apart() { + use super::store_maintenance::{ForgetBackup, Forgotten, forget_backups}; + let fake = Arc::new(Fake::default()); + let ours = sync_arn(1); + let theirs = sync_arn(2); + seed_rows( + &fake, + &[ + row(&ours, BackupStatus::Failed), + old_row(&theirs, BackupStatus::Failed), + ], + ); + let asked = [ours.clone(), theirs.clone()]; + let report = forget_backups( + &*fake, + &asked, + REGION, + forget_run(Duration::from_secs(300), &never), + ) + .await; + let removed = ForgetBackup::Row( + BackupStatus::Failed, + Forgotten::Removed(BackupStatus::Failed), + ); + assert_eq!( + report.done, + [(ours, removed.clone()), (theirs, removed)], + "{report:?}" + ); + assert!(fake.rows.lock().unwrap().is_empty()); +} + +/// `--forget-store` lists a row whose one status entry is under an +/// earlier status (a DELETING row with an AVAILABLE entry) and removes it. +/// After its removals it sweeps the root's status entries: the entry with +/// no row behind it and the removed row's stale entry are deleted and +/// counted; the entry a row left in use keeps under an earlier status is +/// repaired and counted apart. An entry written after the root's listing +/// (`late_entries`) has its id reported. +#[tokio::test] +async fn forget_store_removes_the_roots_entries_with_no_row_and_reports_the_rest() { + use super::store_maintenance::{Forgotten, forget_store}; + let fake = Arc::new(Fake::default()); + let available = sync_arn(1); + let deleting = sync_arn(2); + seed_rows( + &fake, + &[ + old_row(&available, BackupStatus::Available), + old_row(&deleting, BackupStatus::Deleting), + ], + ); + fake.set_stale_entry(&deleting, BackupStatus::Available); + fake.orphan_entries + .lock() + .unwrap() + .push((old_store(), BackupStatus::Deleting)); + let mut late = old_store(); + late.store_id = "old-2".to_owned(); + fake.late_entries.lock().unwrap().push(late); + let report = forget_store( + &*fake, + StoreKind::Filesystem, + "/srv/old", + forget_run(Duration::ZERO, &never), + ) + .await + .unwrap(); + assert!(report.complete(), "{report:?}"); + assert_eq!(report.removed(), 2); + assert_eq!( + report.entries_swept, + Some(StatusEntrySweep { + removed: 2, + repaired: 0 + }) + ); + assert_eq!(report.entries_left, ["old-2"]); + assert!(fake.orphan_entries.lock().unwrap().is_empty()); + assert!(fake.stale_entries.lock().unwrap().is_empty()); + + // A row left in use, its entry under an earlier status: repaired. + let in_use = sync_arn(3); + seed_rows(&fake, &[old_row(&in_use, BackupStatus::Available)]); + fake.set_stale_entry(&in_use, BackupStatus::Creating); + fake.forget_answers + .lock() + .unwrap() + .insert(in_use.clone(), ForgetRow::InUse("Target".to_owned())); + let report = forget_store( + &*fake, + StoreKind::Filesystem, + "/srv/old", + forget_run(Duration::ZERO, &never), + ) + .await + .unwrap(); + assert!(!report.complete(), "{report:?}"); + assert_eq!( + report.rows[0].1, + Forgotten::InUse("Target".to_owned(), ACCOUNT.to_owned()) + ); + assert_eq!( + report.entries_swept, + Some(StatusEntrySweep { + removed: 0, + repaired: 1 + }) + ); + assert!( + fake.list_backup_rows(Some(BackupStatus::Available), &old_store()) + .try_collect::>() + .await + .unwrap() + .iter() + .any(|r| r.backup_arn == in_use), + "the repaired row is listed under its own status" + ); +} + +/// A DELETING row of the configured store has its deletion finished behind +/// the fence before its removal when the store's check passed, and is left +/// otherwise; a DELETING row of another store is removed and keeps its +/// bytes; a DELETED row whose deletion the command's own pass finished says +/// so. +#[tokio::test] +async fn forget_backup_finishes_a_deleting_row_of_the_configured_store_first() { + use super::store_maintenance::{ + ForgetBackup, ForgetConfigured, ForgetRun, Forgotten, forget_backups, + }; + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let configured = runtime.configured_store().unwrap(); + let identity = configured.identity().unwrap(); + let identity = &identity; + fn run<'a>( + store: &'a ConfiguredStore, + healthy: Option<&'a StoreIdentity>, + finished: &'a [String], + ) -> ForgetRun<'a> { + ForgetRun { + configured: Some(ForgetConfigured { + store, + healthy, + mismatch_row_ids: &[], + file_missing_with_rows: false, + }), + deletions_finished: finished, + ..forget_run(Duration::from_secs(300), &never) + } + } + + // The store's check passed: the deletion is finished, then the row goes. + let deleting = sync_arn(1); + put_backup(&mem, &deleting); + seed_rows(&fake, &[row(&deleting, BackupStatus::Deleting)]); + let report = forget_backups( + &*fake, + std::slice::from_ref(&deleting), + REGION, + run(configured, Some(identity), &[]), + ) + .await; + assert_eq!( + report.done, + [( + deleting.clone(), + ForgetBackup::Row(BackupStatus::Deleting, Forgotten::RemovedAfterDeletion) + )] + ); + assert!( + objects_under(&mem, &deleting).is_empty(), + "the prefix is deleted" + ); + assert!(!fake.rows.lock().unwrap().contains_key(&deleting)); + + // Unhealthy (or the id unknown): left, bytes and row. + let pending = sync_arn(2); + put_backup(&mem, &pending); + seed_rows(&fake, &[row(&pending, BackupStatus::Deleting)]); + let whole = objects_under(&mem, &pending).len(); + let report = forget_backups( + &*fake, + std::slice::from_ref(&pending), + REGION, + run(configured, None, &[]), + ) + .await; + assert_eq!( + report.done, + [( + pending.clone(), + ForgetBackup::Row(BackupStatus::Deleting, Forgotten::DeletionPending) + )] + ); + assert_eq!(objects_under(&mem, &pending).len(), whole); + assert_eq!( + fake.rows.lock().unwrap()[&pending].status, + BackupStatus::Deleting + ); + + // Another store's DELETING row: removed, nothing deleted. + let theirs = sync_arn(3); + put_backup(&mem, &theirs); + seed_rows(&fake, &[old_row(&theirs, BackupStatus::Deleting)]); + let whole = objects_under(&mem, &theirs).len(); + let report = forget_backups( + &*fake, + std::slice::from_ref(&theirs), + REGION, + run(configured, Some(identity), &[]), + ) + .await; + assert_eq!( + report.done, + [( + theirs.clone(), + ForgetBackup::Row( + BackupStatus::Deleting, + Forgotten::Removed(BackupStatus::Deleting) + ) + )] + ); + assert_eq!(objects_under(&mem, &theirs).len(), whole); + + // A DELETED row the command's own pass finished. + let finished = sync_arn(4); + seed_rows(&fake, &[row(&finished, BackupStatus::Deleted)]); + let list = [finished.clone()]; + let report = forget_backups( + &*fake, + std::slice::from_ref(&finished), + REGION, + run(configured, Some(identity), &list), + ) + .await; + assert_eq!( + report.done, + [( + finished.clone(), + ForgetBackup::Row(BackupStatus::Deleted, Forgotten::RemovedAfterDeletion) + )] + ); +} + +/// While the configured root has no identity file and the catalog holds +/// rows for it (an unmounted volume), its rows are refused and kept; a row +/// of another root in the same run is removed. +#[tokio::test] +async fn forget_backup_refuses_the_rows_of_a_configured_root_without_its_file() { + use super::store_maintenance::{ + ForgetBackup, ForgetConfigured, ForgetRun, Forgotten, forget_backups, + }; + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_unhealthy( + &mem, + StoreProblem::FileMissingWithRows(vec!["store-1".to_owned()]), + ); + let configured = runtime.configured_store().unwrap(); + let theirs = sync_arn(1); + let ours = sync_arn(2); + seed_rows( + &fake, + &[ + old_row(&theirs, BackupStatus::Available), + row(&ours, BackupStatus::Available), + ], + ); + let run = ForgetRun { + configured: Some(ForgetConfigured { + store: configured, + healthy: None, + mismatch_row_ids: &[], + file_missing_with_rows: true, + }), + ..forget_run(Duration::from_secs(300), &never) + }; + let asked = [theirs.clone(), ours.clone()]; + let report = forget_backups(&*fake, &asked, REGION, run).await; + assert_eq!( + report.done, + [ + ( + theirs.clone(), + ForgetBackup::Row( + BackupStatus::Available, + Forgotten::Removed(BackupStatus::Available) + ) + ), + ( + ours.clone(), + ForgetBackup::Row(BackupStatus::Available, Forgotten::RootWithoutFile) + ), + ] + ); + assert!(fake.rows.lock().unwrap().contains_key(&ours)); + assert!( + Forgotten::RootWithoutFile + .to_string() + .contains("point [backup.store] at the root that holds these backups") + ); +} + +/// On a backend whose summary counts status entries (Cassandra), the report +/// of other stores checks each count against the store's listing: what no +/// row stands behind is reported as status entries, not rows. +#[tokio::test] +async fn the_report_of_other_stores_names_status_entries_with_no_row() { + let fake = Arc::new(Fake::default()); + fake.counts_entries + .store(true, std::sync::atomic::Ordering::Release); + seed_rows(&fake, &[old_row(&sync_arn(1), BackupStatus::Available)]); + fake.orphan_entries + .lock() + .unwrap() + .push((old_store(), BackupStatus::Deleting)); + let others = super::store_maintenance::other_stores(&*fake, &store()) + .await + .unwrap(); + assert_eq!(others.len(), 1, "{others:?}"); + assert_eq!(others[0].counts, [(BackupStatus::Available, 1)]); + assert_eq!(others[0].entries, [(BackupStatus::Deleting, 1)]); + assert!(others[0].unresolved.is_empty()); + + // Elsewhere the counts are rows, taken as they are. + fake.counts_entries + .store(false, std::sync::atomic::Ordering::Release); + fake.orphan_entries.lock().unwrap().clear(); + let others = super::store_maintenance::other_stores(&*fake, &store()) + .await + .unwrap(); + assert!(others[0].entries.is_empty()); +} + +/// The tombstone duty's removal names the store it listed the row with: a +/// row a move rewrote to another store after the duty's fence is left for +/// that store's duties. +#[tokio::test] +async fn the_tombstone_duty_leaves_a_row_moved_after_its_fence() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let tombstone = sync_arn(1); + let mut tombstone_row = row(&tombstone, BackupStatus::Deleted); + tombstone_row.completed_at = Some(1.0); + seed_rows(&fake, &[tombstone_row]); + let moved = tombstone.clone(); + *fake.after_maintenance_check.lock().unwrap() = Some(( + 2, + Box::new(move |fake: &Fake| { + fake.rows.lock().unwrap().get_mut(&moved).unwrap().store = old_store(); + }), + )); + let cycle = cycle_on(&fake, &runtime).await; + assert!(cycle.tombstones_removed.is_empty(), "{cycle:?}"); + assert_eq!(fake.rows.lock().unwrap()[&tombstone].store, old_store()); + assert_eq!(*fake.remove_attempts.lock().unwrap(), [tombstone]); +} + +/// A prefix deletion the store stopped at its deadline (it answers what it +/// removed, unfinished) stops the duty for this cycle: the row is not +/// reported deleted, and the next row's prefix is not listed. The deadline +/// the duty hands the store is `DUTY_DELETION_BOUND` after its read of the +/// claim was sent, not after the answer (the read here answers 5 s later). +#[tokio::test(start_paused = true)] +async fn a_deletion_the_store_stopped_at_its_deadline_stops_the_duty() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let first = sync_arn(1); + let second = sync_arn(2); + for (n, arn) in [&first, &second].into_iter().enumerate() { + put_backup(&mem, arn); + let mut failed = row(arn, BackupStatus::Failed); + failed.heartbeat_at = 1.0; + failed.created_at = n as f64; + seed_rows(&fake, &[failed]); + } + mem.deadline_reached + .store(true, std::sync::atomic::Ordering::Release); + // The cycle's own read is the first; the duty's, after its re-read, the + // second. + let read_sent = Arc::new(Mutex::new(None)); + *fake.after_maintenance_check.lock().unwrap() = Some(( + 2, + Box::new({ + let read_sent = Arc::clone(&read_sent); + move |fake: &Fake| { + *read_sent.lock().unwrap() = Some(tokio::time::Instant::now()); + *fake.maintenance_delay.lock().unwrap() = Some(Duration::from_secs(5)); + } + }), + )); + let cycle = cycle_on(&fake, &runtime).await; + assert!(cycle.failed_prefixes_deleted.is_empty(), "{cycle:?}"); + assert!( + !mem.lists.lock().unwrap().contains(&prefix_of(&second)), + "the duty stopped before the next row" + ); + let read_sent = read_sent.lock().unwrap().expect("the duty read the claim"); + assert_eq!( + *mem.deadlines_seen.lock().unwrap(), + [( + prefix_of(&first), + read_sent + super::reconciler::DUTY_DELETION_BOUND + )] + ); +} + +/// The cycle a forget option runs on SQLite (the claim held, no store +/// check) writes nothing to the root: a root that lost its identity file +/// is not given a new one, and the store's health stays as the command +/// recorded it. +#[tokio::test] +async fn a_cycle_without_the_store_check_writes_nothing_to_the_root() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + mem.objects.lock().unwrap().remove(STORE_IDENTITY_KEY); + super::reconciler::cycle( + &fake, + &runtime, + Duration::ZERO, + "sqlite", + &mut super::reconciler::ReconcilerState::for_maintenance_holder_without_store_check(), + ) + .await + .unwrap(); + assert!( + !mem.objects.lock().unwrap().contains_key(STORE_IDENTITY_KEY), + "no identity file is written" + ); + assert!(mem.puts.lock().unwrap().is_empty(), "nothing is put"); + assert!(runtime.configured_store().unwrap().is_healthy()); +} + +/// Every row of one backup id among the moved (rows unique by ARN alone, as +/// on Cassandra) is set aside before any row is checked: the first in +/// listing order is DELETING and the second AVAILABLE, and the copy of +/// their shared prefix lacks a data file. Neither is checked, so the +/// DELETING row's deletion is not finished and the shared prefix keeps +/// every object; both are named. +#[tokio::test] +async fn every_row_of_one_backup_id_among_the_moved_is_checked_no_further() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + fake.unique_by_arn + .store(true, std::sync::atomic::Ordering::Release); + let runtime = runtime_on(&mem, store()); + let ours = sync_arn(1); + let other = ours.replace(REGION, "eu-west-1"); + let mut objects = backup_objects(&other); + let dropped = objects + .iter() + .position(|(k, _)| k.contains("/data/")) + .unwrap(); + objects.remove(dropped); + for (key, bytes) in objects { + mem.insert(&key, &bytes); + } + let before = objects_under(&mem, &other).len(); + let mut deleting = old_row(&ours, BackupStatus::Deleting); + deleting.created_at = 1.0; + let mut available = old_row(&other, BackupStatus::Available); + available.created_at = 2.0; + seed_rows(&fake, &[deleting, available]); + let report = move_old(&fake, &runtime).await; + assert!(report.finished.is_empty(), "{report:?}"); + let named: Vec<&str> = report + .stopping + .iter() + .map(|(row, _)| row.backup_arn.as_str()) + .collect(); + assert_eq!(named, [ours.as_str(), other.as_str()], "{report:?}"); + assert!( + report + .stopping + .iter() + .all(|(_, why)| why.contains("two rows of backup id")), + "{report:?}" + ); + assert_eq!(objects_under(&mem, &other).len(), before); + assert_eq!( + fake.rows.lock().unwrap()[&ours].status, + BackupStatus::Deleting + ); + assert!(fake.rewrites_seen.lock().unwrap().is_empty()); +} + +/// The FAILED-prefix duty keeps the prefix of a row that a move rewrote to +/// another store after the duty listed it, its status unchanged: only the +/// store half of `Duty::fenced`'s re-read stands between them (the prefix +/// deletion has no store condition of its own). +#[tokio::test] +async fn the_failed_prefix_duty_keeps_a_row_moved_after_its_listing() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let moved = sync_arn(1); + let leftover = format!("{}data/000001.json.gz", prefix_of(&moved)); + mem.insert(&leftover, b"x"); + seed_rows(&fake, &[row(&moved, BackupStatus::Failed)]); + let elsewhere = moved.clone(); + *fake.after_older_than.lock().unwrap() = Some(( + BackupStatus::Failed, + StaleClock::Heartbeat, + Box::new(move |fake: &Fake| { + fake.rows.lock().unwrap().get_mut(&elsewhere).unwrap().store = old_store(); + }), + )); + let cycle = cycle_on(&fake, &runtime).await; + assert!( + fake.after_older_than.lock().unwrap().is_none(), + "the hook ran" + ); + assert!(cycle.failed_prefixes_deleted.is_empty(), "{cycle:?}"); + assert!( + mem.keys().contains(&leftover), + "the moved row's prefix is kept" + ); +} + +/// A row of the old root whose one status entry is under an earlier status +/// (on Cassandra, a DeleteBackup whose entry batch failed: the row +/// DELETING, its entry AVAILABLE) is listed by the root's listing and +/// checked like any other: with a complete copy under the configured root +/// it stops the move; with none, its deletion is finished and it moves. +#[tokio::test] +async fn move_store_checks_a_row_whose_status_entry_is_under_an_earlier_status() { + let deleting = sync_arn(1); + + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + put_backup(&mem, &deleting); + seed_rows(&fake, &[old_row(&deleting, BackupStatus::Deleting)]); + fake.set_stale_entry(&deleting, BackupStatus::Available); + let report = move_old(&fake, &runtime).await; + assert_eq!(report.rows, 1, "{report:?}"); + assert_eq!(report.stopping.len(), 1, "{report:?}"); + assert!( + report.stopping[0].1.contains("verified manifest"), + "{report:?}" + ); + assert!(fake.rewrites_seen.lock().unwrap().is_empty()); + + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + seed_rows(&fake, &[old_row(&deleting, BackupStatus::Deleting)]); + fake.set_stale_entry(&deleting, BackupStatus::Available); + let report = move_old(&fake, &runtime).await; + assert!(report.complete(), "{report:?}"); + assert_eq!(report.finished, std::slice::from_ref(&deleting)); + assert_eq!(report.moved, std::slice::from_ref(&deleting)); + let row = fake.rows.lock().unwrap()[&deleting].clone(); + assert_eq!((row.status, row.store), (BackupStatus::Deleted, store())); +} + +/// On a backend whose summary counts status entries (Cassandra), a row +/// whose one entry is under an earlier status counts as a row under its own +/// status in the report of other stores (and a DELETING one is named), +/// while its stale entry is reported as a status entry. +#[tokio::test] +async fn the_report_of_other_stores_counts_a_row_whose_entry_is_under_an_earlier_status() { + let fake = Arc::new(Fake::default()); + fake.counts_entries + .store(true, std::sync::atomic::Ordering::Release); + let available = sync_arn(1); + let deleting = sync_arn(2); + seed_rows( + &fake, + &[ + old_row(&available, BackupStatus::Available), + old_row(&deleting, BackupStatus::Deleting), + ], + ); + fake.set_stale_entry(&deleting, BackupStatus::Available); + let others = super::store_maintenance::other_stores(&*fake, &store()) + .await + .unwrap(); + assert_eq!(others.len(), 1, "{others:?}"); + assert_eq!( + others[0].counts, + [(BackupStatus::Available, 1), (BackupStatus::Deleting, 1)] + ); + assert_eq!(others[0].entries, [(BackupStatus::Available, 1)]); + let named: Vec<&str> = others[0] + .unresolved + .iter() + .map(|r| r.backup_arn.as_str()) + .collect(); + assert_eq!(named, [deleting.as_str()]); +} + +/// A stop during the wait before a DELETING row's finished deletion leaves +/// the row DELETING and its objects untouched: the ARN is not reached. +#[tokio::test] +async fn a_stop_during_the_wait_before_a_deleting_rows_finish_deletes_nothing() { + use super::store_maintenance::{ForgetConfigured, ForgetRun, forget_backups}; + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let configured = runtime.configured_store().unwrap(); + let identity = configured.identity().unwrap(); + let deleting = sync_arn(1); + put_backup(&mem, &deleting); + seed_rows(&fake, &[row(&deleting, BackupStatus::Deleting)]); + let whole = objects_under(&mem, &deleting).len(); + let waiting = std::sync::atomic::AtomicBool::new(false); + let on_wait = |_: Duration| waiting.store(true, std::sync::atomic::Ordering::Release); + let stop = || { + waiting + .load(std::sync::atomic::Ordering::Acquire) + .then(|| "SIGTERM received".to_owned()) + }; + let run = ForgetRun { + removals_from: Some(tokio::time::Instant::now() + Duration::from_secs(60)), + on_wait: &on_wait, + configured: Some(ForgetConfigured { + store: configured, + healthy: Some(&identity), + mismatch_row_ids: &[], + file_missing_with_rows: false, + }), + ..forget_run(Duration::from_secs(300), &stop) + }; + let report = forget_backups(&*fake, std::slice::from_ref(&deleting), REGION, run).await; + assert_eq!(report.stopped.as_deref(), Some("SIGTERM received")); + assert!(report.done.is_empty(), "{report:?}"); + assert_eq!(objects_under(&mem, &deleting).len(), whole); + assert_eq!( + fake.rows.lock().unwrap()[&deleting].status, + BackupStatus::Deleting + ); +} + +/// The backup manual's copied-identity-file case with a DELETING row among +/// the new root's rows: the root's file names the old root's id +/// (`store-old`) and its rows carry `store-1`, so the store's check fails +/// (`IdMismatch`). `--forget-backup` removes the AVAILABLE row and refuses +/// the DELETING one, saying that its own id takes part in the mismatch and +/// naming the way out (the same command with no store configured); the +/// check still fails afterwards. +#[tokio::test] +async fn a_deleting_row_behind_an_identity_mismatch_is_refused_naming_the_way_out() { + use super::store_maintenance::{ + ForgetBackup, ForgetConfigured, ForgetRun, Forgotten, forget_backups, + }; + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-old", "catalog-1")); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let configured = runtime.configured_store().unwrap(); + let available = sync_arn(1); + let deleting = sync_arn(2); + put_backup(&mem, &available); + put_backup(&mem, &deleting); + seed_rows( + &fake, + &[ + row(&available, BackupStatus::Available), + row(&deleting, BackupStatus::Deleting), + ], + ); + let problem = super::check_configured_store_read_only(configured, &*fake, None) + .await + .unwrap_err(); + assert!( + matches!(problem, StoreProblem::IdMismatch { .. }), + "{problem:?}" + ); + let run = ForgetRun { + configured: Some(ForgetConfigured { + store: configured, + healthy: None, + mismatch_row_ids: problem.mismatch_row_ids(), + file_missing_with_rows: false, + }), + ..forget_run(Duration::from_secs(300), &never) + }; + let asked = [available.clone(), deleting.clone()]; + let report = forget_backups(&*fake, &asked, REGION, run).await; + let refused = Forgotten::DeletionBehindMismatch("store-1".to_owned()); + assert_eq!( + report.done, + [ + ( + available.clone(), + ForgetBackup::Row( + BackupStatus::Available, + Forgotten::Removed(BackupStatus::Available) + ) + ), + ( + deleting.clone(), + ForgetBackup::Row(BackupStatus::Deleting, refused.clone()) + ), + ] + ); + let text = refused.to_string(); + assert!( + text.contains("store id store-1") + && text.contains("kind = \"disabled\"") + && text.contains("DeleteBackup it again"), + "{text}" + ); + assert_eq!( + objects_under(&mem, &deleting).len(), + backup_objects(&deleting).len() + ); + assert!(matches!( + super::check_configured_store_read_only(configured, &*fake, None).await, + Err(StoreProblem::IdMismatch { .. }) + )); +} + +/// A filesystem root the command found missing with rows for it (the +/// defaulted root moved away, or a link to a volume that is gone): its rows +/// are refused as an unmounted root's, with no store configured; a row of +/// another root in the same run is removed, one whose root's label extends +/// the missing one's included (the root is matched byte for byte). +#[tokio::test] +async fn forget_backup_refuses_the_rows_of_a_missing_root_with_rows() { + use super::store_maintenance::{ForgetBackup, ForgetRun, Forgotten, forget_backups}; + let fake = Arc::new(Fake::default()); + let theirs = sync_arn(1); + let ours = sync_arn(2); + let longer = sync_arn(3); + let mut longer_row = row(&longer, BackupStatus::Available); + longer_row.store.store_root = format!("{}2", store().store_root); + seed_rows( + &fake, + &[ + old_row(&theirs, BackupStatus::Available), + longer_row, + row(&ours, BackupStatus::Available), + ], + ); + let configured = store(); + let run = ForgetRun { + unmounted_root: Some(&configured.store_root), + ..forget_run(Duration::from_secs(300), &never) + }; + let asked = [theirs.clone(), longer.clone(), ours.clone()]; + let report = forget_backups(&*fake, &asked, REGION, run).await; + assert_eq!( + report.done, + [ + ( + theirs.clone(), + ForgetBackup::Row( + BackupStatus::Available, + Forgotten::Removed(BackupStatus::Available) + ) + ), + ( + longer.clone(), + ForgetBackup::Row( + BackupStatus::Available, + Forgotten::Removed(BackupStatus::Available) + ) + ), + ( + ours.clone(), + ForgetBackup::Row(BackupStatus::Available, Forgotten::RootWithoutFile) + ), + ] + ); + assert!(fake.rows.lock().unwrap().contains_key(&ours)); +} + +/// A store error while the move checks a row of the old root whose copy is +/// complete stops the move with that row named, for a DELETED, a FAILED, a +/// DELETING and an AVAILABLE row: no row is rewritten, no object is +/// deleted (the DELETING row's deletion is not finished) and the row is as +/// it was. +#[tokio::test] +async fn a_store_error_during_the_moves_checks_stops_it() { + for status in [ + BackupStatus::Deleted, + BackupStatus::Failed, + BackupStatus::Deleting, + BackupStatus::Available, + ] { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let arn = sync_arn(1); + put_backup(&mem, &arn); + let mut row = old_row(&arn, status); + if matches!(status, BackupStatus::Deleted | BackupStatus::Failed) { + row.completed_at = Some(1.0); + } + seed_rows(&fake, std::slice::from_ref(&row)); + *mem.fail_gets_under.lock().unwrap() = Some(prefix_of(&arn)); + let keys_before = mem.keys(); + let report = move_old(&fake, &runtime).await; + assert!(!report.complete(), "{status}: {report:?}"); + assert!(report.finished.is_empty(), "{status}: {report:?}"); + assert_eq!(report.stopping.len(), 1, "{status}: {report:?}"); + assert!( + report.stopping[0] + .1 + .contains("the configured store could not be read"), + "{status}: {}", + report.stopping[0].1 + ); + assert!( + fake.rewrites_seen.lock().unwrap().is_empty(), + "{status}: nothing is rewritten" + ); + assert_eq!(mem.keys(), keys_before, "{status}: no object was deleted"); + assert_eq!(fake.rows.lock().unwrap()[&arn], row, "{status}"); + } +} + +/// A failed read of the FAILED rows whose owner is dead leaves the row: a +/// job that may still write under its prefix is not taken for dead. +#[tokio::test] +async fn a_failed_read_of_the_failed_rows_owners_removes_nothing() { + use super::store_maintenance::{ForgetBackup, Forgotten, forget_backups}; + let fake = Arc::new(Fake::default()); + let failed = sync_arn(1); + let mut failed_row = old_row(&failed, BackupStatus::Failed); + failed_row.heartbeat_at = 1.0; + seed_rows(&fake, &[failed_row]); + *fake.older_than_error.lock().unwrap() = Some("pool exhausted".to_owned()); + let report = forget_backups( + &*fake, + std::slice::from_ref(&failed), + REGION, + forget_run(Duration::from_secs(300), &never), + ) + .await; + assert_eq!(report.done.len(), 1, "{report:?}"); + let ForgetBackup::Row(BackupStatus::Failed, Forgotten::Failed(why)) = &report.done[0].1 else { + panic!("the FAILED row is left as a failure: {report:?}"); + }; + assert!( + why.contains("the row's owner could not be tested") && why.contains("pool exhausted"), + "{why}" + ); + assert!(!report.complete(1)); + assert!(fake.rows.lock().unwrap().contains_key(&failed)); + assert!(fake.remove_attempts.lock().unwrap().is_empty()); + + // With the listing answering, the same row (its owner long dead) goes. + *fake.older_than_error.lock().unwrap() = None; + let report = forget_backups( + &*fake, + std::slice::from_ref(&failed), + REGION, + forget_run(Duration::from_secs(300), &never), + ) + .await; + assert!(report.complete(1), "{report:?}"); + assert!(!fake.rows.lock().unwrap().contains_key(&failed)); +} + +/// A DELETING row of the configured store that a server's reconciler +/// finishes during the forget's wait (it deletes the prefix and sets the +/// row DELETED under the claim): the forget's own finish then finds the row +/// no longer DELETING, reads it again and removes the DELETED row, so the +/// ARN is done with no row left behind. +#[tokio::test(start_paused = true)] +async fn a_deleting_row_a_server_finishes_during_the_wait_is_removed_deleted() { + use super::store_maintenance::{ + ForgetBackup, ForgetConfigured, ForgetRun, Forgotten, forget_backups, + }; + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let configured = runtime.configured_store().unwrap(); + let identity = configured.identity().unwrap(); + let deleting = sync_arn(1); + put_backup(&mem, &deleting); + seed_rows(&fake, &[row(&deleting, BackupStatus::Deleting)]); + let prefix = prefix_of(&deleting); + let server_finishes = |_: Duration| { + mem.objects + .lock() + .unwrap() + .retain(|key, _| !key.starts_with(&prefix)); + fake.rows.lock().unwrap().get_mut(&deleting).unwrap().status = BackupStatus::Deleted; + }; + let run = ForgetRun { + removals_from: Some(tokio::time::Instant::now() + Duration::from_secs(60)), + on_wait: &server_finishes, + configured: Some(ForgetConfigured { + store: configured, + healthy: Some(&identity), + mismatch_row_ids: &[], + file_missing_with_rows: false, + }), + ..forget_run(Duration::from_secs(300), &never) + }; + let report = forget_backups(&*fake, std::slice::from_ref(&deleting), REGION, run).await; + assert_eq!( + report.done, + [( + deleting.clone(), + ForgetBackup::Row( + BackupStatus::Deleting, + Forgotten::Removed(BackupStatus::Deleted) + ) + )] + ); + assert!(report.complete(1)); + assert!(!fake.rows.lock().unwrap().contains_key(&deleting)); + assert!( + fake.set_deleted.lock().unwrap().is_empty(), + "the forget's own write found the row DELETED" + ); +} + +/// On a backend whose summary counts status entries (Cassandra), the report +/// of other stores keeps each store id of a root to its own rows: with two +/// ids under one root (what an identity mismatch leaves), each id counts +/// and names only the rows that carry it. +#[tokio::test] +async fn the_report_of_other_stores_keeps_two_ids_of_one_root_apart() { + let fake = Arc::new(Fake::default()); + fake.counts_entries + .store(true, std::sync::atomic::Ordering::Release); + let available = sync_arn(1); + let deleting = sync_arn(2); + let second = StoreIdentity { + store_id: "store-9".to_owned(), + ..old_store() + }; + let mut deleting_row = old_row(&deleting, BackupStatus::Deleting); + deleting_row.store = second.clone(); + seed_rows( + &fake, + &[old_row(&available, BackupStatus::Available), deleting_row], + ); + let others = super::store_maintenance::other_stores(&*fake, &store()) + .await + .unwrap(); + assert_eq!(others.len(), 2, "{others:?}"); + let of = |store: &StoreIdentity| { + others + .iter() + .find(|other| other.store == *store) + .unwrap_or_else(|| panic!("{others:?}")) + }; + let first = of(&old_store()); + assert_eq!(first.counts, [(BackupStatus::Available, 1)]); + assert!(first.unresolved.is_empty(), "{first:?}"); + assert!(first.entries.is_empty(), "{first:?}"); + let other = of(&second); + assert_eq!(other.counts, [(BackupStatus::Deleting, 1)]); + assert_eq!( + other + .unresolved + .iter() + .map(|r| r.backup_arn.as_str()) + .collect::>(), + [deleting.as_str()] + ); + assert!(other.entries.is_empty(), "{other:?}"); +} + +// --- the claim re-checked before the options' steps that cannot be undone -- + +/// `--forget-store` re-checks the claim with a beat right before each +/// removal, before a DELETING row's DELETED write and again before that +/// row's removal, and before the sweep of the root's status entries. A +/// claim another holder took over stops the run at the step its beat +/// precedes: the row is as the steps before it left it, and the run's stop +/// says so. +#[tokio::test] +async fn forget_store_rechecks_the_claim_before_each_step_that_cannot_be_undone() { + use super::store_maintenance::{ForgetRun, forget_store}; + async fn forget( + fake: &Fake, + claim: &maintenance::HeldClaim, + ) -> super::store_maintenance::ForgetStoreReport { + forget_store( + fake, + StoreKind::Filesystem, + "/srv/old", + ForgetRun { + claim: Some(claim), + ..forget_run(Duration::ZERO, &never) + }, + ) + .await + .unwrap() + } + let arn = sync_arn(1); + for status in [ + BackupStatus::Available, + BackupStatus::Creating, + BackupStatus::Failed, + BackupStatus::Deleted, + ] { + let mut seeded = old_row(&arn, status); + seeded.completed_at = Some(1.0); + // Lost at the removal's beat: the row stays. + let fake = Fake::default(); + seed_rows(&fake, std::slice::from_ref(&seeded)); + let (claim, beats) = scripted_claim(0); + let report = forget(&fake, &claim).await; + let stopped = report.stopped.clone().expect("stopped"); + assert!( + stopped.contains("another holder took it over") + && stopped.contains(&format!("before the {status} row's removal")), + "{status}: {stopped}" + ); + assert_eq!(beats_of(&beats), 1, "{status}"); + assert!(report.rows.is_empty(), "{status}: {report:?}"); + assert_eq!(fake.rows.lock().unwrap()[&arn].status, status); + assert!(report.entries_swept.is_none()); + + // Lost at the sweep's beat: the row is removed, the sweep not run. + let fake = Fake::default(); + seed_rows(&fake, std::slice::from_ref(&seeded)); + let (claim, beats) = scripted_claim(1); + let report = forget(&fake, &claim).await; + assert_eq!(beats_of(&beats), 2, "{status}"); + assert_eq!(report.removed(), 1, "{status}: {report:?}"); + assert!(!fake.rows.lock().unwrap().contains_key(&arn)); + let stopped = report.stopped.clone().expect("stopped"); + assert!( + stopped.contains("before the sweep of the root's status entries"), + "{status}: {stopped}" + ); + assert!(report.entries_swept.is_none(), "{status}"); + assert!(!report.complete()); + + // Every beat confirms. + let fake = Fake::default(); + seed_rows(&fake, std::slice::from_ref(&seeded)); + let (claim, beats) = scripted_claim(2); + let report = forget(&fake, &claim).await; + assert!(report.complete(), "{status}: {report:?}"); + assert_eq!(beats_of(&beats), 2, "{status}"); + } + + // A DELETING row of another store: a beat before its DELETED write and + // one before its removal. + let seeded = old_row(&arn, BackupStatus::Deleting); + let fake = Fake::default(); + seed_rows(&fake, std::slice::from_ref(&seeded)); + let (claim, beats) = scripted_claim(0); + let report = forget(&fake, &claim).await; + let stopped = report.stopped.clone().expect("stopped"); + assert!( + stopped.contains("before the DELETING row's DELETED write; the row is unchanged"), + "{stopped}" + ); + assert_eq!(beats_of(&beats), 1); + assert_eq!( + fake.rows.lock().unwrap()[&arn].status, + BackupStatus::Deleting + ); + + let fake = Fake::default(); + seed_rows(&fake, std::slice::from_ref(&seeded)); + let (claim, beats) = scripted_claim(1); + let report = forget(&fake, &claim).await; + let stopped = report.stopped.clone().expect("stopped"); + assert!( + stopped.contains("before the DELETED row's removal; the row is DELETED"), + "{stopped}" + ); + assert_eq!(beats_of(&beats), 2); + assert_eq!( + fake.rows.lock().unwrap()[&arn].status, + BackupStatus::Deleted + ); + + let fake = Fake::default(); + seed_rows(&fake, std::slice::from_ref(&seeded)); + let (claim, beats) = scripted_claim(3); + let report = forget(&fake, &claim).await; + assert!(report.complete(), "{report:?}"); + assert_eq!(beats_of(&beats), 3); + assert!(fake.rows.lock().unwrap().is_empty()); +} + +/// `--forget-backup` of a DELETING row of the configured store re-checks the +/// claim right before it finishes the row's deletion and again right before +/// it removes the DELETED row: lost at the first, the row and its objects +/// stay; lost at the second, the objects are deleted and the row is left +/// DELETED for a second run. +#[tokio::test] +async fn forget_backup_rechecks_the_claim_before_a_deleting_rows_deletion_and_its_removal() { + use super::store_maintenance::{ + ForgetBackup, ForgetConfigured, ForgetRun, Forgotten, forget_backups, + }; + let mem = Arc::new(MemStore::default()); + let runtime = runtime_on(&mem, store()); + let configured = runtime.configured_store().unwrap(); + let identity = configured.identity().unwrap(); + let deleting = sync_arn(1); + fn run<'a>( + store: &'a ConfiguredStore, + healthy: &'a StoreIdentity, + claim: &'a maintenance::HeldClaim, + ) -> ForgetRun<'a> { + ForgetRun { + configured: Some(ForgetConfigured { + store, + healthy: Some(healthy), + mismatch_row_ids: &[], + file_missing_with_rows: false, + }), + claim: Some(claim), + ..forget_run(Duration::from_secs(300), &never) + } + } + + // Lost before the deletion. + put_backup(&mem, &deleting); + let whole = objects_under(&mem, &deleting).len(); + assert!(whole > 0); + let fake = Fake::default(); + seed_rows(&fake, &[row(&deleting, BackupStatus::Deleting)]); + let (claim, beats) = scripted_claim(0); + let report = forget_backups( + &fake, + std::slice::from_ref(&deleting), + REGION, + run(configured, &identity, &claim), + ) + .await; + let stopped = report.stopped.clone().expect("stopped"); + assert!( + stopped.contains("another holder took it over") + && stopped.contains( + "before the DELETING row's deletion in the configured store; the row and its \ + objects are unchanged" + ), + "{stopped}" + ); + assert!(report.done.is_empty(), "{report:?}"); + assert_eq!(beats_of(&beats), 1); + assert_eq!(objects_under(&mem, &deleting).len(), whole); + assert_eq!( + fake.rows.lock().unwrap()[&deleting].status, + BackupStatus::Deleting + ); + + // Lost before the DELETED row's removal. + let (claim, beats) = scripted_claim(1); + let report = forget_backups( + &fake, + std::slice::from_ref(&deleting), + REGION, + run(configured, &identity, &claim), + ) + .await; + let stopped = report.stopped.clone().expect("stopped"); + assert!( + stopped.contains("before the DELETED row's removal; its deletion is finished"), + "{stopped}" + ); + assert_eq!(beats_of(&beats), 2); + assert!( + objects_under(&mem, &deleting).is_empty(), + "the prefix is deleted" + ); + assert_eq!( + fake.rows.lock().unwrap()[&deleting].status, + BackupStatus::Deleted + ); + + // Both confirm: a fresh DELETING row is finished and removed. + put_backup(&mem, &deleting); + let fake = Fake::default(); + seed_rows(&fake, &[row(&deleting, BackupStatus::Deleting)]); + let (claim, beats) = scripted_claim(2); + let report = forget_backups( + &fake, + std::slice::from_ref(&deleting), + REGION, + run(configured, &identity, &claim), + ) + .await; + assert_eq!( + report.done, + [( + deleting.clone(), + ForgetBackup::Row(BackupStatus::Deleting, Forgotten::RemovedAfterDeletion) + )] + ); + assert_eq!(beats_of(&beats), 2); + assert!(!fake.rows.lock().unwrap().contains_key(&deleting)); +} + +/// `--move-store` under `claim` into the store `runtime` is configured +/// with, with the servers' 300 s owner window. +async fn move_old_under( + fake: &Fake, + runtime: &BackupRuntime, + claim: &maintenance::HeldClaim, +) -> super::store_maintenance::MoveReport { + let configured = runtime.configured_store().unwrap(); + let identity = configured.identity().unwrap(); + super::store_maintenance::move_store( + fake, + super::store_maintenance::MoveTarget { + configured, + identity: &identity, + from_kind: StoreKind::Filesystem, + from_root: "/srv/old", + owner_window: Duration::from_secs(300), + stop: &never, + claim: Some(claim), + }, + ) + .await + .unwrap() +} + +/// `--move-store` re-checks the claim with a beat right before it resolves +/// a dead CREATING row, right before it finishes a DELETING row's deletion +/// in the configured store, and right before the rewrite. A claim another +/// holder took over stops the move there: the row keeps its status, the +/// copy its objects, and no row is moved. +#[tokio::test] +async fn move_store_rechecks_the_claim_before_each_resolution_deletion_and_the_rewrite() { + // A dead CREATING row whose copy is complete: lost before its + // resolution. + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let creating = seed_dead_creating_with_manifest(&fake, &mem, true); + fake.rows.lock().unwrap().get_mut(&creating).unwrap().store = old_store(); + let (claim, beats) = scripted_claim(0); + let report = move_old_under(&fake, &runtime, &claim).await; + let stopped = report.stopped.clone().expect("stopped"); + assert!( + stopped.contains("another holder took it over") + && stopped.contains(&format!( + "before the resolution of the CREATING row {creating}; no row was moved" + )), + "{stopped}" + ); + assert_eq!(beats_of(&beats), 1); + assert!(report.resolved.is_empty()); + assert_eq!( + fake.rows.lock().unwrap()[&creating].status, + BackupStatus::Creating + ); + assert!(fake.rewrites_seen.lock().unwrap().is_empty()); + + // Its resolution confirmed, lost before the rewrite. + let (claim, beats) = scripted_claim(1); + let report = move_old_under(&fake, &runtime, &claim).await; + assert_eq!(beats_of(&beats), 2); + assert_eq!( + report.resolved, + [(creating.clone(), BackupStatus::Available)] + ); + let stopped = report.stopped.clone().expect("stopped"); + assert!( + stopped.contains("before the rewrite; no row was moved"), + "{stopped}" + ); + assert!(fake.rewrites_seen.lock().unwrap().is_empty()); + assert_eq!(fake.rows.lock().unwrap()[&creating].store, old_store()); + + // Now AVAILABLE: one beat, before the rewrite, which moves it. + let (claim, beats) = scripted_claim(1); + let report = move_old_under(&fake, &runtime, &claim).await; + assert!(report.complete(), "{report:?}"); + assert_eq!(beats_of(&beats), 1); + assert_eq!(fake.rewrites_seen.lock().unwrap().len(), 1); + assert_eq!(fake.rows.lock().unwrap()[&creating].store, store()); + + // A DELETING row whose copy holds no verified manifest: lost before its + // deletion in the configured store. + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let deleting = sync_arn(2); + let leftover = format!("{}data/000001.json.gz", prefix_of(&deleting)); + mem.insert(&leftover, b"x"); + seed_rows(&fake, &[old_row(&deleting, BackupStatus::Deleting)]); + let (claim, beats) = scripted_claim(0); + let report = move_old_under(&fake, &runtime, &claim).await; + let stopped = report.stopped.clone().expect("stopped"); + assert!( + stopped.contains(&format!( + "before the deletion of the DELETING row {deleting} in the configured store; no row \ + was moved" + )), + "{stopped}" + ); + assert_eq!(beats_of(&beats), 1); + assert!(report.finished.is_empty()); + assert!(mem.keys().contains(&leftover), "the copy's object is left"); + assert_eq!( + fake.rows.lock().unwrap()[&deleting].status, + BackupStatus::Deleting + ); + assert!(fake.rewrites_seen.lock().unwrap().is_empty()); + + // Both confirm: finished, then moved. + let (claim, beats) = scripted_claim(2); + let report = move_old_under(&fake, &runtime, &claim).await; + assert!(report.complete(), "{report:?}"); + assert_eq!(beats_of(&beats), 2); + assert_eq!(report.finished, std::slice::from_ref(&deleting)); + assert!(!mem.keys().contains(&leftover)); + assert_eq!(fake.rows.lock().unwrap()[&deleting].store, store()); +} + +/// The rewrite runs in slices of `REWRITE_SLICE` keys with a beat of the +/// claim before each: a claim another holder took after the first slice +/// (a command stopped past the stale window on MongoDB or Cassandra, whose +/// rewrites run key by key or in chunks) stops the move there, with the +/// first slice's rows moved and the rest left under the old root and +/// listed, and the stop says which rows it came before. A claim kept +/// through every slice moves them all, one `rewrite_store` call per slice. +#[tokio::test] +async fn move_store_rechecks_the_claim_before_each_slice_of_the_rewrite() { + use super::store_maintenance::REWRITE_SLICE; + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let count = REWRITE_SLICE + 1; + let rows: Vec = (1..=count) + .map(|n| { + let mut row = old_row(&sync_arn(u32::try_from(n).unwrap()), BackupStatus::Failed); + row.completed_at = Some(1.0); + row + }) + .collect(); + seed_rows(&fake, &rows); + let last = rows.last().unwrap().backup_arn.clone(); + + // Confirmed before the first slice, lost before the second. + let (claim, beats) = scripted_claim(1); + let report = move_old_under(&fake, &runtime, &claim).await; + assert_eq!(beats_of(&beats), 2); + let stopped = report.stopped.clone().expect("stopped"); + assert!( + stopped.contains("another holder took it over") + && stopped.contains(&format!( + "before the rewrite of row {count} of {count}; the {REWRITE_SLICE} row(s) \ + rewritten before it are moved, and the rest are left under the old root" + )), + "{stopped}" + ); + assert!(!report.complete()); + assert_eq!(report.rewritten, REWRITE_SLICE as u64); + assert_eq!(report.moved.len(), REWRITE_SLICE); + assert!(!report.moved.contains(&last)); + assert_eq!( + report.left, + [super::store_maintenance::NamedRow { + status: BackupStatus::Failed, + backup_arn: last.clone(), + account_id: ACCOUNT.to_owned(), + }] + ); + { + let calls = fake.rewrites_seen.lock().unwrap(); + assert_eq!(calls.len(), 1, "one slice was sent"); + assert_eq!(calls[0].2.len(), REWRITE_SLICE); + assert!(!calls[0].2.iter().any(|k| k.backup_arn == last)); + } + { + let held = fake.rows.lock().unwrap(); + assert_eq!(held[&last].store, old_store(), "the last row is left"); + for row in &rows[..REWRITE_SLICE] { + assert_eq!(held[&row.backup_arn].store, store(), "{}", row.backup_arn); + } + } + + // The claim kept: the row left is moved in one slice of one key. + fake.rewrites_seen.lock().unwrap().clear(); + let (claim, beats) = scripted_claim(2); + let report = move_old_under(&fake, &runtime, &claim).await; + assert!(report.complete(), "{report:?}"); + assert_eq!(beats_of(&beats), 1); + assert_eq!(report.moved, std::slice::from_ref(&last)); + assert_eq!(fake.rewrites_seen.lock().unwrap().len(), 1); + + // Every slice confirmed from the start: two calls, the second of one key. + let fake = Arc::new(Fake::default()); + seed_rows(&fake, &rows); + let (claim, beats) = scripted_claim(2); + let report = move_old_under(&fake, &runtime, &claim).await; + assert!(report.complete(), "{report:?}"); + assert_eq!(beats_of(&beats), 2); + assert_eq!(report.rewritten, count as u64); + assert_eq!(report.moved.len(), count); + let calls = fake.rewrites_seen.lock().unwrap(); + assert_eq!(calls.len(), 2); + assert_eq!(calls[0].2.len(), REWRITE_SLICE); + assert_eq!(calls[1].2.len(), 1); + assert_eq!(calls[1].2[0].backup_arn, last); +} + +/// `count` FAILED rows of the old root, each with no prefix in the +/// configured store, so each moves. +fn failed_old_rows(count: u32) -> Vec { + (1..=count) + .map(|n| { + let mut row = old_row(&sync_arn(n), BackupStatus::Failed); + row.completed_at = Some(1.0); + row + }) + .collect() +} + +/// On a backend that rewrites key by key (Cassandra) the move calls +/// `rewrite_store` with one key at a time and reads the claim's local state +/// before each call, with no beat. A command stopped inside the rewrite of +/// a root of fewer than `REWRITE_SLICE` rows (here 10, stopped in the +/// second key's call) whose claim another holder took during the stop +/// reads the claim lost as it resumes and stops before the third key: the +/// two keys rewritten are moved, the eight others are left under the old +/// root and listed, and no further call is sent. The beat before the first +/// call confirmed the claim; the timer's beat during the stop found it +/// lost. +#[tokio::test(start_paused = true)] +async fn move_store_on_a_key_by_key_backend_stops_at_the_next_key_once_the_claim_reads_lost() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let rows = failed_old_rows(10); + seed_rows(&fake, &rows); + fake.rewrites_key_by_key + .store(true, std::sync::atomic::Ordering::Release); + *fake.rewrite_sleep.lock().unwrap() = Some((2, Duration::from_secs(10))); + let beats = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let claim = maintenance::HeldClaim::start( + Box::new(ScriptedClaim { + confirms: 1, + beats: Arc::clone(&beats), + }), + Duration::from_secs(1), + ); + let report = move_old_under(&fake, &runtime, &claim).await; + assert_eq!( + beats_of(&beats), + 2, + "the beat before the rewrite, the timer's" + ); + let stopped = report.stopped.clone().expect("stopped"); + assert!( + stopped.contains("another holder took it over") + && stopped.contains( + "before the rewrite of row 3 of 10; the 2 row(s) rewritten before it are moved, \ + and the rest are left under the old root" + ), + "{stopped}" + ); + assert_eq!(report.rewritten, 2); + assert_eq!( + report.moved, + [rows[0].backup_arn.clone(), rows[1].backup_arn.clone()] + ); + let left: Vec<&str> = report.left.iter().map(|r| r.backup_arn.as_str()).collect(); + let expected: Vec<&str> = rows[2..].iter().map(|r| r.backup_arn.as_str()).collect(); + assert_eq!(left, expected); + { + let calls = fake.rewrites_seen.lock().unwrap(); + assert_eq!(calls.len(), 2, "no call after the claim read lost"); + assert!( + calls.iter().all(|call| call.2.len() == 1), + "one key per call" + ); + } + let held = fake.rows.lock().unwrap(); + for (n, row) in rows.iter().enumerate() { + let want = if n < 2 { store() } else { old_store() }; + assert_eq!(held[&row.backup_arn].store, want, "{}", row.backup_arn); + } +} + +/// A signal (Ctrl-C, SIGTERM) received during the rewrite stops the move +/// before its next call, as a claim lost there does: the first slice's rows +/// are moved, the rest are left under the old root and listed, and the +/// stop says which row it came before. No beat is sent for the slice not +/// started. +#[tokio::test] +async fn a_signal_during_the_rewrite_stops_the_move_before_its_next_slice() { + use super::store_maintenance::REWRITE_SLICE; + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let count = REWRITE_SLICE + 1; + let rows = failed_old_rows(u32::try_from(count).unwrap()); + seed_rows(&fake, &rows); + let last = rows.last().unwrap().backup_arn.clone(); + let signalled = + || (!fake.rewrites_seen.lock().unwrap().is_empty()).then(|| "SIGTERM received".to_owned()); + let (claim, beats) = scripted_claim(5); + let configured = runtime.configured_store().unwrap(); + let identity = configured.identity().unwrap(); + let report = super::store_maintenance::move_store( + &*fake, + super::store_maintenance::MoveTarget { + configured, + identity: &identity, + from_kind: StoreKind::Filesystem, + from_root: "/srv/old", + owner_window: Duration::from_secs(300), + stop: &signalled, + claim: Some(&claim), + }, + ) + .await + .unwrap(); + assert_eq!(beats_of(&beats), 1, "the beat before the first slice only"); + let stopped = report.stopped.clone().expect("stopped"); + assert!( + stopped.starts_with("SIGTERM received") + && stopped.contains(&format!( + "before the rewrite of row {count} of {count}; the {REWRITE_SLICE} row(s) \ + rewritten before it are moved" + )), + "{stopped}" + ); + assert!(!report.complete()); + assert_eq!(report.moved.len(), REWRITE_SLICE); + assert_eq!( + report.left, + [super::store_maintenance::NamedRow { + status: BackupStatus::Failed, + backup_arn: last.clone(), + account_id: ACCOUNT.to_owned(), + }] + ); + assert_eq!(fake.rewrites_seen.lock().unwrap().len(), 1); + assert_eq!(fake.rows.lock().unwrap()[&last].store, old_store()); +} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 89cb73648..205a201ca 100755 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -12,7 +12,8 @@ pub use backup::{ BackupRuntime, ConfiguredStore, HeldClaim, JobSettings, KillPoint, LOCAL_CHECK_TIMEOUT, OTHER_STORE, RESTORE_REGISTRATION_TIMEOUT, S3_CHECK_TIMEOUT, STORE_IDENTITY_KEY, Spool, StoreFence, StoreHealth, StoreHealthReport, StoreIdentityFile, StoreProblem, TARGET_NOT_EMPTY, - check_configured_store, maintenance, read_identity_bounded, reconciler, store_identity, sync, + check_configured_store, check_configured_store_read_only, maintenance, read_identity_bounded, + reconciler, store_identity, store_maintenance, sync, }; mod batch_get_item; mod batch_write_item; diff --git a/crates/server/src/backup_store.rs b/crates/server/src/backup_store.rs index c0046a032..6155d9cbb 100644 --- a/crates/server/src/backup_store.rs +++ b/crates/server/src/backup_store.rs @@ -87,7 +87,7 @@ //! the operations that touch the store refused, and the check run again //! each reconciler cycle until it passes. -use std::path::{Path, PathBuf}; +use std::path::{Component, Path, PathBuf}; use std::sync::Arc; use extenddb_backup_store::{BackupStore, BackupStoreKind, StoreError}; @@ -138,8 +138,10 @@ pub enum Caller { /// run): as `Command`, and no check of the store runs, so a root is left /// as it is found (neither the default directory nor an identity file is /// created; a default root that does not exist yet leaves the store - /// unconfigured, logged); the store is returned `checking` and the - /// command reads what it needs itself. + /// unconfigured, logged, as [`NotConfigured::DefaultRootMissing`] or, + /// when the catalog holds rows for it, as + /// [`NotConfigured::DefaultRootMissingWithRows`]); the store is + /// returned `checking` and the command reads what it needs itself. ReadOnlyCommand, } @@ -151,9 +153,19 @@ pub enum NotConfigured { Absent, /// `kind = "disabled"`. Disabled, - /// A read-only command found that the default root does not exist yet: - /// no server has run with this database file. + /// A read-only command found that the default root does not exist yet, + /// and the catalog holds no rows for it: no server has run with this + /// database file. DefaultRootMissing(PathBuf), + /// A read-only command found the default root missing while the catalog + /// holds rows for it: moved away, or a symbolic link to a volume that + /// is gone. `label` is the root as the rows record it (the link's + /// target, for a link), `store_ids` the ids they carry. + DefaultRootMissingWithRows { + root: PathBuf, + label: String, + store_ids: Vec, + }, /// The defaulted root could not be created, opened, validated or checked /// while the catalog has no rows for it (module doc): the warning path. DefaultRootUnusable { @@ -181,6 +193,18 @@ impl std::fmt::Display for NotConfigured { file, so there are no store-backed backups to list, purge or sync", root.display() ), + Self::DefaultRootMissingWithRows { + root, + label, + store_ids, + } => write!( + f, + "the default root {} is missing while the catalog holds rows for {label} (store \ + id {}): a volume that is not mounted, a directory moved away, or a link to a \ + volume that is gone; put the root back before listing, purging or syncing it", + root.display(), + store_ids.join(", ") + ), Self::DefaultRootUnusable { root, problem } => write!( f, "the default root {root} failed its check ({problem}) and the catalog has no \ @@ -335,6 +359,15 @@ pub async fn configure_described( && let Some(fs) = &config.filesystem && !fs.path.exists() { + if let Some((label, store_ids)) = missing_root_with_rows(catalog, &fs.path).await? { + let why = NotConfigured::DefaultRootMissingWithRows { + root: fs.path.clone(), + label, + store_ids, + }; + tracing::warn!("backup store: {why}"); + return Ok(StoreSetup::without_store(runtime, why)); + } tracing::info!( "backup store: the default root {} does not exist yet (no server has run with this \ database file); there are no store-backed backups to list", @@ -779,6 +812,117 @@ fn banner_target(config: &extenddb_backup_store::BackupStoreConfig) -> String { } } +/// How many symbolic links [`link_chain`] replaces before it stops: the +/// usual `SYMLOOP_MAX`, past which the kernel refuses the path (`ELOOP`). +const LINK_HOPS: usize = 40; + +/// The label of a defaulted filesystem root that does not exist, and the +/// store ids the catalog records for it, when it records any. The labels +/// tried, in order: each path of [`link_chain`] (the path as configured, +/// then the path after each symbolic link along it is replaced by its +/// target), each also with its longest existing ancestor canonicalized +/// ([`canonical_ancestor`]). Rows recorded through links carry the +/// canonical directory the last target names, which the last of them is +/// when the target's own directory is gone. +async fn missing_root_with_rows( + catalog: &C, + path: &Path, +) -> anyhow::Result)>> { + let mut labels: Vec = Vec::new(); + for path in &link_chain(path) { + for label in [Some(path.clone()), canonical_ancestor(path)] + .into_iter() + .flatten() + { + let label = label.display().to_string(); + if !labels.contains(&label) { + labels.push(label); + } + } + } + for label in labels { + let ids = catalog + .read_store_ids(StoreKind::Filesystem, &label) + .await + .map_err(|e| { + anyhow::anyhow!("the catalog's rows for the default root could not be read: {e}") + })?; + if !ids.is_empty() { + return Ok(Some((label, ids))); + } + } + Ok(None) +} + +/// `path`, then the path each time the first symbolic link along it, read +/// component by component from the start, is replaced by its target (a +/// relative target joined to the link's directory), whether or not the +/// target exists, the way the kernel resolves a path; at most +/// [`LINK_HOPS`] links, so a loop of links ends. +fn link_chain(path: &Path) -> Vec { + let mut chain = vec![path.to_path_buf()]; + let mut current = path.to_path_buf(); + for _ in 0..LINK_HOPS { + let components: Vec> = current.components().collect(); + let mut prefix = PathBuf::new(); + let mut replaced = None; + for (i, component) in components.iter().enumerate() { + prefix.push(component); + if !matches!(component, Component::Normal(_)) { + continue; + } + if let Ok(target) = std::fs::read_link(&prefix) { + let base = match prefix.parent() { + Some(parent) if target.is_relative() => parent.join(target), + _ => target, + }; + replaced = Some( + components[i + 1..] + .iter() + .fold(base, |path, rest| path.join(rest)), + ); + break; + } + } + match replaced { + Some(next) => { + chain.push(next.clone()); + current = next; + } + None => break, + } + } + chain +} + +/// `path` with its longest existing ancestor canonicalized and the rest of +/// its components joined back on (a `..` among them taking off the +/// component before it, since none of them exists to be a link); `None` +/// when no ancestor resolves. +fn canonical_ancestor(path: &Path) -> Option { + let mut rest: Vec> = Vec::new(); + let mut current = path; + loop { + if let Ok(canonical) = std::fs::canonicalize(current) { + let mut resolved = canonical; + for component in rest.iter().rev() { + if *component == Component::ParentDir { + resolved.pop(); + } else { + resolved.push(component); + } + } + return Some(resolved); + } + match current.components().next_back()? { + component @ (Component::Normal(_) | Component::ParentDir) => rest.push(component), + Component::CurDir => {} + Component::RootDir | Component::Prefix(_) => return None, + } + current = current.parent()?; + } +} + /// A path for an operator's line: quoted when it is empty or begins or ends /// with whitespace, so the line shows where it ends. fn shown_path(path: &Path) -> String { @@ -1027,17 +1171,19 @@ mod tests { use extenddb_storage::BoxedFuture; use extenddb_storage::backup_catalog::{ BackupKey, BackupRow, BackupStatus, DeleteStart, JobClaim, MaintenanceClaim, RestoreJobRow, - RestoreTargetKey, StoreIdentity, TimeBounds, + RestoreTargetKey, StoreIdentity, StoreRootCount, TimeBounds, }; use extenddb_storage::error::StorageError; use futures::stream::BoxStream; - /// A catalog whose rows for every root carry `ids`, or whose rows cannot - /// be read (`rows_unreadable`), with no backup row to list; every other - /// method is an error, since the startup checks never reach it. + /// A catalog whose rows for every root carry `ids` (only for the root + /// `ids_root` when it is set), or whose rows cannot be read + /// (`rows_unreadable`), with no backup row to list; every other method + /// is an error, since the startup checks never reach it. #[derive(Default)] struct StubCatalog { ids: Vec, + ids_root: Option, rows_unreadable: bool, } @@ -1153,6 +1299,8 @@ mod tests { fn remove_backup_row<'a>( &'a self, _key: &'a BackupKey, + _store_kind: StoreKind, + _store_root: &'a str, ) -> BoxedFuture<'a, Result> { unused() } @@ -1209,7 +1357,7 @@ mod tests { fn read_store_ids<'a>( &'a self, _store_kind: StoreKind, - _store_root: &'a str, + store_root: &'a str, ) -> BoxedFuture<'a, Result, StorageError>> { if self.rows_unreadable { return Box::pin(async { @@ -1218,9 +1366,34 @@ mod tests { )) }); } - let ids = self.ids.clone(); + let ids = match &self.ids_root { + Some(root) if root != store_root => Vec::new(), + _ => self.ids.clone(), + }; Box::pin(async move { Ok(ids) }) } + fn list_backup_rows_by_root<'a>( + &'a self, + _store_kind: StoreKind, + _store_root: &'a str, + _status: Option, + ) -> BoxStream<'a, Result> { + Box::pin(futures::stream::once(unused())) + } + fn rewrite_store<'a>( + &'a self, + _from_kind: StoreKind, + _from_root: &'a str, + _keys: &'a [BackupKey], + _to: &'a StoreIdentity, + ) -> BoxedFuture<'a, Result> { + unused() + } + fn summarize_store_roots( + &self, + ) -> BoxedFuture<'_, Result, StorageError>> { + unused() + } fn maintenance_claim<'a>( &'a self, _holder: &'a str, @@ -1652,6 +1825,17 @@ mod tests { .to_string() .contains("/srv/db/backups does not exist yet") ); + let with_rows = NotConfigured::DefaultRootMissingWithRows { + root: PathBuf::from("/srv/db/backups"), + label: "/mnt/vol/backups".to_owned(), + store_ids: vec!["store-1".to_owned()], + } + .to_string(); + assert!( + with_rows.contains("/srv/db/backups is missing while the catalog holds rows for /mnt/vol/backups (store id store-1)") + && !with_rows.contains("no server has run"), + "{with_rows}" + ); let unusable = NotConfigured::DefaultRootUnusable { root: "/srv/db/backups".to_owned(), problem: "extenddb-store.json names catalog other".to_owned(), @@ -1751,4 +1935,102 @@ mod tests { } let _ = std::fs::remove_dir_all(&dir); } + + /// The labels a missing default root's rows are looked up under follow + /// every symbolic link along the path: a chain of two links, a + /// directory link in the middle of the path, a chain of relative + /// links, a path spelled through a linked directory, and one spelled + /// with `..` (its canonical form), each with the link's volume renamed + /// away; a loop of links ends with no label found. + #[tokio::test] + async fn a_missing_default_root_is_looked_up_through_every_link_of_its_path() { + use std::os::unix::fs::symlink; + let dir = tempfile::tempdir().unwrap(); + let base = dir.path().canonicalize().unwrap(); + let found = |label: &Path| StubCatalog { + ids: vec!["store-1".to_owned()], + ids_root: Some(label.display().to_string()), + ..StubCatalog::default() + }; + let lookup = |catalog: StubCatalog, path: PathBuf| async move { + missing_root_with_rows(&catalog, &path).await.unwrap() + }; + let expect = |label: &Path| Some((label.display().to_string(), vec!["store-1".to_owned()])); + + // backups -> l2 -> vol/backups, absolute targets, vol renamed away. + let a = base.join("a"); + std::fs::create_dir_all(a.join("vol/backups")).unwrap(); + symlink(a.join("vol/backups"), a.join("l2")).unwrap(); + symlink(a.join("l2"), a.join("backups")).unwrap(); + std::fs::rename(a.join("vol"), a.join("vol-away")).unwrap(); + let label = a.join("vol/backups"); + assert_eq!( + lookup(found(&label), a.join("backups")).await, + expect(&label) + ); + + // data -> mnt/disk1, the path data/backups, disk1 renamed away. + let b = base.join("b"); + std::fs::create_dir_all(b.join("mnt/disk1/backups")).unwrap(); + symlink(b.join("mnt/disk1"), b.join("data")).unwrap(); + std::fs::rename(b.join("mnt/disk1"), b.join("mnt/disk1-away")).unwrap(); + let label = b.join("mnt/disk1/backups"); + assert_eq!( + lookup(found(&label), b.join("data/backups")).await, + expect(&label) + ); + + // backups -> l2 -> vol/backups, relative targets, vol renamed away. + let c = base.join("c"); + std::fs::create_dir_all(c.join("vol/backups")).unwrap(); + symlink("vol/backups", c.join("l2")).unwrap(); + symlink("l2", c.join("backups")).unwrap(); + std::fs::rename(c.join("vol"), c.join("vol-away")).unwrap(); + let label = c.join("vol/backups"); + assert_eq!( + lookup(found(&label), c.join("backups")).await, + expect(&label) + ); + + // The database's directory reached through a linked directory: the + // rows carry the real directory. + let e = base.join("e"); + std::fs::create_dir_all(e.join("real/db")).unwrap(); + symlink(e.join("real"), e.join("alias")).unwrap(); + let label = e.join("real/db/backups"); + assert_eq!( + lookup(found(&label), e.join("alias/db/backups")).await, + expect(&label) + ); + + // A path spelled with `..`: its canonical form. + let d = base.join("d"); + std::fs::create_dir_all(d.join("db/sub")).unwrap(); + let label = d.join("db/backups"); + assert_eq!( + lookup(found(&label), d.join("db/sub/../backups")).await, + expect(&label) + ); + + // A loop of links: the walk stops at the hop limit, nothing found. + // The walk runs on its own thread, so a walk that does not stop + // fails this cell by name within seconds rather than hanging it. + let f = base.join("f"); + std::fs::create_dir_all(&f).unwrap(); + symlink(f.join("y"), f.join("x")).unwrap(); + symlink(f.join("x"), f.join("y")).unwrap(); + let (sent, received) = std::sync::mpsc::channel(); + let looped = f.join("x/backups"); + std::thread::spawn(move || { + let _ = sent.send(link_chain(&looped).len()); + }); + let hops = received + .recv_timeout(std::time::Duration::from_secs(5)) + .expect("the walk of a loop of links ends at the hop limit"); + assert!(hops <= LINK_HOPS + 1, "{hops} hops"); + assert_eq!( + lookup(found(&base.join("elsewhere")), f.join("x/backups")).await, + None + ); + } } diff --git a/crates/storage-cassandra/Cargo.toml b/crates/storage-cassandra/Cargo.toml index ad9bb4b7e..c9ad3a977 100644 --- a/crates/storage-cassandra/Cargo.toml +++ b/crates/storage-cassandra/Cargo.toml @@ -41,4 +41,10 @@ test-hooks = ["extenddb-storage/test-hooks"] [dev-dependencies] extenddb-storage = { workspace = true, features = ["test-util"] } extenddb-storage-cassandra = { path = ".", features = ["test-util"] } +# The move's claim test runs the engine's `--move-store` over this backend and +# a filesystem store (the engine depends on `extenddb-storage`, not on this +# crate, so the edge is acyclic). +extenddb-engine = { workspace = true } +extenddb-backup-store = { workspace = true } +tempfile = "3" tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/storage-cassandra/src/backup_catalog.rs b/crates/storage-cassandra/src/backup_catalog.rs index 61e7ee05e..344af92fa 100644 --- a/crates/storage-cassandra/src/backup_catalog.rs +++ b/crates/storage-cassandra/src/backup_catalog.rs @@ -32,21 +32,65 @@ //! DeleteBackup's lock is the lightweight transaction on the `backups_by_arn` //! row, `IF backup_status = 'AVAILABLE' AND active_restores = null`, on the //! same partition a restore registers on, so the two order wholly one before -//! the other, as the in-catalog paths do. +//! the other, as the in-catalog paths do. The condition also carries the +//! store kind and root the delete read, as does `set_backup_deleted`'s +//! `IF backup_status = 'DELETING'`: each moves the row's status entry under +//! the root it read, so a row `rewrite_store` moved after the read fails the +//! condition, is read again, and gets its entry under the root it names. //! //! A status change moves the row's `backup_index_by_status` entry with a //! plain batch after the lightweight transaction on the row. A process that -//! dies between the two leaves an entry whose status the row no longer -//! matches; every reader of the status table re-reads the row and skips such -//! an entry, one wasted read each. `remove_backup_row` deletes the entries -//! under the status it read, so an entry under an earlier status stays until -//! a sweep of orphan entries exists, and no sweep removes them yet. One path -//! writes such an entry by design: `insert_backup_row`'s row is AVAILABLE -//! from its lightweight transaction on, so a DeleteBackup between that -//! transaction and the insert's entry batch moves a status entry the batch -//! has not written yet, and the batch then writes an AVAILABLE entry that no -//! later write removes (an insert completed by a later sync has the same -//! window). +//! dies between the two, or a batch that fails (it is logged, not sent +//! again), leaves the row's only entry under the earlier status; every +//! reader of one status partition re-reads the row and skips such an entry, +//! one wasted read each, so `list_backup_rows` does not list that row until +//! its entry is repaired. `list_backup_rows_by_root` reads all five +//! partitions of the root and lists the row under its own status. +//! `remove_backup_row` deletes the entries under the status it read, so an +//! entry under an earlier status stays. +//! Three paths leave such entries without a crash or a failed batch: +//! - `insert_backup_row`'s row is AVAILABLE from its lightweight transaction +//! on, so a DeleteBackup between that transaction and the insert's entry +//! batch moves a status entry the batch has not written yet, and the batch +//! then writes an AVAILABLE entry that no later write removes (an insert +//! completed by a later sync has the same window). +//! - A deleter (`start_backup_delete`'s DELETING, `set_backup_deleted`'s +//! DELETED) whose lightweight transaction lands before a `rewrite_store` +//! of the row and whose entry batch lands after it writes the new status's +//! entry under the old root, where no row stands behind it. +//! - A `rewrite_store` whose answer was lost and whose row changed status +//! before the retry leaves the new root's entry under the earlier status. +//! +//! Every reader skips them. Plain sync's report of other stores verifies +//! the summary's counts against the rows on this backend +//! (`summary_counts_status_entries`, through `list_backup_rows_by_root`) +//! and names the counts no row of that status stands behind as status +//! entries, and `backup sync --forget-store `, under the maintenance +//! claim, deletes the root's entries whose row is gone or names another +//! store, and repairs those whose row is of the root under another status +//! (`remove_orphan_status_entries`). The second case +//! cannot arise for the tombstone removal: `remove_backup_row`'s +//! transaction is conditioned on the store it read. +//! +//! `rewrite_store` (`backup sync --move-store`) is the one writer that +//! writes an entry before its row: per row it writes the new root's status +//! entry, then the lightweight transaction that rewrites the row's kind, +//! root and id (conditioned on the old kind and root and the status it +//! read), then deletes the old root's entry. With the row first, a stop +//! before the new entry would leave a moved row that no listing of the +//! configured store returns, so the reconciler, sync's row read and the +//! manifest probe would not see it. With the entry first, a stop before the +//! transaction leaves an entry under the new root whose row still names the +//! old root, which every reader skips, and the row still listed under the +//! old root, so the move runs again; a stop after it leaves the old root's +//! entry, which readers skip and `read_store_ids` of the old root counts, +//! until `--forget-store` of the old root deletes it. A transaction that does not +//! apply (another writer changed the row after the read) is followed by a +//! serial read of the row: one that already names the new store keeps the +//! new entry and counts as moved; one of the new root under another store +//! id with the status read (another move to that root, whose entry has the +//! new entry's key) has that entry written again with its own store id and +//! is not counted; and only otherwise is the new entry removed. //! //! A conditional write whose error is the driver's `Io` or `Timeout`, or the //! server's `WriteTimeout` or `WriteFailure`, has an unknown outcome @@ -83,8 +127,9 @@ use extenddb_core::types::CreateTableInput; use extenddb_storage::backup_catalog::{ BackupCatalog, BackupKey, BackupRow, BackupStatus, DeleteStart, ForgetRow, Heartbeat, JobClaim, JobRow, JobStatus, MaintenanceClaim, RestoreJobRow, RestoreKind, RestoreRegistered, - RestoreRegistrationRow, RestoreTargetKey, StaleClock, StatusFields, StoreIdentity, StoreKind, - TimeBounds, UNOWNED_GENERATION, WriteOutcome, billing_mode_name, new_generation, + RestoreRegistrationRow, RestoreTargetKey, StaleClock, StatusEntrySweep, StatusFields, + StoreIdentity, StoreKind, StoreRootCount, TimeBounds, UNOWNED_GENERATION, WriteOutcome, + billing_mode_name, new_generation, }; use extenddb_storage::error::StorageError; use futures::future::BoxFuture; @@ -99,8 +144,13 @@ use crate::engine::CassandraSession; const DELETE_CLAIM_ATTEMPTS: usize = 3; /// The condition of the DELETING write and of the outright removal: the row -/// is AVAILABLE and no restore is registered on it. -const DELETE_CLAIM_CONDITION: &str = "IF backup_status = 'AVAILABLE' AND active_restores = null"; +/// is AVAILABLE, no restore is registered on it, and it still names the +/// store (kind and root) the writer read, bound after the key. With the +/// store in the condition, a row `rewrite_store` moved after the read is not +/// matched, and the re-read moves its status entry under the root it now +/// names, not the one read. +const DELETE_CLAIM_CONDITION: &str = "IF backup_status = 'AVAILABLE' AND active_restores = null \ + AND store_kind = ? AND store_root = ?"; /// The `owner_generation` column value of a generation: the column is a /// signed 64-bit integer and `new_generation` never exceeds `i64::MAX`. @@ -622,8 +672,10 @@ impl CassandraEngine { /// Move the `backup_index_by_status` entry of a row from one status /// partition to another, after the authoritative row changed. A failure - /// is logged: a reader of the status index re-reads the authoritative - /// row and skips an entry whose status it no longer matches. + /// is logged and the batch is not sent again: the row's entry stays + /// under `from`, readers of one status partition re-read the row and + /// skip it, `list_backup_rows_by_root` lists the row under its own + /// status, and `remove_orphan_status_entries` of the root repairs it. async fn move_status_index_row( &self, key: &BackupKey, @@ -675,7 +727,10 @@ impl CassandraEngine { /// [`DELETE_CLAIM_CONDITION`] that writes DELETING or, with `remove`, /// deletes the row (`Deleting` then means removed); then the row's index /// entries. Every writer of `active_restores` is conditional, so the - /// condition orders the write wholly before or after a registration. + /// condition orders the write wholly before or after a registration; the + /// condition also carries the store read, so a row `rewrite_store` + /// moved between the read and the write is read again and its entry is + /// moved under the root it names. /// /// A lost answer of the removal is settled by a serial read: no row /// means removed, a row means the loop runs again (a lost answer on @@ -729,8 +784,16 @@ impl CassandraEngine { None }; crate::test_hooks::before_delete_claim(&key.backup_arn).await; - let values = - || cdrs_tokio::query_values!(key.account_id.as_str(), key.backup_arn.as_str()); + let read_kind = stored.store_kind.as_deref().unwrap_or(""); + let read_root = stored.store_root.as_deref().unwrap_or(""); + let values = || { + cdrs_tokio::query_values!( + key.account_id.as_str(), + key.backup_arn.as_str(), + read_kind, + read_root + ) + }; let fault = if remove { crate::test_hooks::forget_write_fault(&key.backup_arn) } else { @@ -771,6 +834,7 @@ impl CassandraEngine { ); } } else { + crate::test_hooks::after_delete_claim(&key.backup_arn).await; let store = StoreIdentity { store_kind: StoreKind::parse(stored.store_kind.as_deref().unwrap_or("")) .unwrap_or(StoreKind::Filesystem), @@ -790,8 +854,9 @@ impl CassandraEngine { key.backup_arn ))); } - // A registration or another writer landed between the check - // and the write on every attempt. + // A registration or another writer (a move of the row to + // another store included) landed between the check and the + // write on every attempt. let stored = self .load_stored_backup(&key.account_id, &key.backup_arn) .await? @@ -816,6 +881,24 @@ impl CassandraEngine { &self, store: &StoreIdentity, status: BackupStatus, + ) -> Result, StorageError> { + self.root_index_arns( + store.store_kind, + &store.store_root, + Some(&store.store_id), + status, + ) + .await + } + + /// The ARNs the status index lists under `store_root` and `status` whose + /// entry carries `store_kind`, and `store_id` when given. + async fn root_index_arns( + &self, + store_kind: StoreKind, + store_root: &str, + store_id: Option<&str>, + status: BackupStatus, ) -> Result, StorageError> { let query = format!( "SELECT backup_arn, account_id, store_kind, store_id FROM {}.backup_index_by_status \ @@ -825,16 +908,16 @@ impl CassandraEngine { let rows = crate::cassandra_util::query_rows_quorum::( &self.session, &query, - cdrs_tokio::query_values!(store.store_root.as_str(), status.as_str()), + cdrs_tokio::query_values!(store_root, status.as_str()), "status index", ) .await?; let mut arns = Vec::with_capacity(rows.len()); for row in rows { - let store_kind: Option = row.get_by_name("store_kind").ok().flatten(); - let store_id: Option = row.get_by_name("store_id").ok().flatten(); - if store_kind.as_deref() != Some(store.store_kind.as_str()) - || store_id.as_deref() != Some(store.store_id.as_str()) + let entry_kind: Option = row.get_by_name("store_kind").ok().flatten(); + let entry_id: Option = row.get_by_name("store_id").ok().flatten(); + if entry_kind.as_deref() != Some(store_kind.as_str()) + || store_id.is_some_and(|id| entry_id.as_deref() != Some(id)) { continue; } @@ -847,6 +930,225 @@ impl CassandraEngine { Ok(arns) } + /// Write one `backup_index_by_status` entry, or with `remove` delete it, + /// at `LOCAL_QUORUM`: `rewrite_store`'s two entry writes. + async fn write_status_entry( + &self, + key: &BackupKey, + store: &StoreIdentity, + status: BackupStatus, + remove: bool, + ) -> Result<(), StorageError> { + let catalog = self.catalog_keyspace(); + if remove { + crate::cassandra_util::query_rows_quorum::( + &self.session, + &format!( + "DELETE FROM {catalog}.backup_index_by_status \ + WHERE store_root = ? AND backup_status = ? AND backup_arn = ?" + ), + cdrs_tokio::query_values!( + store.store_root.as_str(), + status.as_str(), + key.backup_arn.as_str() + ), + "remove a status entry", + ) + .await?; + } else { + crate::cassandra_util::query_rows_quorum::( + &self.session, + &format!( + "INSERT INTO {catalog}.backup_index_by_status \ + (store_root, backup_status, backup_arn, account_id, store_kind, store_id) \ + VALUES (?, ?, ?, ?, ?, ?)" + ), + cdrs_tokio::query_values!( + store.store_root.as_str(), + status.as_str(), + key.backup_arn.as_str(), + key.account_id.as_str(), + store.store_kind.as_str(), + store.store_id.as_str() + ), + "write a status entry", + ) + .await?; + } + Ok(()) + } + + /// `rewrite_store` for one key, in the order the module doc gives: the + /// row read at `LOCAL_SERIAL` and skipped unless it is of the old kind + /// and root and not CREATING; the new root's status entry; the + /// lightweight transaction on the row, conditioned on the old kind and + /// root and the status read; then the old root's entry deleted. + /// `NotMatched` (another writer changed the row in between) reads the + /// row again at `LOCAL_SERIAL`: a row that already carries `to` (kind, + /// root and store id) counts as this call's write and keeps the new + /// entry; a row of `to`'s root under another store id with the status + /// read has its entry, which shares the new entry's key, written again + /// with its own store; otherwise the new entry is removed; then the loop + /// reads again, up to [`DELETE_CLAIM_ATTEMPTS`]; + /// a lost answer reads again, and a row that then carries `to` whole + /// (kind, root and store id; `to`'s root under another store id is + /// another store, left and not counted) is this call's write. `true` + /// when this call rewrote the row. + async fn rewrite_store_row( + &self, + from_kind: StoreKind, + from_root: &str, + key: &BackupKey, + to: &StoreIdentity, + ) -> Result { + let update = format!( + "UPDATE {}.backups_by_arn SET store_kind = ?, store_root = ?, store_id = ? \ + WHERE account_id = ? AND backup_arn = ? \ + IF store_kind = ? AND store_root = ? AND backup_status = ?", + self.catalog_keyspace() + ); + let mut lost: Option = None; + for _ in 0..DELETE_CLAIM_ATTEMPTS { + let Some(row) = self + .read_index_row_serial(&key.account_id, &key.backup_arn) + .await? + else { + return Ok(false); + }; + if let Some(status) = lost + && row.store == *to + { + // The lost answer was this call's write. + self.remove_old_status_entry(key, from_kind, from_root, status) + .await; + return Ok(true); + } + if row.store.store_kind != from_kind + || row.store.store_root != from_root + || row.status == BackupStatus::Creating + { + return Ok(false); + } + // The new root's entry before the row: a stop after it leaves an + // entry readers skip, and the row still listed under the old root. + self.write_status_entry(key, to, row.status, false).await?; + crate::test_hooks::rewrite_stop( + &key.backup_arn, + crate::test_hooks::RewriteStop::AfterNewEntry, + )?; + crate::test_hooks::before_rewrite_write(&key.backup_arn).await; + let values = || { + cdrs_tokio::query_values!( + to.store_kind.as_str(), + to.store_root.as_str(), + to.store_id.as_str(), + key.account_id.as_str(), + key.backup_arn.as_str(), + from_kind.as_str(), + from_root, + row.status.as_str() + ) + }; + let fault = crate::test_hooks::rewrite_write_fault(&key.backup_arn); + let outcome = if fault == Some(crate::test_hooks::ProgressWriteFault::Lost) { + WriteOutcome::Unknown + } else { + let outcome = conditional_write(&self.session, &update, values()).await?; + if fault.is_some() { + WriteOutcome::Unknown + } else { + outcome + } + }; + match outcome { + WriteOutcome::Matched => { + crate::test_hooks::rewrite_stop( + &key.backup_arn, + crate::test_hooks::RewriteStop::AfterRow, + )?; + self.remove_old_status_entry(key, from_kind, from_root, row.status) + .await; + return Ok(true); + } + WriteOutcome::NotMatched => { + lost = None; + // Read again before the new entry goes: a row that + // already carries `to` is this call's write (an earlier + // answer lost in transit whose proposal the retry's own + // Paxos prepare completed) or another move to the same store, + // and the new entry is its only correct one. + let now = self + .read_index_row_serial(&key.account_id, &key.backup_arn) + .await?; + if now.as_ref().is_some_and(|now| now.store == *to) { + self.remove_old_status_entry(key, from_kind, from_root, row.status) + .await; + return Ok(true); + } + // A row now of `to`'s root under another store id, with + // the status read (another move to that root), has its + // entry under the new entry's key: it is written again + // with the row's store rather than removed. Any other + // row has the new entry removed. + let entry = match &now { + Some(now) + if now.store.store_kind == to.store_kind + && now.store.store_root == to.store_root + && now.status == row.status => + { + self.write_status_entry(key, &now.store, now.status, false) + .await + } + _ => self.write_status_entry(key, to, row.status, true).await, + }; + if let Err(error) = entry { + tracing::warn!( + "backup {}: the status entry written under {} for a move that \ + did not apply is left ({error}); readers verify against the row", + key.backup_arn, + to.store_root + ); + } + } + WriteOutcome::Unknown => lost = Some(row.status), + } + } + if lost.is_some() { + return Err(StorageError::Internal(format!( + "the store rewrite of backup {} has an unknown outcome on every attempt", + key.backup_arn + ))); + } + // Another writer changed the row between the read and the write on + // every attempt: left, and the caller's re-list names it. + Ok(false) + } + + /// Delete the old root's status entry of a row `rewrite_store` moved, + /// under the status the move read. A failure is logged: readers skip + /// the entry, and `read_store_ids` of the old root counts it until + /// `--forget-store` of the old root deletes it. + async fn remove_old_status_entry( + &self, + key: &BackupKey, + from_kind: StoreKind, + from_root: &str, + status: BackupStatus, + ) { + let old = StoreIdentity { + store_kind: from_kind, + store_root: from_root.to_owned(), + store_id: String::new(), + }; + if let Err(error) = self.write_status_entry(key, &old, status, true).await { + tracing::warn!( + "backup {}: moved, and its status entry under {from_root} is left ({error}); \ + readers verify against the row", + key.backup_arn + ); + } + } + /// Re-apply the target's `active_restores[table_id] = table_name` entry /// on the backup's row when it is absent, under the takeover of a dead /// manifest record (the trait doc of `claim_restore_dead`): one serial @@ -1410,6 +1712,43 @@ impl BackupCatalog for CassandraEngine { }) } + fn list_backup_rows_by_root<'a>( + &'a self, + store_kind: StoreKind, + store_root: &'a str, + status: Option, + ) -> BoxStream<'a, Result> { + rows_stream(async move { + // Every status partition of the root, whatever `status` is: the + // entry is a hint and the authoritative row decides, so a row + // whose entry is under an earlier status (a status change whose + // entry batch failed, or whose writer died between its + // transaction and its batch) is found and listed under its own + // status. Each row is read once, however many entries it has. + let mut seen = std::collections::HashSet::new(); + let mut rows = Vec::new(); + for entry_status in BackupStatus::ALL { + for (account_id, backup_arn) in self + .root_index_arns(store_kind, store_root, None, entry_status) + .await? + { + if !seen.insert((account_id.clone(), backup_arn.clone())) { + continue; + } + if let Some(row) = self.read_index_row(&account_id, &backup_arn).await? + && status.is_none_or(|status| row.status == status) + && row.store.store_kind == store_kind + && row.store.store_root == store_root + { + rows.push(row); + } + } + } + sort_listing(&mut rows); + Ok(rows) + }) + } + fn newest_available_backup_row<'a>( &'a self, store: &'a StoreIdentity, @@ -1548,6 +1887,33 @@ impl BackupCatalog for CassandraEngine { }) } + fn rewrite_store<'a>( + &'a self, + from_kind: StoreKind, + from_root: &'a str, + keys: &'a [BackupKey], + to: &'a StoreIdentity, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let mut rewritten = 0; + for key in keys { + if self + .rewrite_store_row(from_kind, from_root, key, to) + .await? + { + rewritten += 1; + } + } + Ok(rewritten) + }) + } + + /// Each key is a serial read, the new entry, the lightweight transaction + /// and the old entry's delete, so the move calls with one key at a time. + fn rewrite_store_runs_key_by_key(&self) -> bool { + true + } + fn set_backup_deleted<'a>( &'a self, key: &'a BackupKey, @@ -1559,30 +1925,64 @@ impl BackupCatalog for CassandraEngine { else { return Ok(false); }; + // The store read is in the condition, so a row `rewrite_store` + // moved after the read is not matched: it is read once more at + // `LOCAL_SERIAL` and, still DELETING under the root it now + // names, written again with that root, and its status entry + // moves there. let query = format!( "UPDATE {}.backups_by_arn SET backup_status = 'DELETED', \ completed_at = toTimestamp(now()) \ - WHERE account_id = ? AND backup_arn = ? IF backup_status = 'DELETING'", + WHERE account_id = ? AND backup_arn = ? \ + IF backup_status = 'DELETING' AND store_kind = ? AND store_root = ?", self.catalog_keyspace() ); - let result = crate::cassandra_util::query_lwt( - &self.session, - &query, - cdrs_tokio::query_values!(key.account_id.as_str(), key.backup_arn.as_str()), - ) - .await?; - let applied = crate::cassandra_util::lwt_applied(&result)?; - if applied { - self.move_status_index_row(key, &row.store, "DELETING", "DELETED") - .await; + crate::test_hooks::before_deleted_write(&key.backup_arn).await; + let mut row = row; + for attempt in 0..2 { + let result = crate::cassandra_util::query_lwt( + &self.session, + &query, + cdrs_tokio::query_values!( + key.account_id.as_str(), + key.backup_arn.as_str(), + row.store.store_kind.as_str(), + row.store.store_root.as_str() + ), + ) + .await?; + if crate::cassandra_util::lwt_applied(&result)? { + crate::test_hooks::after_deleted_write(&key.backup_arn).await; + self.move_status_index_row(key, &row.store, "DELETING", "DELETED") + .await; + return Ok(true); + } + if attempt > 0 { + break; + } + match self + .read_index_row_serial(&key.account_id, &key.backup_arn) + .await? + { + Some(now) + if now.status == BackupStatus::Deleting + && (now.store.store_kind != row.store.store_kind + || now.store.store_root != row.store.store_root) => + { + row = now; + } + _ => break, + } } - Ok(applied) + Ok(false) }) } fn remove_backup_row<'a>( &'a self, key: &'a BackupKey, + store_kind: StoreKind, + store_root: &'a str, ) -> BoxFuture<'a, Result> { Box::pin(async move { let Some(row) = self @@ -1591,14 +1991,20 @@ impl BackupCatalog for CassandraEngine { else { return Ok(false); }; - if !matches!(row.status, BackupStatus::Deleted | BackupStatus::Failed) { + if !matches!(row.status, BackupStatus::Deleted | BackupStatus::Failed) + || row.store.store_kind != store_kind + || row.store.store_root != store_root + { return Ok(false); } + crate::test_hooks::before_tombstone_removal(&key.backup_arn).await; // The authoritative row first, conditioned on the tombstone - // status read; then its index entries. + // status and the store read, so a row `rewrite_store` moved + // after the read is left (`NotMatched` answers `false`); then + // its index entries. let delete = format!( "DELETE FROM {}.backups_by_arn WHERE account_id = ? AND backup_arn = ? \ - IF backup_status = ?", + IF backup_status = ? AND store_kind = ? AND store_root = ?", self.catalog_keyspace() ); let result = crate::cassandra_util::query_lwt( @@ -1607,7 +2013,9 @@ impl BackupCatalog for CassandraEngine { cdrs_tokio::query_values!( key.account_id.as_str(), key.backup_arn.as_str(), - row.status.as_str() + row.status.as_str(), + store_kind.as_str(), + store_root ), ) .await?; @@ -2360,7 +2768,8 @@ impl BackupCatalog for CassandraEngine { // `backup_index_by_status` is partitioned by (root, status), so a // root's rows are its five status partitions; the entry carries // the kind and id, which is all this read needs (an orphan entry - // left by a crash names the same store its row did). + // left by a crash names the store its row had or was being + // moved to). let query = format!( "SELECT store_kind, store_id FROM {}.backup_index_by_status \ WHERE store_root = ? AND backup_status = ?", @@ -2397,6 +2806,120 @@ impl BackupCatalog for CassandraEngine { }) } + fn summary_counts_status_entries(&self) -> bool { + true + } + + fn remove_orphan_status_entries<'a>( + &'a self, + store_kind: StoreKind, + store_root: &'a str, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + // The root's five status partitions; each entry's row is read at + // `LOCAL_SERIAL` (a lightweight transaction in progress is + // seen). An entry whose row is gone or names another store is + // deleted. An entry whose row is of this root under another + // status (a status change whose entry batch failed, or whose + // writer died between its transaction and its batch) is + // repaired: the entry under the row's status is written (an + // idempotent upsert), then the stale one is deleted. + let delete = format!( + "DELETE FROM {}.backup_index_by_status \ + WHERE store_root = ? AND backup_status = ? AND backup_arn = ?", + self.catalog_keyspace() + ); + let mut sweep = StatusEntrySweep::default(); + for status in BackupStatus::ALL { + for (account_id, backup_arn) in self + .root_index_arns(store_kind, store_root, None, status) + .await? + { + match self.read_index_row_serial(&account_id, &backup_arn).await? { + Some(row) + if row.store.store_kind == store_kind + && row.store.store_root == store_root => + { + if row.status == status { + continue; + } + self.write_status_entry(&row.key(), &row.store, row.status, false) + .await?; + sweep.repaired += 1; + } + _ => sweep.removed += 1, + } + crate::cassandra_util::query_rows_quorum::( + &self.session, + &delete, + cdrs_tokio::query_values!(store_root, status.as_str(), backup_arn.as_str()), + "remove an orphan status entry", + ) + .await?; + } + } + Ok(sweep) + }) + } + + fn summarize_store_roots(&self) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + // The status table's partitions are (root, status): a scan of + // its partition keys, then each partition's entries counted by + // the kind and id they carry. Hints, not verified against the + // rows (the trait doc). + let catalog = self.catalog_keyspace(); + let partitions = crate::cassandra_util::query_rows_quorum::( + &self.session, + &format!("SELECT DISTINCT store_root, backup_status FROM {catalog}.backup_index_by_status"), + cdrs_tokio::query_values!(), + "status index partitions", + ) + .await?; + let entries_query = format!( + "SELECT store_kind, store_id FROM {catalog}.backup_index_by_status \ + WHERE store_root = ? AND backup_status = ?" + ); + let mut entries = Vec::new(); + for partition in partitions { + let root: String = + crate::cassandra_util::get_column(&partition, "store_root", "status index")?; + let status: String = + crate::cassandra_util::get_column(&partition, "backup_status", "status index")?; + let status = BackupStatus::parse(&status)?; + let rows = crate::cassandra_util::query_rows_quorum::( + &self.session, + &entries_query, + cdrs_tokio::query_values!(root.as_str(), status.as_str()), + "status index", + ) + .await?; + for row in rows { + let kind: Option = row.get_by_name("store_kind").ok().flatten(); + let id: Option = row.get_by_name("store_id").ok().flatten(); + let (Some(kind), Some(id)) = (kind, id) else { + continue; + }; + let store_kind = StoreKind::parse(&kind).ok_or_else(|| { + StorageError::Internal(format!( + "a status entry of root {root:?} carries an unknown store kind \ + {kind:?}" + )) + })?; + entries.push(( + StoreIdentity { + store_kind, + store_root: root.clone(), + store_id: id, + }, + status, + )); + } + } + Ok(StoreRootCount::tally(entries)) + }) + } + fn maintenance_claim<'a>( &'a self, holder: &'a str, diff --git a/crates/storage-cassandra/src/test_hooks.rs b/crates/storage-cassandra/src/test_hooks.rs index 62f318c95..67e698b85 100644 --- a/crates/storage-cassandra/src/test_hooks.rs +++ b/crates/storage-cassandra/src/test_hooks.rs @@ -51,7 +51,18 @@ //! * a fault on the lightweight transaction of `forget_available_backup_row`, //! and an error on the logged batch that removes a row's index entries, so //! a test can see the serial re-read settle a lost removal and a removal -//! whose entries stay answer that the row is removed. +//! whose entries stay answer that the row is removed; +//! * a fault on the lightweight transaction of `rewrite_store`, and a stop +//! of one row's rewrite after its first write or after its transaction, so +//! a test can see the serial re-read settle a lost rewrite and see where a +//! stopped rewrite leaves the row listed; +//! * a pause in one row's `rewrite_store` between the new root's status +//! entry and the lightweight transaction, so a test can delete or move +//! the row in that gap and see the transaction's conditions and the +//! serial re-read after it; +//! * a pause in `set_backup_deleted` between its read of the row and its +//! lightweight transaction, so a test can move the row in that gap and see +//! the transaction's store condition send it under the root it names. use std::time::Duration; @@ -69,6 +80,16 @@ pub enum ProgressWriteFault { Lost, } +/// Where a test stops one row's `rewrite_store`, with an error, as a process +/// that dies there would: after the write that precedes the lightweight +/// transaction (the new root's status entry), or after the transaction, +/// before the old root's entry is deleted. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RewriteStop { + AfterNewEntry, + AfterRow, +} + #[cfg(feature = "test-hooks")] mod armed { use std::collections::{HashMap, VecDeque}; @@ -77,7 +98,7 @@ mod armed { use tokio::sync::Notify; - use super::ProgressWriteFault; + use super::{ProgressWriteFault, RewriteStop}; /// A pause a test arms at one point of one operation: the operation /// signals `reached` when it gets there and waits for `released`. @@ -128,6 +149,13 @@ mod armed { pub(super) index_entry_batch_errors: HashMap, pub(super) forget_write_faults: HashMap>, pub(super) index_entry_removal_errors: HashMap, + pub(super) rewrite_write_faults: HashMap>, + pub(super) rewrite_stops: HashMap, + pub(super) before_rewrite_write: HashMap>, + pub(super) before_deleted_write: HashMap>, + pub(super) after_delete_claim: HashMap>, + pub(super) after_deleted_write: HashMap>, + pub(super) before_tombstone_removal: HashMap>, } pub(super) static HOOKS: LazyLock> = LazyLock::new(Mutex::default); @@ -184,6 +212,76 @@ pub fn pause_before_delete_claim(backup_arn: &str) -> std::sync::Arc { pause } +/// Arm a pause in the next `rewrite_store` of the row of `backup_arn`, +/// between the new root's status entry and the lightweight transaction on +/// the row. The pause is taken once and then disarmed. +#[cfg(feature = "test-hooks")] +pub fn pause_before_rewrite_write(backup_arn: &str) -> std::sync::Arc { + let pause = std::sync::Arc::new(Pause::new()); + armed::with_hooks(|hooks| { + hooks + .before_rewrite_write + .insert(backup_arn.to_owned(), std::sync::Arc::clone(&pause)); + }); + pause +} + +/// Arm a pause in the next `set_backup_deleted` of `backup_arn`, between +/// its read of the row and its lightweight transaction. The pause is taken +/// once and then disarmed. +#[cfg(feature = "test-hooks")] +pub fn pause_before_deleted_write(backup_arn: &str) -> std::sync::Arc { + let pause = std::sync::Arc::new(Pause::new()); + armed::with_hooks(|hooks| { + hooks + .before_deleted_write + .insert(backup_arn.to_owned(), std::sync::Arc::clone(&pause)); + }); + pause +} + +/// Arm a pause in the next delete of `backup_arn`, after its applied +/// DELETING claim and before the batch that moves its status entry. The +/// pause is taken once and then disarmed. +#[cfg(feature = "test-hooks")] +pub fn pause_after_delete_claim(backup_arn: &str) -> std::sync::Arc { + let pause = std::sync::Arc::new(Pause::new()); + armed::with_hooks(|hooks| { + hooks + .after_delete_claim + .insert(backup_arn.to_owned(), std::sync::Arc::clone(&pause)); + }); + pause +} + +/// Arm a pause in the next `set_backup_deleted` of `backup_arn`, after its +/// applied lightweight transaction and before the batch that moves its +/// status entry. The pause is taken once and then disarmed. +#[cfg(feature = "test-hooks")] +pub fn pause_after_deleted_write(backup_arn: &str) -> std::sync::Arc { + let pause = std::sync::Arc::new(Pause::new()); + armed::with_hooks(|hooks| { + hooks + .after_deleted_write + .insert(backup_arn.to_owned(), std::sync::Arc::clone(&pause)); + }); + pause +} + +/// Arm a pause in the next `remove_backup_row` of `backup_arn`, between its +/// read of the row and its lightweight transaction. The pause is taken once +/// and then disarmed. +#[cfg(feature = "test-hooks")] +pub fn pause_before_tombstone_removal(backup_arn: &str) -> std::sync::Arc { + let pause = std::sync::Arc::new(Pause::new()); + armed::with_hooks(|hooks| { + hooks + .before_tombstone_removal + .insert(backup_arn.to_owned(), std::sync::Arc::clone(&pause)); + }); + pause +} + /// Arm a pause in the next abandoned-create scan that reaches the CREATING /// row `backup_arn`, between the read of the row and the fence that moves it /// to DELETING. The pause is taken once and then disarmed. @@ -345,6 +443,69 @@ pub fn fail_index_entry_removals(backup_arn: &str, count: usize) { }); } +/// Make the next lightweight transactions of `rewrite_store` on the row of +/// `backup_arn` answer `WriteOutcome::Unknown`, one fault per write in the +/// order given, as [`fail_insert_writes`] does for the insert. +#[cfg(feature = "test-hooks")] +pub fn fail_rewrite_writes(backup_arn: &str, faults: &[ProgressWriteFault]) { + armed::with_hooks(|hooks| { + hooks + .rewrite_write_faults + .insert(backup_arn.to_owned(), faults.iter().copied().collect()); + }); +} + +/// Stop the next `rewrite_store` of the row of `backup_arn` at `at` with an +/// error, which ends the whole call. Taken once, then disarmed. +#[cfg(feature = "test-hooks")] +pub fn stop_rewrite(backup_arn: &str, at: RewriteStop) { + armed::with_hooks(|hooks| { + hooks.rewrite_stops.insert(backup_arn.to_owned(), at); + }); +} + +/// The fault the next rewrite lightweight transaction of `backup_arn` takes, +/// if a test armed one. +pub(crate) fn rewrite_write_fault(backup_arn: &str) -> Option { + #[cfg(feature = "test-hooks")] + { + armed::with_hooks(|hooks| { + hooks + .rewrite_write_faults + .get_mut(backup_arn) + .and_then(std::collections::VecDeque::pop_front) + }) + } + #[cfg(not(feature = "test-hooks"))] + { + let _ = backup_arn; + None + } +} + +/// The error a rewrite of `backup_arn` that reached `at` stops with, if a +/// test armed [`stop_rewrite`] there. +pub(crate) fn rewrite_stop(backup_arn: &str, at: RewriteStop) -> Result<(), StorageError> { + #[cfg(feature = "test-hooks")] + { + let armed_here = armed::with_hooks(|hooks| { + if hooks.rewrite_stops.get(backup_arn) == Some(&at) { + hooks.rewrite_stops.remove(backup_arn); + true + } else { + false + } + }); + if armed_here { + return Err(StorageError::Internal(format!( + "test hook: the rewrite of {backup_arn} stopped at {at:?}" + ))); + } + } + let _ = (backup_arn, at); + Ok(()) +} + /// The fault the next removal lightweight transaction of `backup_arn` takes, /// if a test armed one. pub(crate) fn forget_write_fault(backup_arn: &str) -> Option { @@ -477,6 +638,56 @@ pub(crate) async fn before_delete_claim(backup_arn: &str) { let _ = backup_arn; } +/// Hold here if a test armed a pause for the rewrite of `backup_arn`. +pub(crate) async fn before_rewrite_write(backup_arn: &str) { + #[cfg(feature = "test-hooks")] + if let Some(pause) = armed::with_hooks(|hooks| hooks.before_rewrite_write.remove(backup_arn)) { + pause.hold().await; + } + let _ = backup_arn; +} + +/// Hold here if a test armed a pause for the DELETED write of `backup_arn`. +pub(crate) async fn before_deleted_write(backup_arn: &str) { + #[cfg(feature = "test-hooks")] + if let Some(pause) = armed::with_hooks(|hooks| hooks.before_deleted_write.remove(backup_arn)) { + pause.hold().await; + } + let _ = backup_arn; +} + +/// Hold here if a test armed a pause after the DELETING claim of +/// `backup_arn`. +pub(crate) async fn after_delete_claim(backup_arn: &str) { + #[cfg(feature = "test-hooks")] + if let Some(pause) = armed::with_hooks(|hooks| hooks.after_delete_claim.remove(backup_arn)) { + pause.hold().await; + } + let _ = backup_arn; +} + +/// Hold here if a test armed a pause after the DELETED write of +/// `backup_arn`. +pub(crate) async fn after_deleted_write(backup_arn: &str) { + #[cfg(feature = "test-hooks")] + if let Some(pause) = armed::with_hooks(|hooks| hooks.after_deleted_write.remove(backup_arn)) { + pause.hold().await; + } + let _ = backup_arn; +} + +/// Hold here if a test armed a pause for the tombstone removal of +/// `backup_arn`. +pub(crate) async fn before_tombstone_removal(backup_arn: &str) { + #[cfg(feature = "test-hooks")] + if let Some(pause) = + armed::with_hooks(|hooks| hooks.before_tombstone_removal.remove(backup_arn)) + { + pause.hold().await; + } + let _ = backup_arn; +} + /// Hold here if a test armed a pause for the abandoned-create fence of /// `backup_arn`. pub(crate) async fn before_abandoned_fence(backup_arn: &str) { diff --git a/crates/storage-cassandra/tests/direct/backup_engine.rs b/crates/storage-cassandra/tests/direct/backup_engine.rs index 8302fd0e8..d7eae43bb 100644 --- a/crates/storage-cassandra/tests/direct/backup_engine.rs +++ b/crates/storage-cassandra/tests/direct/backup_engine.rs @@ -2555,7 +2555,10 @@ async fn test_start_backup_delete_checks_the_status_and_the_in_use_rule_under_th Err(StorageError::Validation(m)) if m.contains("Backup not found") )); assert!( - !engine.remove_backup_row(&key).await.unwrap(), + !engine + .remove_backup_row(&key, row.store.store_kind, &row.store.store_root) + .await + .unwrap(), "DELETING is not a tombstone" ); assert!(engine.set_backup_deleted(&key).await.unwrap()); @@ -2573,8 +2576,82 @@ async fn test_start_backup_delete_checks_the_status_and_the_in_use_rule_under_th .await .unwrap(); assert_eq!(deleted.len(), 1, "the status index followed: {deleted:?}"); - assert!(engine.remove_backup_row(&key).await.unwrap()); - assert!(!engine.remove_backup_row(&key).await.unwrap()); + // A row another writer moved to another store after the caller read it + // is left by a removal that names the store read. + let moved_to = StoreIdentity { + store_kind: row.store.store_kind, + store_root: format!("{}-moved", row.store.store_root), + store_id: "moved-1".to_owned(), + }; + assert_eq!( + engine + .rewrite_store( + row.store.store_kind, + &row.store.store_root, + std::slice::from_ref(&key), + &moved_to + ) + .await + .unwrap(), + 1 + ); + assert!( + !engine + .remove_backup_row(&key, row.store.store_kind, &row.store.store_root) + .await + .unwrap(), + "a row moved to another store is left" + ); + assert_eq!( + engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .map(|r| (r.status, r.store)), + Some((BackupStatus::Deleted, moved_to.clone())) + ); + // The same root under the other kind is another store: a removal that + // names the kind it read leaves the row. + let other_kind = StoreIdentity { + store_kind: if moved_to.store_kind == StoreKind::S3 { + StoreKind::Filesystem + } else { + StoreKind::S3 + }, + ..moved_to.clone() + }; + assert_eq!( + engine + .rewrite_store( + moved_to.store_kind, + &moved_to.store_root, + std::slice::from_ref(&key), + &other_kind + ) + .await + .unwrap(), + 1 + ); + assert!( + !engine + .remove_backup_row(&key, moved_to.store_kind, &moved_to.store_root) + .await + .unwrap(), + "a row moved to the other kind is left" + ); + let moved_to = other_kind; + assert!( + engine + .remove_backup_row(&key, moved_to.store_kind, &moved_to.store_root) + .await + .unwrap() + ); + assert!( + !engine + .remove_backup_row(&key, moved_to.store_kind, &moved_to.store_root) + .await + .unwrap() + ); assert!( engine .read_backup_row(&arn_of(&row)) @@ -4276,7 +4353,12 @@ async fn test_a_restore_registering_between_the_in_use_read_and_the_claim_keeps_ DeleteStart::Deleting ); assert!(engine.set_backup_deleted(&key).await.unwrap()); - assert!(engine.remove_backup_row(&key).await.unwrap()); + assert!( + engine + .remove_backup_row(&key, row.store.store_kind, &row.store.store_root) + .await + .unwrap() + ); drop(claim); engine .delete_table( @@ -4891,7 +4973,12 @@ async fn test_a_stale_creating_index_row_is_skipped_by_the_abandoned_create_scan ), "the legacy reader does not see an index row" ); - assert!(engine.remove_backup_row(&row.key()).await.is_ok()); + assert!( + engine + .remove_backup_row(&row.key(), row.store.store_kind, &row.store.store_root) + .await + .is_ok() + ); } /// `fail_restore_job` is conditioned on the `owner_generation` and @@ -5072,6 +5159,1251 @@ async fn test_read_store_ids_lists_the_distinct_ids_of_a_root() { ); } +/// A row of `index_row(account, table, name, ..)` with `status`, under the +/// store of `kind`, `root` and `id`. +fn row_at( + account: &str, + name: &str, + status: BackupStatus, + (kind, root, id): (StoreKind, &str, &str), +) -> BackupRow { + let store = StoreIdentity { + store_kind: kind, + store_root: root.to_owned(), + store_id: id.to_owned(), + }; + let mut row = index_row(account, "Music", name, &store); + row.status = status; + row.size_bytes = 10; + row.item_count = 2; + row +} + +/// The ARNs of `rows` in the listings' (`created_at`, `backup_arn`) order. +fn in_listing_order(rows: &[&BackupRow]) -> Vec { + let mut rows = rows.to_vec(); + rows.sort_by(|a, b| { + a.created_at + .total_cmp(&b.created_at) + .then_with(|| a.backup_arn.cmp(&b.backup_arn)) + }); + rows.into_iter().map(|r| r.backup_arn.clone()).collect() +} + +/// The kind and id the status entry of `arn` under `root` and `status` +/// carries, read at quorum; `None` when there is no such entry. +async fn status_entry( + engine: &CassandraEngine, + root: &str, + status: BackupStatus, + arn: &str, +) -> Option<(String, String)> { + let rows = extenddb_storage_cassandra::cassandra_util::query_rows_quorum::( + &engine.session_arc(), + &format!( + "SELECT store_kind, store_id FROM {}.backup_index_by_status \ + WHERE store_root = ? AND backup_status = ? AND backup_arn = ?", + engine.catalog_keyspace() + ), + cdrs_tokio::query_values!(root, status.as_str(), arn), + "status entry", + ) + .await + .expect("read a status entry"); + rows.first().map(|row| { + use cdrs_tokio::types::IntoRustByName; + let kind: Option = row.get_by_name("store_kind").ok().flatten(); + let id: Option = row.get_by_name("store_id").ok().flatten(); + (kind.unwrap_or_default(), id.unwrap_or_default()) + }) +} + +/// The rows a by-root listing returns, by ARN. +async fn by_root(engine: &CassandraEngine, kind: StoreKind, root: &str) -> Vec { + let rows: Vec = engine + .list_backup_rows_by_root(kind, root, None) + .try_collect() + .await + .expect("listing"); + rows.into_iter().map(|r| r.backup_arn).collect() +} + +/// `list_backup_rows_by_root`: every row of the kind and root, whatever its +/// id, of every status or of one, whole and in the listings' order, read +/// from the root's status partitions and re-read; nothing of a root that +/// differs by one byte (a trailing `/`, one character fewer), of an S3 root +/// that differs by its `?endpoint=` suffix, or of the other kind at the same +/// label. +#[tokio::test] +async fn test_list_backup_rows_by_root_returns_every_id_and_status_of_the_root_and_nothing_else() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let tag = crate::helpers::unique_test_id(); + let root = format!("/srv/backups-{tag}"); + let at = |id| (StoreKind::Filesystem, root.as_str(), id); + let mut of_root = Vec::new(); + for (i, status) in [ + BackupStatus::Available, + BackupStatus::Deleting, + BackupStatus::Deleted, + BackupStatus::Failed, + ] + .into_iter() + .enumerate() + { + let id = if i % 2 == 0 { "store-1" } else { "store-2" }; + of_root.push(row_at(&account_id, &format!("r{i}"), status, at(id))); + } + // Inserted newest first: the order asserted is the listing's. + for row in of_root.iter().rev() { + assert!(engine.insert_backup_row(row).await.expect("insert")); + } + let creating = row_at(&account_id, "job", BackupStatus::Creating, at("store-1")); + let claim = engine.claim_backup(&creating).await.expect("claim"); + let slash = format!("{root}/"); + let shorter = root[..root.len() - 1].to_owned(); + let s3 = format!("s3://bucket-{tag}/prefix"); + let s3_endpoint = format!("{s3}?endpoint=http://127.0.0.1:9200"); + let others = [ + row_at( + &account_id, + "slash", + BackupStatus::Available, + (StoreKind::Filesystem, &slash, "store-1"), + ), + row_at( + &account_id, + "shorter", + BackupStatus::Available, + (StoreKind::Filesystem, &shorter, "store-1"), + ), + row_at( + &account_id, + "kind", + BackupStatus::Available, + (StoreKind::S3, &root, "store-1"), + ), + row_at( + &account_id, + "s3", + BackupStatus::Available, + (StoreKind::S3, &s3, "store-3"), + ), + row_at( + &account_id, + "endpoint", + BackupStatus::Available, + (StoreKind::S3, &s3_endpoint, "store-3"), + ), + ]; + for row in &others { + assert!(engine.insert_backup_row(row).await.expect("insert")); + } + let listed = |kind: StoreKind, root: String, status: Option| { + let engine = &engine; + async move { + let rows: Vec = engine + .list_backup_rows_by_root(kind, &root, status) + .try_collect() + .await + .expect("listing"); + rows + } + }; + let arns = + |rows: Vec| -> Vec { rows.into_iter().map(|r| r.backup_arn).collect() }; + let mut all: Vec<&BackupRow> = of_root.iter().collect(); + all.push(&creating); + let whole = listed(StoreKind::Filesystem, root.clone(), None).await; + assert_eq!(arns(whole.clone()), in_listing_order(&all)); + for row in &whole { + assert_eq!( + Some(row), + engine + .read_backup_row(&arn_of(row)) + .await + .expect("read") + .as_ref(), + "the listing returns rows whole" + ); + } + for status in BackupStatus::ALL { + let of_status: Vec<&BackupRow> = + all.iter().copied().filter(|r| r.status == status).collect(); + assert_eq!( + arns(listed(StoreKind::Filesystem, root.clone(), Some(status)).await), + in_listing_order(&of_status), + "{status}" + ); + } + for (kind, label, row) in [ + (StoreKind::Filesystem, slash.as_str(), &others[0]), + (StoreKind::Filesystem, shorter.as_str(), &others[1]), + (StoreKind::S3, root.as_str(), &others[2]), + (StoreKind::S3, s3.as_str(), &others[3]), + (StoreKind::S3, s3_endpoint.as_str(), &others[4]), + ] { + assert_eq!( + arns(listed(kind, label.to_owned(), None).await), + std::slice::from_ref(&row.backup_arn), + "{kind} {label}" + ); + } + drop(claim); + let mut every: Vec<&BackupRow> = all.clone(); + every.extend(others.iter()); + remove_index_rows(&engine, &every).await; +} + +/// `rewrite_store`: of the keys given, the rows still of the old kind and +/// root and not CREATING get the new store, with every other column left, +/// and are counted; their status entries move from the old root's +/// partition to the new root's, under the same status, carrying the new +/// kind and id. A CREATING row, a row of another root, a row of the other +/// kind at the same label and a key with no row are left and not counted; a +/// second call counts 0; a row another writer moved to a third root before +/// the call is left there. +#[tokio::test] +async fn test_rewrite_store_moves_only_the_old_roots_rows_that_are_not_creating() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let tag = crate::helpers::unique_test_id(); + let old_root = format!("/srv/backups-old-{tag}"); + let old = (StoreKind::Filesystem, old_root.as_str(), "old-1"); + let to = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: format!("/srv/backups-new-{tag}"), + store_id: "new-1".to_owned(), + }; + let mut movable = Vec::new(); + for (i, status) in [ + BackupStatus::Available, + BackupStatus::Deleting, + BackupStatus::Deleted, + BackupStatus::Failed, + ] + .into_iter() + .enumerate() + { + movable.push(row_at(&account_id, &format!("m{i}"), status, old)); + } + let moved_away = row_at(&account_id, "away", BackupStatus::Available, old); + let other_root_label = format!("/srv/backups-other-{tag}"); + let other_root = row_at( + &account_id, + "other-root", + BackupStatus::Available, + (StoreKind::Filesystem, &other_root_label, "old-1"), + ); + let other_kind = row_at( + &account_id, + "other-kind", + BackupStatus::Available, + (StoreKind::S3, &old_root, "old-1"), + ); + for row in movable + .iter() + .chain([&moved_away, &other_root, &other_kind]) + { + assert!(engine.insert_backup_row(row).await.expect("insert")); + } + let creating = row_at(&account_id, "job", BackupStatus::Creating, old); + let claim = engine.claim_backup(&creating).await.expect("claim"); + let stored = |row: &BackupRow| { + let engine = &engine; + let arn = arn_of(row); + async move { + engine + .read_backup_row(&arn) + .await + .expect("read") + .expect("the row exists") + } + }; + let mut before = Vec::new(); + for row in &movable { + before.push(stored(row).await); + } + // Another writer moves one row to a third root first. + let third = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: format!("/srv/backups-third-{tag}"), + store_id: "third-1".to_owned(), + }; + assert_eq!( + engine + .rewrite_store(old.0, old.1, &[moved_away.key()], &third) + .await + .expect("rewrite"), + 1 + ); + let mut ghost = row_at(&account_id, "ghost", BackupStatus::Available, old).key(); + ghost.backup_arn = format!("{}-ghost", ghost.backup_arn); + let keys: Vec = movable + .iter() + .chain([&creating, &moved_away, &other_root, &other_kind]) + .map(BackupRow::key) + .chain([ghost]) + .collect(); + assert_eq!( + engine + .rewrite_store(old.0, old.1, &keys, &to) + .await + .expect("rewrite"), + 4, + "the four rows of the old root that are not CREATING" + ); + for (row, before) in movable.iter().zip(&before) { + assert_eq!( + stored(row).await, + BackupRow { + store: to.clone(), + ..before.clone() + }, + "only the store changed" + ); + assert_eq!( + status_entry(&engine, &to.store_root, row.status, &row.backup_arn).await, + Some(("filesystem".to_owned(), "new-1".to_owned())), + "{}: the new root's entry", + row.status + ); + assert_eq!( + status_entry(&engine, old.1, row.status, &row.backup_arn).await, + None, + "{}: the old root's entry is gone", + row.status + ); + } + let creating_now = stored(&creating).await; + assert_eq!( + (creating_now.status, creating_now.store.store_root.as_str()), + (BackupStatus::Creating, old.1) + ); + assert_eq!(creating_now.owner_generation, claim.generation()); + assert_eq!(stored(&moved_away).await.store, third); + assert_eq!(stored(&other_root).await.store, other_root.store); + assert_eq!(stored(&other_kind).await.store, other_kind.store); + assert_eq!( + engine + .rewrite_store(old.0, old.1, &keys, &to) + .await + .expect("rewrite"), + 0, + "a second call finds nothing left to move" + ); + let listed: Vec = engine + .list_backup_rows(None, &to) + .try_collect() + .await + .expect("listing"); + let movable_refs: Vec<&BackupRow> = movable.iter().collect(); + assert_eq!( + listed.into_iter().map(|r| r.backup_arn).collect::>(), + in_listing_order(&movable_refs) + ); + assert_eq!( + by_root(&engine, old.0, old.1).await, + std::slice::from_ref(&creating.backup_arn) + ); + assert_eq!( + engine.read_store_ids(old.0, old.1).await.expect("ids"), + ["old-1"], + "the CREATING row keeps the old root, and no other entry is left there" + ); + drop(claim); + let mut cleanup: Vec = Vec::new(); + for row in movable.iter().chain([&moved_away]) { + for store in [&to, &third] { + cleanup.push(BackupRow { + store: store.clone(), + ..row.clone() + }); + } + } + let mut every: Vec<&BackupRow> = cleanup.iter().collect(); + every.extend( + movable + .iter() + .chain([&creating, &moved_away, &other_root, &other_kind]), + ); + remove_index_rows(&engine, &every).await; +} + +/// `summarize_store_roots`: one count per kind, root, id and status entry, +/// in the one order; two ids of one root, two statuses of one id, two roots +/// and the two kinds are each counted apart. The catalog keyspace is shared, +/// so only this test's roots are compared. +#[tokio::test] +async fn test_summarize_store_roots_counts_the_rows_of_each_store_and_status() { + use extenddb_storage::backup_catalog::StoreRootCount; + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let tag = crate::helpers::unique_test_id(); + let (root_a, root_b, root_s3) = ( + format!("/srv/a-{tag}"), + format!("/srv/b-{tag}"), + format!("s3://bucket-{tag}/a"), + ); + let a1 = (StoreKind::Filesystem, root_a.as_str(), "id-1"); + let a2 = (StoreKind::Filesystem, root_a.as_str(), "id-2"); + let b1 = (StoreKind::Filesystem, root_b.as_str(), "id-1"); + let s3 = (StoreKind::S3, root_s3.as_str(), "id-1"); + let rows = [ + row_at(&account_id, "1", BackupStatus::Available, a1), + row_at(&account_id, "2", BackupStatus::Available, a1), + row_at(&account_id, "3", BackupStatus::Failed, a1), + row_at(&account_id, "4", BackupStatus::Available, a2), + row_at(&account_id, "5", BackupStatus::Deleting, b1), + row_at(&account_id, "7", BackupStatus::Available, b1), + row_at(&account_id, "6", BackupStatus::Available, s3), + ]; + for row in &rows { + assert!(engine.insert_backup_row(row).await.expect("insert")); + } + let creating = row_at(&account_id, "job", BackupStatus::Creating, b1); + let claim = engine.claim_backup(&creating).await.expect("claim"); + let line = |(kind, root, id): (StoreKind, &str, &str), status, rows| StoreRootCount { + store: StoreIdentity { + store_kind: kind, + store_root: root.to_owned(), + store_id: id.to_owned(), + }, + status, + rows, + }; + let ours: Vec = engine + .summarize_store_roots() + .await + .expect("summary") + .into_iter() + .filter(|c| [&root_a, &root_b, &root_s3].contains(&&c.store.store_root)) + .collect(); + assert_eq!( + ours, + [ + line(a1, BackupStatus::Available, 2), + line(a1, BackupStatus::Failed, 1), + line(a2, BackupStatus::Available, 1), + line(b1, BackupStatus::Creating, 1), + line(b1, BackupStatus::Available, 1), + line(b1, BackupStatus::Deleting, 1), + line(s3, BackupStatus::Available, 1), + ] + ); + drop(claim); + let mut every: Vec<&BackupRow> = rows.iter().collect(); + every.push(&creating); + remove_index_rows(&engine, &every).await; +} + +/// A rewrite whose answer is lost is settled by a serial read: one whose +/// transaction ran finds the row under the new root, counts it and removes +/// the old root's entry; one whose transaction never ran finds the row under +/// the old root, sends it once more and counts it; a lost answer on every +/// attempt is an error that leaves the row under the old root and listed +/// there. +#[cfg(feature = "test-hooks")] +#[tokio::test] +async fn test_an_unknown_rewrite_outcome_is_settled_by_a_serial_read() { + use extenddb_storage_cassandra::test_hooks::{ProgressWriteFault, fail_rewrite_writes}; + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let tag = crate::helpers::unique_test_id(); + let old_root = format!("/srv/backups-old-{tag}"); + let old = (StoreKind::Filesystem, old_root.as_str(), "old-1"); + let to = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: format!("/srv/backups-new-{tag}"), + store_id: "new-1".to_owned(), + }; + let mut cleanup = Vec::new(); + for (name, fault) in [ + ("landed", ProgressWriteFault::Landed), + ("lost", ProgressWriteFault::Lost), + ] { + let row = row_at(&account_id, name, BackupStatus::Available, old); + assert!(engine.insert_backup_row(&row).await.expect("insert")); + fail_rewrite_writes(&row.backup_arn, &[fault]); + assert_eq!( + engine + .rewrite_store(old.0, old.1, &[row.key()], &to) + .await + .expect("rewrite"), + 1, + "{name}" + ); + let stored = engine.read_backup_row(&arn_of(&row)).await.unwrap(); + assert_eq!(stored.map(|r| r.store), Some(to.clone()), "{name}"); + assert_eq!( + status_entry(&engine, old.1, row.status, &row.backup_arn).await, + None, + "{name}: the old root's entry is gone" + ); + assert!( + status_entry(&engine, &to.store_root, row.status, &row.backup_arn) + .await + .is_some(), + "{name}: the new root's entry" + ); + cleanup.push(row); + } + + let unknown = row_at(&account_id, "unknown", BackupStatus::Available, old); + assert!(engine.insert_backup_row(&unknown).await.expect("insert")); + fail_rewrite_writes(&unknown.backup_arn, &[ProgressWriteFault::Lost; 3]); + let failed = engine + .rewrite_store(old.0, old.1, &[unknown.key()], &to) + .await + .expect_err("a lost answer on every attempt"); + assert!( + matches!(&failed, StorageError::Internal(m) if m.contains("unknown outcome")), + "{failed}" + ); + assert_eq!( + engine + .read_backup_row(&arn_of(&unknown)) + .await + .unwrap() + .map(|r| r.store.store_root), + Some(old_root.clone()) + ); + assert_eq!( + by_root(&engine, old.0, old.1).await, + std::slice::from_ref(&unknown.backup_arn) + ); + cleanup.push(unknown); + let moved: Vec = cleanup + .iter() + .map(|row| BackupRow { + store: to.clone(), + ..row.clone() + }) + .collect(); + let every: Vec<&BackupRow> = cleanup.iter().chain(moved.iter()).collect(); + remove_index_rows(&engine, &every).await; +} + +/// A lost answer whose serial read finds the row under `to`'s root but +/// another store id (another move of the row to that root, landed between +/// this call's read and its lightweight transaction, which never ran) is +/// not this call's write: the row is of another store, so the call leaves +/// it, counts nothing, and the row keeps the store the other move gave it. +/// The old root's entry is gone (the other move removed it) and the row is +/// listed under the new root. +#[cfg(feature = "test-hooks")] +#[tokio::test] +async fn test_a_lost_rewrite_answer_does_not_count_the_row_under_anothers_store_id() { + use extenddb_storage_cassandra::test_hooks::{ + ProgressWriteFault, fail_rewrite_writes, pause_before_rewrite_write, + }; + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = std::sync::Arc::new(setup_engine().await); + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let tag = crate::helpers::unique_test_id(); + let old_root = format!("/srv/backups-old-{tag}"); + let old = (StoreKind::Filesystem, old_root.as_str(), "old-1"); + let to = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: format!("/srv/backups-new-{tag}"), + store_id: "new-1".to_owned(), + }; + let other_id = StoreIdentity { + store_id: "new-2".to_owned(), + ..to.clone() + }; + let row = row_at(&account_id, "lost", BackupStatus::Available, old); + assert!(engine.insert_backup_row(&row).await.expect("insert")); + let pause = pause_before_rewrite_write(&row.backup_arn); + let lost = { + let engine = std::sync::Arc::clone(&engine); + let key = row.key(); + let (to, root) = (to.clone(), old_root.clone()); + tokio::spawn(async move { + engine + .rewrite_store(StoreKind::Filesystem, &root, &[key], &to) + .await + }) + }; + pause.reached().await; + assert_eq!( + engine + .rewrite_store(old.0, old.1, &[row.key()], &other_id) + .await + .expect("the other move"), + 1 + ); + // Armed once the other move has written, so the paused call's + // transaction is the one whose answer is lost (it never runs). + fail_rewrite_writes(&row.backup_arn, &[ProgressWriteFault::Lost]); + pause.release(); + assert_eq!( + lost.await.expect("task").expect("rewrite"), + 0, + "the row under the new root with another store id is not this call's write" + ); + let stored = engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .expect("the row exists"); + assert_eq!(stored.store, other_id, "the other move's store is kept"); + assert_eq!( + status_entry(&engine, old.1, row.status, &row.backup_arn).await, + None, + "the old root's entry is gone" + ); + assert!( + listed_with(&engine, BackupStatus::Available, &other_id, &row.backup_arn).await, + "listed under the new root with the other store id" + ); + assert!(by_root(&engine, old.0, old.1).await.is_empty()); + let every = [ + row.clone(), + BackupRow { + store: other_id, + ..row.clone() + }, + ]; + remove_index_rows(&engine, &every.iter().collect::>()).await; +} + +/// A transaction that does not apply because another move put the row under +/// `to`'s root with another store id (landed between this call's read and +/// its transaction) is not this call's write: the call counts nothing, the +/// row keeps the other move's store, and the entry under the new root, +/// whose key the two moves share, carries the other move's store id, so +/// that store's listing returns the row and `to`'s does not. +#[cfg(feature = "test-hooks")] +#[tokio::test] +async fn test_a_rewrite_that_does_not_apply_does_not_count_the_row_under_anothers_store_id() { + use extenddb_storage_cassandra::test_hooks::pause_before_rewrite_write; + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = std::sync::Arc::new(setup_engine().await); + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let tag = crate::helpers::unique_test_id(); + let old_root = format!("/srv/backups-old-{tag}"); + let old = (StoreKind::Filesystem, old_root.as_str(), "old-1"); + let to = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: format!("/srv/backups-new-{tag}"), + store_id: "new-1".to_owned(), + }; + let other_id = StoreIdentity { + store_id: "new-2".to_owned(), + ..to.clone() + }; + let row = row_at(&account_id, "unapplied", BackupStatus::Available, old); + assert!(engine.insert_backup_row(&row).await.expect("insert")); + let pause = pause_before_rewrite_write(&row.backup_arn); + let first = { + let engine = std::sync::Arc::clone(&engine); + let key = row.key(); + let (to, root) = (to.clone(), old_root.clone()); + tokio::spawn(async move { + engine + .rewrite_store(StoreKind::Filesystem, &root, &[key], &to) + .await + }) + }; + pause.reached().await; + assert_eq!( + engine + .rewrite_store(old.0, old.1, &[row.key()], &other_id) + .await + .expect("the other move"), + 1 + ); + pause.release(); + assert_eq!( + first.await.expect("task").expect("rewrite"), + 0, + "the row under the new root with another store id is not this call's write" + ); + let stored = engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .expect("the row exists"); + assert_eq!(stored.store, other_id, "the other move's store is kept"); + assert_eq!( + status_entry(&engine, &to.store_root, row.status, &row.backup_arn).await, + Some(("filesystem".to_owned(), "new-2".to_owned())), + "the new root's entry carries the other move's store id" + ); + assert_eq!( + status_entry(&engine, old.1, row.status, &row.backup_arn).await, + None, + "the old root's entry is gone" + ); + assert!( + listed_with(&engine, BackupStatus::Available, &other_id, &row.backup_arn).await, + "listed under the new root with the other store id" + ); + assert!( + !listed_with(&engine, BackupStatus::Available, &to, &row.backup_arn).await, + "not listed under this call's store" + ); + let every = [ + row.clone(), + BackupRow { + store: other_id, + ..row.clone() + }, + ]; + remove_index_rows(&engine, &every.iter().collect::>()).await; +} + +/// A rewrite that stops between its writes leaves the row listed under one +/// root: stopped after its first write (the new root's entry), the row +/// still names the old root and is listed there and not under the new one, +/// and the next call moves it; stopped after its transaction, the row names +/// the new root and is listed there, and the old root's entry is left, which +/// the old root's listing skips and `read_store_ids` of the old root still +/// counts. +#[cfg(feature = "test-hooks")] +#[tokio::test] +async fn test_a_stopped_rewrite_leaves_the_row_listed_under_one_root() { + use extenddb_storage_cassandra::test_hooks::{RewriteStop, stop_rewrite}; + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let tag = crate::helpers::unique_test_id(); + let old_root = format!("/srv/backups-old-{tag}"); + let old = (StoreKind::Filesystem, old_root.as_str(), "old-1"); + let to = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: format!("/srv/backups-new-{tag}"), + store_id: "new-1".to_owned(), + }; + let listed_under_new = |row: &BackupRow| { + let engine = &engine; + let to = &to; + let arn = row.backup_arn.clone(); + async move { + let rows: Vec = engine + .list_backup_rows(None, to) + .try_collect() + .await + .expect("listing"); + rows.iter().any(|r| r.backup_arn == arn) + } + }; + + let first = row_at(&account_id, "first", BackupStatus::Failed, old); + assert!(engine.insert_backup_row(&first).await.expect("insert")); + stop_rewrite(&first.backup_arn, RewriteStop::AfterNewEntry); + engine + .rewrite_store(old.0, old.1, &[first.key()], &to) + .await + .expect_err("stopped after the first write"); + assert_eq!( + engine + .read_backup_row(&arn_of(&first)) + .await + .unwrap() + .map(|r| r.store.store_root), + Some(old_root.clone()) + ); + assert_eq!( + by_root(&engine, old.0, old.1).await, + std::slice::from_ref(&first.backup_arn), + "still listed under the old root" + ); + assert!( + !listed_under_new(&first).await, + "the new root's entry is skipped" + ); + assert_eq!( + engine + .rewrite_store(old.0, old.1, &[first.key()], &to) + .await + .expect("rewrite"), + 1, + "the next call moves it" + ); + assert!(listed_under_new(&first).await); + assert!(by_root(&engine, old.0, old.1).await.is_empty()); + + let second = row_at(&account_id, "second", BackupStatus::Available, old); + assert!(engine.insert_backup_row(&second).await.expect("insert")); + stop_rewrite(&second.backup_arn, RewriteStop::AfterRow); + engine + .rewrite_store(old.0, old.1, &[second.key()], &to) + .await + .expect_err("stopped after the transaction"); + assert_eq!( + engine + .read_backup_row(&arn_of(&second)) + .await + .unwrap() + .map(|r| r.store), + Some(to.clone()) + ); + assert!(listed_under_new(&second).await, "listed under the new root"); + assert!( + by_root(&engine, old.0, old.1).await.is_empty(), + "the old root's listing skips the entry left" + ); + assert!( + status_entry(&engine, old.1, second.status, &second.backup_arn) + .await + .is_some(), + "the old root's entry is left" + ); + assert_eq!( + engine.read_store_ids(old.0, old.1).await.expect("ids"), + ["old-1"], + "an entry left counts for the old root's ids" + ); + let moved: Vec = [&first, &second] + .into_iter() + .map(|row| BackupRow { + store: to.clone(), + ..row.clone() + }) + .collect(); + let every: Vec<&BackupRow> = [&first, &second].into_iter().chain(moved.iter()).collect(); + remove_index_rows(&engine, &every).await; +} + +/// The stores of a test's rows moved from `old` to `to`, for the cleanup: +/// each row under both roots, so every status entry of either is removed. +#[cfg(feature = "test-hooks")] +fn under_both(rows: &[&BackupRow], to: &StoreIdentity) -> Vec { + rows.iter() + .flat_map(|row| { + [ + (*row).clone(), + BackupRow { + store: to.clone(), + ..(*row).clone() + }, + ] + }) + .collect() +} + +/// Whether `list_backup_rows(Some(status), store)` returns `arn`. +#[cfg(feature = "test-hooks")] +async fn listed_with( + engine: &CassandraEngine, + status: BackupStatus, + store: &StoreIdentity, + arn: &str, +) -> bool { + let rows: Vec = engine + .list_backup_rows(Some(status), store) + .try_collect() + .await + .expect("listing"); + rows.iter().any(|r| r.backup_arn == arn) +} + +/// A DeleteBackup or the deletion's DELETED write that read the row before +/// `rewrite_store` moved it, and writes after: each one's lightweight +/// transaction carries the store it read, so it does not apply to the moved +/// row; the row is read again and written under the root it now names, and +/// its status entry moves there. The row ends DELETING (then DELETED) under +/// the new root and is listed by the new store's listing of that status, +/// with no entry left under either root for the status it had. +#[cfg(feature = "test-hooks")] +#[tokio::test] +async fn test_a_delete_that_read_the_row_before_a_rewrite_moves_its_entry_under_the_new_root() { + use extenddb_storage_cassandra::test_hooks::{ + pause_before_delete_claim, pause_before_deleted_write, + }; + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = std::sync::Arc::new(setup_engine().await); + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let tag = crate::helpers::unique_test_id(); + let old_root = format!("/srv/backups-old-{tag}"); + let old = (StoreKind::Filesystem, old_root.as_str(), "old-1"); + let to = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: format!("/srv/backups-new-{tag}"), + store_id: "new-1".to_owned(), + }; + let new_entry = Some(("filesystem".to_owned(), "new-1".to_owned())); + + // DeleteBackup: read AVAILABLE under the old root, then the move. + let available = row_at(&account_id, "deleted", BackupStatus::Available, old); + assert!(engine.insert_backup_row(&available).await.expect("insert")); + let pause = pause_before_delete_claim(&available.backup_arn); + let start = { + let engine = std::sync::Arc::clone(&engine); + let key = available.key(); + tokio::spawn(async move { engine.start_backup_delete(&key).await }) + }; + pause.reached().await; + assert_eq!( + engine + .rewrite_store(old.0, old.1, &[available.key()], &to) + .await + .expect("rewrite"), + 1 + ); + pause.release(); + assert_eq!( + start.await.expect("delete task").expect("delete"), + DeleteStart::Deleting + ); + let stored = engine + .read_backup_row(&arn_of(&available)) + .await + .unwrap() + .expect("the row exists"); + assert_eq!( + (stored.status, &stored.store), + (BackupStatus::Deleting, &to) + ); + assert!( + listed_with(&engine, BackupStatus::Deleting, &to, &available.backup_arn).await, + "the new store's DELETING listing returns the row" + ); + let arn = available.backup_arn.as_str(); + assert_eq!( + status_entry(&engine, &to.store_root, BackupStatus::Deleting, arn).await, + new_entry + ); + for (root, status) in [ + (to.store_root.as_str(), BackupStatus::Available), + (old.1, BackupStatus::Available), + (old.1, BackupStatus::Deleting), + ] { + assert_eq!( + status_entry(&engine, root, status, arn).await, + None, + "{root} {status}" + ); + } + + // The DELETED write: read DELETING under the old root, then the move. + let deleting = row_at(&account_id, "finished", BackupStatus::Deleting, old); + assert!(engine.insert_backup_row(&deleting).await.expect("insert")); + let pause = pause_before_deleted_write(&deleting.backup_arn); + let finish = { + let engine = std::sync::Arc::clone(&engine); + let key = deleting.key(); + tokio::spawn(async move { engine.set_backup_deleted(&key).await }) + }; + pause.reached().await; + assert_eq!( + engine + .rewrite_store(old.0, old.1, &[deleting.key()], &to) + .await + .expect("rewrite"), + 1 + ); + pause.release(); + assert!(finish.await.expect("task").expect("set DELETED")); + let stored = engine + .read_backup_row(&arn_of(&deleting)) + .await + .unwrap() + .expect("the row exists"); + assert_eq!((stored.status, &stored.store), (BackupStatus::Deleted, &to)); + assert!( + listed_with(&engine, BackupStatus::Deleted, &to, &deleting.backup_arn).await, + "the new store's DELETED listing returns the row" + ); + let arn = deleting.backup_arn.as_str(); + assert_eq!( + status_entry(&engine, &to.store_root, BackupStatus::Deleted, arn).await, + new_entry + ); + for (root, status) in [ + (to.store_root.as_str(), BackupStatus::Deleting), + (old.1, BackupStatus::Deleting), + (old.1, BackupStatus::Deleted), + ] { + assert_eq!( + status_entry(&engine, root, status, arn).await, + None, + "{root} {status}" + ); + } + let every = under_both(&[&available, &deleting], &to); + remove_index_rows(&engine, &every.iter().collect::>()).await; +} + +/// A rewrite whose row another writer changes between the new root's entry +/// and the lightweight transaction. A DeleteBackup in the gap: the +/// transaction, conditioned on the status read, does not apply; the serial +/// read finds the row still under the old root, the new entry is removed, +/// and the next attempt moves the DELETING row, which is then listed under +/// the new root with its one entry there. Another move of the row to the +/// same store in the gap: the serial read after the transaction that did not +/// apply finds the row already under the new store, so the call counts it +/// and keeps the new entry, the row's only correct one. +#[cfg(feature = "test-hooks")] +#[tokio::test] +async fn test_a_rewrite_whose_row_changes_before_its_transaction_reads_the_row_again() { + use extenddb_storage_cassandra::test_hooks::pause_before_rewrite_write; + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = std::sync::Arc::new(setup_engine().await); + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let tag = crate::helpers::unique_test_id(); + let old_root = format!("/srv/backups-old-{tag}"); + let old = (StoreKind::Filesystem, old_root.as_str(), "old-1"); + let to = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: format!("/srv/backups-new-{tag}"), + store_id: "new-1".to_owned(), + }; + let rewrite = |row: &BackupRow| { + let engine = std::sync::Arc::clone(&engine); + let key = row.key(); + let (to, root) = (to.clone(), old_root.clone()); + tokio::spawn(async move { + engine + .rewrite_store(StoreKind::Filesystem, &root, &[key], &to) + .await + }) + }; + + let deleted = row_at(&account_id, "deleted", BackupStatus::Available, old); + assert!(engine.insert_backup_row(&deleted).await.expect("insert")); + let pause = pause_before_rewrite_write(&deleted.backup_arn); + let moving = rewrite(&deleted); + pause.reached().await; + assert_eq!( + engine + .start_backup_delete(&deleted.key()) + .await + .expect("delete"), + DeleteStart::Deleting + ); + pause.release(); + assert_eq!(moving.await.expect("task").expect("rewrite"), 1); + let stored = engine + .read_backup_row(&arn_of(&deleted)) + .await + .unwrap() + .expect("the row exists"); + assert_eq!( + (stored.status, &stored.store), + (BackupStatus::Deleting, &to) + ); + assert!(listed_with(&engine, BackupStatus::Deleting, &to, &deleted.backup_arn).await); + let arn = deleted.backup_arn.as_str(); + assert!( + status_entry(&engine, &to.store_root, BackupStatus::Deleting, arn) + .await + .is_some() + ); + for (root, status) in [ + (to.store_root.as_str(), BackupStatus::Available), + (old.1, BackupStatus::Available), + (old.1, BackupStatus::Deleting), + ] { + assert_eq!( + status_entry(&engine, root, status, arn).await, + None, + "{root} {status}: only the new root's DELETING entry is left" + ); + } + + let raced = row_at(&account_id, "raced", BackupStatus::Available, old); + assert!(engine.insert_backup_row(&raced).await.expect("insert")); + let pause = pause_before_rewrite_write(&raced.backup_arn); + let first = rewrite(&raced); + pause.reached().await; + assert_eq!( + engine + .rewrite_store(old.0, old.1, &[raced.key()], &to) + .await + .expect("the other move"), + 1 + ); + pause.release(); + assert_eq!( + first.await.expect("task").expect("rewrite"), + 1, + "the row already under the new store counts as this call's write" + ); + assert!( + listed_with(&engine, BackupStatus::Available, &to, &raced.backup_arn).await, + "the new root's entry is kept" + ); + assert_eq!( + status_entry(&engine, old.1, BackupStatus::Available, &raced.backup_arn).await, + None + ); + let every = under_both(&[&deleted, &raced], &to); + remove_index_rows(&engine, &every.iter().collect::>()).await; +} + +/// `list_backup_rows_by_root` lists a row once, under its own status and +/// kind: a second entry of the row under another status of the same root +/// is passed over (each row is read once), and an entry whose kind is not +/// its row's is skipped on the re-read. +#[tokio::test] +async fn test_list_backup_rows_by_root_skips_an_entry_whose_row_has_another_status_or_kind() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let tag = crate::helpers::unique_test_id(); + let root = format!("/srv/backups-{tag}"); + let row = row_at( + &account_id, + "row", + BackupStatus::Available, + (StoreKind::Filesystem, &root, "id-1"), + ); + let s3_row = row_at( + &account_id, + "s3", + BackupStatus::Available, + (StoreKind::S3, &root, "id-1"), + ); + for r in [&row, &s3_row] { + assert!(engine.insert_backup_row(r).await.expect("insert")); + } + let catalog = engine.catalog_keyspace(); + let entry = |status: BackupStatus, arn: &str| { + ( + format!( + "INSERT INTO {catalog}.backup_index_by_status \ + (store_root, backup_status, backup_arn, account_id, store_kind, store_id) \ + VALUES (?, ?, ?, ?, 'filesystem', 'id-1')" + ), + cdrs_tokio::query_values!( + root.as_str(), + status.as_str(), + arn.to_owned(), + account_id.as_str() + ), + ) + }; + // A stale entry of the filesystem row under FAILED, and the S3 row's + // AVAILABLE entry overwritten with the filesystem kind. + for (cql, values) in [ + entry(BackupStatus::Failed, &row.backup_arn), + entry(BackupStatus::Available, &s3_row.backup_arn), + ] { + engine + .session() + .query_with_values(&cql, values) + .await + .expect("write an entry"); + } + assert_eq!( + by_root(&engine, StoreKind::Filesystem, &root).await, + std::slice::from_ref(&row.backup_arn), + "listed once, and the S3 row not at all" + ); + for status in [BackupStatus::Failed, BackupStatus::Available] { + let rows: Vec = engine + .list_backup_rows_by_root(StoreKind::Filesystem, &root, Some(status)) + .try_collect() + .await + .expect("listing"); + let expected: &[String] = if status == BackupStatus::Available { + std::slice::from_ref(&row.backup_arn) + } else { + &[] + }; + assert_eq!( + rows.into_iter().map(|r| r.backup_arn).collect::>(), + expected, + "{status}" + ); + } + remove_index_rows(&engine, &[&row, &s3_row]).await; +} + +/// After a rewrite stopped between its transaction and the old entry's +/// removal, `summarize_store_roots` counts the row under both roots: the +/// new root's entry and the old root's entry left behind, which the summary +/// does not verify against the row. +#[cfg(feature = "test-hooks")] +#[tokio::test] +async fn test_the_summary_counts_the_old_entry_a_stopped_rewrite_leaves() { + use extenddb_storage::backup_catalog::StoreRootCount; + use extenddb_storage_cassandra::test_hooks::{RewriteStop, stop_rewrite}; + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let tag = crate::helpers::unique_test_id(); + let old_root = format!("/srv/backups-old-{tag}"); + let old = (StoreKind::Filesystem, old_root.as_str(), "old-1"); + let to = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: format!("/srv/backups-new-{tag}"), + store_id: "new-1".to_owned(), + }; + let row = row_at(&account_id, "stopped", BackupStatus::Available, old); + assert!(engine.insert_backup_row(&row).await.expect("insert")); + stop_rewrite(&row.backup_arn, RewriteStop::AfterRow); + engine + .rewrite_store(old.0, old.1, &[row.key()], &to) + .await + .expect_err("stopped after the transaction"); + let ours: Vec = engine + .summarize_store_roots() + .await + .expect("summary") + .into_iter() + .filter(|c| { + [old_root.as_str(), to.store_root.as_str()].contains(&c.store.store_root.as_str()) + }) + .collect(); + let mut expected = vec![ + StoreRootCount { + store: row.store.clone(), + status: BackupStatus::Available, + rows: 1, + }, + StoreRootCount { + store: to.clone(), + status: BackupStatus::Available, + rows: 1, + }, + ]; + StoreRootCount::sort(&mut expected); + assert_eq!(ours, expected, "the orphan entry counts for the old root"); + let every = under_both(&[&row], &to); + remove_index_rows(&engine, &every.iter().collect::>()).await; +} + /// `insert_backup_row`: the row lands as given, unowned, with `completed_at` /// and `heartbeat_at` on the coordinator's clock, its status entry and both /// list rows written (`list_backup_rows` reads the first, the account and @@ -5937,6 +7269,10 @@ async fn test_rows_older_than_are_listed_on_the_named_clock() { ); } +/// The cells that take the catalog-wide maintenance claim, one at a time: +/// the catalog is shared by this binary's tests. +static MAINTENANCE_CLAIM_CELLS: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + /// The maintenance claim: taken once, refused to a second holder naming the /// first, heartbeated on the coordinator's clock, released; a holder whose /// heartbeat is older than the stale window is dead, so its claim is not @@ -5948,11 +7284,13 @@ async fn test_the_maintenance_claim_excludes_a_second_holder_and_expires_when_st if crate::helpers::skip_without_cassandra() { return; } + let _one_at_a_time = MAINTENANCE_CLAIM_CELLS.lock().await; let engine = setup_engine().await; let stale = Duration::from_secs(300); let check = || engine.check_maintenance_claim(stale); - // Another test of this binary never holds the claim; a stale row an - // interrupted run left is taken over below like any dead holder's. + // Another test of this binary holds the claim only under + // MAINTENANCE_CLAIM_CELLS; a stale row an interrupted run left is taken + // over below like any dead holder's. let first = engine .maintenance_claim("purge on host-a", stale) .await @@ -6016,6 +7354,169 @@ async fn test_the_maintenance_claim_excludes_a_second_holder_and_expires_when_st assert!(!check().await.expect("check")); } +/// `--move-store` of 10 FAILED rows on Cassandra, whose maintenance claim +/// another command takes over while the third key's rewrite is in flight +/// (the holder's heartbeat aged past the stale window, as a command stopped +/// past `stale_job_after_secs` leaves it): the key in flight lands, the +/// holder's next heartbeat finds the claim lost, and the move stops before +/// the fourth key with the seven others left under the old root and +/// listed there. Cassandra rewrites key by key, so the move calls +/// `rewrite_store` one key at a time and reads the claim before each. +#[cfg(feature = "test-hooks")] +#[tokio::test] +async fn test_a_move_whose_claim_is_taken_over_inside_its_rewrite_stops_at_the_next_key() { + use std::os::unix::fs::PermissionsExt; + use std::sync::Arc; + + use extenddb_backup_store::{BackupStore, FilesystemStore}; + use extenddb_engine::store_maintenance::{MoveTarget, move_store}; + use extenddb_engine::{ConfiguredStore, HeldClaim, check_configured_store}; + use extenddb_storage_cassandra::test_hooks::pause_before_rewrite_write; + if crate::helpers::skip_without_cassandra() { + return; + } + let _one_at_a_time = MAINTENANCE_CLAIM_CELLS.lock().await; + let engine = Arc::new(setup_engine().await); + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let tag = crate::helpers::unique_test_id(); + let old_root = format!("/srv/backups-old-{tag}"); + let old = (StoreKind::Filesystem, old_root.as_str(), "old-1"); + + // The configured store: a private filesystem root its first check + // gives this catalog's identity file. + let dir = tempfile::tempdir().expect("tempdir"); + std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o700)).expect("chmod"); + let store: Arc = + Arc::new(FilesystemStore::open(dir.path()).await.expect("store")); + let label = dir.path().canonicalize().unwrap().display().to_string(); + let configured = ConfiguredStore::checking(StoreKind::Filesystem, label, store, None); + let outcome = check_configured_store(&configured, &*engine, "cassandra").await; + assert!(outcome.is_ok(), "the store's first check: {outcome:?}"); + configured.record_check(outcome); + let identity = configured.identity().expect("healthy"); + + let mut rows = Vec::new(); + for n in 0..10 { + let mut row = row_at(&account_id, &format!("m{n}"), BackupStatus::Failed, old); + row.completed_at = Some(row.created_at); + assert!(engine.insert_backup_row(&row).await.expect("insert")); + rows.push(row); + } + let order = by_root(&engine, old.0, old.1).await; + assert_eq!(order.len(), 10); + let pause = pause_before_rewrite_write(&order[2]); + + let stale = Duration::from_secs(300); + let claim = engine + .maintenance_claim("backup sync --move-store on host-a", stale) + .await + .expect("claim"); + let generation = i64::try_from(claim.generation()).unwrap(); + let held = HeldClaim::start(claim, Duration::from_secs(1)); + let never = || None; + let mover = move_store( + &*engine, + MoveTarget { + configured: &configured, + identity: &identity, + from_kind: old.0, + from_root: old.1, + owner_window: stale, + stop: &never, + claim: Some(&held), + }, + ); + let aged = format!( + "UPDATE {}.backup_maintenance SET heartbeat_at = ? WHERE claim = 'claim' \ + IF owner_generation = ?", + engine.catalog_keyspace() + ); + let taker = async { + pause.reached().await; + // The holder's heartbeat aged past the window, then another command + // takes the claim over; a beat of the holder's timer landing between + // the two refuses the take, which is tried again. + let taker = loop { + let ten_minutes_ago = i64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis(), + ) + .unwrap() + - 600_000; + apply( + &engine, + &aged, + cdrs_tokio::query_values!(ten_minutes_ago, generation), + ) + .await; + match engine + .maintenance_claim("backup sync on host-b", stale) + .await + { + Ok(taker) => break taker, + Err(StorageError::AlreadyExists(_)) => continue, + Err(_) => panic!("the take-over failed"), + } + }; + let started = Instant::now(); + while held.lost().is_none() { + assert!( + started.elapsed() < Duration::from_secs(10), + "the holder's heartbeat found the claim lost" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + pause.release(); + taker + }; + let (report, taker) = tokio::join!(mover, taker); + let report = report.expect("the move read its rows"); + let stopped = report.stopped.clone().expect("stopped"); + assert!( + stopped.contains("another holder took it over") + && stopped.contains( + "before the rewrite of row 4 of 10; the 3 row(s) rewritten before it are moved" + ), + "{stopped}" + ); + assert_eq!(report.rewritten, 3); + assert_eq!(report.moved, order[..3]); + let left: Vec = report.left.iter().map(|r| r.backup_arn.clone()).collect(); + assert_eq!(left, order[3..]); + assert_eq!(by_root(&engine, old.0, old.1).await, order[3..]); + for (n, arn) in order.iter().enumerate() { + let stored = engine + .read_backup_row(&BackupArn::parse(arn).unwrap()) + .await + .unwrap() + .expect("the row exists"); + let want = if n < 3 { + identity.clone() + } else { + rows[0].store.clone() + }; + assert_eq!(stored.store, want, "{arn}"); + } + taker.release().await.expect("release"); + held.release().await.ok(); + let every: Vec = rows + .iter() + .flat_map(|row| { + [ + row.clone(), + BackupRow { + store: identity.clone(), + ..row.clone() + }, + ] + }) + .collect(); + remove_index_rows(&engine, &every.iter().collect::>()).await; +} + /// Two takers racing for one dead holder's row see one win: each reads the /// stale row (the test hook holds both after their read, so neither sees /// the other's write first) and runs `UPDATE ... IF owner_generation = `, @@ -6028,6 +7529,7 @@ async fn test_two_takers_of_one_stale_maintenance_claim_see_one_win() { if crate::helpers::skip_without_cassandra() { return; } + let _one_at_a_time = MAINTENANCE_CLAIM_CELLS.lock().await; let engine = setup_engine().await; let stale = Duration::from_secs(300); let dead = engine @@ -6105,6 +7607,7 @@ async fn test_a_maintenance_insert_whose_answer_is_lost_is_settled_by_the_re_rea if crate::helpers::skip_without_cassandra() { return; } + let _one_at_a_time = MAINTENANCE_CLAIM_CELLS.lock().await; let engine = setup_engine().await; let stale = Duration::from_secs(300); assert!( @@ -9143,3 +10646,386 @@ async fn test_a_running_store_backed_restore_keeps_its_backup_in_use_until_activ None ); } + +/// Deleters whose lightweight transaction lands before a `rewrite_store` +/// of their row and whose entry batch lands after it: DeleteBackup's +/// DELETING and the deletion's DELETED each leave an entry under the old +/// root with no row behind it, which the old root's ids and the summary +/// count. `remove_orphan_status_entries` of the old root deletes both, +/// checked against the rows, and leaves the moved rows listed under the new +/// root. A tombstone removal that read its row before the rewrite is +/// conditioned on the store it read, so it leaves the moved row. +#[cfg(feature = "test-hooks")] +#[tokio::test] +async fn test_the_entries_deleters_leave_behind_a_rewrite_are_swept_from_the_old_root() { + use extenddb_storage::backup_catalog::StatusEntrySweep; + use extenddb_storage_cassandra::test_hooks::{ + pause_after_delete_claim, pause_after_deleted_write, pause_before_tombstone_removal, + }; + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = std::sync::Arc::new(setup_engine().await); + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let tag = crate::helpers::unique_test_id(); + let old_root = format!("/srv/backups-old-{tag}"); + let old = (StoreKind::Filesystem, old_root.as_str(), "old-1"); + let to = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: format!("/srv/backups-new-{tag}"), + store_id: "new-1".to_owned(), + }; + let old_entry = Some(("filesystem".to_owned(), "old-1".to_owned())); + + // DeleteBackup: its DELETING claim applies, then the move, then its + // entry batch. + let available = row_at(&account_id, "deleting", BackupStatus::Available, old); + assert!(engine.insert_backup_row(&available).await.expect("insert")); + let pause = pause_after_delete_claim(&available.backup_arn); + let start = { + let engine = std::sync::Arc::clone(&engine); + let key = available.key(); + tokio::spawn(async move { engine.start_backup_delete(&key).await }) + }; + pause.reached().await; + assert_eq!( + engine + .rewrite_store(old.0, old.1, &[available.key()], &to) + .await + .expect("rewrite"), + 1 + ); + pause.release(); + assert_eq!( + start.await.expect("delete task").expect("delete"), + DeleteStart::Deleting + ); + assert_eq!( + status_entry( + &engine, + old.1, + BackupStatus::Deleting, + &available.backup_arn + ) + .await, + old_entry, + "the batch wrote the DELETING entry under the old root" + ); + + // The DELETED write: its transaction applies, then the move, then its + // entry batch. + let deleting = row_at(&account_id, "deleted", BackupStatus::Deleting, old); + assert!(engine.insert_backup_row(&deleting).await.expect("insert")); + let pause = pause_after_deleted_write(&deleting.backup_arn); + let finish = { + let engine = std::sync::Arc::clone(&engine); + let key = deleting.key(); + tokio::spawn(async move { engine.set_backup_deleted(&key).await }) + }; + pause.reached().await; + assert_eq!( + engine + .rewrite_store(old.0, old.1, &[deleting.key()], &to) + .await + .expect("rewrite"), + 1 + ); + pause.release(); + assert!(finish.await.expect("task").expect("set DELETED")); + assert_eq!( + status_entry(&engine, old.1, BackupStatus::Deleted, &deleting.backup_arn).await, + old_entry + ); + + // The tombstone removal: read before the move, its transaction after. + let tombstone = row_at(&account_id, "tombstone", BackupStatus::Deleted, old); + assert!(engine.insert_backup_row(&tombstone).await.expect("insert")); + let pause = pause_before_tombstone_removal(&tombstone.backup_arn); + let removal = { + let engine = std::sync::Arc::clone(&engine); + let key = tombstone.key(); + let root = old_root.clone(); + tokio::spawn(async move { + engine + .remove_backup_row(&key, StoreKind::Filesystem, &root) + .await + }) + }; + pause.reached().await; + assert_eq!( + engine + .rewrite_store(old.0, old.1, &[tombstone.key()], &to) + .await + .expect("rewrite"), + 1 + ); + pause.release(); + assert!( + !removal.await.expect("task").expect("removal"), + "the removal names the store it read" + ); + assert!( + listed_with(&engine, BackupStatus::Deleted, &to, &tombstone.backup_arn).await, + "the moved tombstone is the new store's" + ); + + assert_eq!( + engine.read_store_ids(old.0, old.1).await.unwrap(), + ["old-1"], + "the old root's ids count the two entries" + ); + assert!(by_root(&engine, old.0, old.1).await.is_empty()); + assert_eq!( + engine + .remove_orphan_status_entries(old.0, old.1) + .await + .unwrap(), + StatusEntrySweep { + removed: 2, + repaired: 0 + } + ); + assert!( + engine + .read_store_ids(old.0, old.1) + .await + .unwrap() + .is_empty() + ); + let summary: Vec<_> = engine + .summarize_store_roots() + .await + .unwrap() + .into_iter() + .filter(|c| c.store.store_root == old_root) + .collect(); + assert!(summary.is_empty(), "{summary:?}"); + for (status, row) in [ + (BackupStatus::Deleting, &available), + (BackupStatus::Deleted, &deleting), + (BackupStatus::Deleted, &tombstone), + ] { + assert!( + listed_with(&engine, status, &to, &row.backup_arn).await, + "{} is still listed under the new root", + row.backup_arn + ); + } + assert_eq!( + engine + .remove_orphan_status_entries(to.store_kind, &to.store_root) + .await + .unwrap(), + StatusEntrySweep::default(), + "the new root's entries have their rows" + ); + let every = under_both(&[&available, &deleting, &tombstone], &to); + let mut all: Vec = every.clone(); + for row in &every { + for status in [ + BackupStatus::Available, + BackupStatus::Deleting, + BackupStatus::Deleted, + ] { + all.push(BackupRow { + status, + ..row.clone() + }); + } + } + remove_index_rows(&engine, &all.iter().collect::>()).await; +} + +/// A row whose only status entry is under an earlier status: a DeleteBackup +/// that stopped between its applied DELETING transaction and its entry +/// batch (the row DELETING, its entry AVAILABLE), and a job whose AVAILABLE +/// transaction applied and whose entry batch failed (the row AVAILABLE, its +/// entry CREATING). The status-scoped `list_backup_rows` lists neither; +/// `list_backup_rows_by_root` lists each under its own status, and a row +/// with two entries (the `doubled` row's extra DELETED entry) once. +/// `rewrite_store` of what the by-root listing returns moves the DELETING +/// row, after which the old root lists it no more. The sweep of the old +/// root then deletes the moved row's stale entry (its row names another +/// store) and repairs the others: each row's entry under its own status is +/// written and the stale one deleted, so `list_backup_rows` lists them. +#[cfg(feature = "test-hooks")] +#[tokio::test] +async fn test_a_row_whose_entry_is_under_an_earlier_status_is_listed_moved_and_repaired() { + use extenddb_storage::backup_catalog::StatusEntrySweep; + use extenddb_storage_cassandra::test_hooks::pause_after_delete_claim; + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = std::sync::Arc::new(setup_engine().await); + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let tag = crate::helpers::unique_test_id(); + let old_root = format!("/srv/backups-old-{tag}"); + let old = (StoreKind::Filesystem, old_root.as_str(), "old-1"); + let to = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: format!("/srv/backups-new-{tag}"), + store_id: "new-1".to_owned(), + }; + let catalog = engine.catalog_keyspace(); + let raw = |cql: String, values: cdrs_tokio::query::QueryValues| { + let engine = std::sync::Arc::clone(&engine); + async move { + engine + .session() + .query_with_values(&cql, values) + .await + .expect("a raw write"); + } + }; + + // DeleteBackup stopped after its DELETING transaction, before its batch. + let deleting = row_at(&account_id, "deleting", BackupStatus::Available, old); + assert!(engine.insert_backup_row(&deleting).await.expect("insert")); + let pause = pause_after_delete_claim(&deleting.backup_arn); + let start = { + let engine = std::sync::Arc::clone(&engine); + let key = deleting.key(); + tokio::spawn(async move { engine.start_backup_delete(&key).await }) + }; + pause.reached().await; + start.abort(); + let _ = start.await; + + // A job's AVAILABLE transaction applied, its entry batch failed. + let job = row_at(&account_id, "job", BackupStatus::Creating, old); + let claim = engine.claim_backup(&job).await.expect("claim"); + raw( + format!( + "UPDATE {catalog}.backups_by_arn SET backup_status = 'AVAILABLE' \ + WHERE account_id = ? AND backup_arn = ?" + ), + cdrs_tokio::query_values!(account_id.as_str(), job.backup_arn.as_str()), + ) + .await; + drop(claim); + + // An AVAILABLE row with an extra DELETED entry beside its own. + let doubled = row_at(&account_id, "doubled", BackupStatus::Available, old); + assert!(engine.insert_backup_row(&doubled).await.expect("insert")); + raw( + format!( + "INSERT INTO {catalog}.backup_index_by_status \ + (store_root, backup_status, backup_arn, account_id, store_kind, store_id) \ + VALUES (?, 'DELETED', ?, ?, 'filesystem', 'old-1')" + ), + cdrs_tokio::query_values!( + old_root.as_str(), + doubled.backup_arn.as_str(), + account_id.as_str() + ), + ) + .await; + + let old_store = deleting.store.clone(); + assert!( + !listed_with( + &engine, + BackupStatus::Deleting, + &old_store, + &deleting.backup_arn + ) + .await + ); + assert!( + !listed_with( + &engine, + BackupStatus::Available, + &old_store, + &job.backup_arn + ) + .await + ); + let listed: Vec = engine + .list_backup_rows_by_root(old.0, old.1, None) + .try_collect() + .await + .expect("listing"); + let mut statuses: Vec<(String, BackupStatus)> = listed + .iter() + .map(|r| (r.backup_arn.clone(), r.status)) + .collect(); + statuses.sort_by(|a, b| a.0.cmp(&b.0)); + let mut expected = vec![ + (deleting.backup_arn.clone(), BackupStatus::Deleting), + (job.backup_arn.clone(), BackupStatus::Available), + (doubled.backup_arn.clone(), BackupStatus::Available), + ]; + expected.sort_by(|a, b| a.0.cmp(&b.0)); + assert_eq!(statuses, expected, "each row once, under its own status"); + let only_deleting: Vec = engine + .list_backup_rows_by_root(old.0, old.1, Some(BackupStatus::Deleting)) + .map_ok(|r| r.backup_arn) + .try_collect() + .await + .expect("listing"); + assert_eq!(only_deleting, std::slice::from_ref(&deleting.backup_arn)); + + // The move of what the by-root listing returns: the DELETING row moves + // (the job and the doubled row are left out of the keys here). + assert_eq!( + engine + .rewrite_store(old.0, old.1, &[deleting.key()], &to) + .await + .expect("rewrite"), + 1 + ); + let mut left = by_root(&engine, old.0, old.1).await; + left.sort(); + let mut expected_left = vec![job.backup_arn.clone(), doubled.backup_arn.clone()]; + expected_left.sort(); + assert_eq!( + left, expected_left, + "the moved row is no longer the old root's" + ); + assert!(listed_with(&engine, BackupStatus::Deleting, &to, &deleting.backup_arn).await); + + assert_eq!( + engine + .remove_orphan_status_entries(old.0, old.1) + .await + .unwrap(), + StatusEntrySweep { + removed: 1, + repaired: 2 + } + ); + assert!( + listed_with( + &engine, + BackupStatus::Available, + &old_store, + &job.backup_arn + ) + .await, + "the job row's entry is under AVAILABLE now" + ); + for (status, arn) in [ + (BackupStatus::Available, &deleting.backup_arn), + (BackupStatus::Creating, &job.backup_arn), + (BackupStatus::Deleted, &doubled.backup_arn), + ] { + assert_eq!(status_entry(&engine, old.1, status, arn).await, None); + } + assert!( + status_entry(&engine, old.1, BackupStatus::Available, &doubled.backup_arn) + .await + .is_some() + ); + assert_eq!( + engine + .remove_orphan_status_entries(old.0, old.1) + .await + .unwrap(), + StatusEntrySweep::default(), + "a second sweep finds nothing" + ); + let every = under_both(&[&deleting, &job, &doubled], &to); + remove_index_rows(&engine, &every.iter().collect::>()).await; +} diff --git a/crates/storage-mongodb/src/backup_catalog.rs b/crates/storage-mongodb/src/backup_catalog.rs index 4dda98fb4..4a8463865 100644 --- a/crates/storage-mongodb/src/backup_catalog.rs +++ b/crates/storage-mongodb/src/backup_catalog.rs @@ -45,7 +45,7 @@ use extenddb_storage::backup_catalog::{ BackupCatalog, BackupKey, BackupRow, BackupStatus, DeleteStart, ForgetRow, Heartbeat, JobClaim, JobRow, JobStatus, MaintenanceClaim, RestoreJobRow, RestoreKind, RestoreRegistered, RestoreRegistrationRow, RestoreTargetKey, StaleClock, StatusFields, StoreIdentity, StoreKind, - TimeBounds, UNOWNED_GENERATION, WriteOutcome, new_generation, + StoreRootCount, TimeBounds, UNOWNED_GENERATION, WriteOutcome, new_generation, }; use extenddb_storage::error::StorageError; use futures::TryStreamExt; @@ -69,6 +69,10 @@ pub(crate) const BACKUP_MAINTENANCE: &str = "backup_maintenance"; pub(crate) const CATALOG_ID: &str = "catalog_id"; /// The counter a row-locking transaction `$inc`s so two of them conflict. pub(crate) const LOCK_SEQ: &str = "lock_seq"; +/// Keys per `update_many` of `rewrite_store`: one statement's `$or` stays +/// far below the 16 MiB command bound, and a root of any size is a few +/// statements. +pub(crate) const REWRITE_CHUNK_KEYS: usize = 1_000; /// The restore job record, a subdocument of a restore target's `tables` /// document, with the field names the job record has on every backend. @@ -946,6 +950,30 @@ impl BackupCatalog for MongoEngine { }) } + fn list_backup_rows_by_root<'a>( + &'a self, + store_kind: StoreKind, + store_root: &'a str, + status: Option, + ) -> BoxStream<'a, Result> { + // The `{store_kind, store_root}` prefix of the store index; the + // status is a residual filter. + rows_stream(async move { + let mut filter = doc! { "store_kind": store_kind.as_str(), "store_root": store_root }; + if let Some(status) = status { + filter.insert("status", status.as_str()); + } + self.backup_index() + .find(filter) + .sort(listing_sort()) + .await + .map_err(internal)? + .try_collect() + .await + .map_err(internal) + }) + } + fn newest_available_backup_row<'a>( &'a self, store: &'a StoreIdentity, @@ -1035,6 +1063,57 @@ impl BackupCatalog for MongoEngine { }) } + fn rewrite_store<'a>( + &'a self, + from_kind: StoreKind, + from_root: &'a str, + keys: &'a [BackupKey], + to: &'a StoreIdentity, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + // One `update_many` per chunk, no transaction: one over every key + // of a large root would meet the transaction size and lifetime + // limits. Each document's update is atomic. This update and a + // transaction that writes the document (a DeleteBackup's + // `start_backup_delete_tx`, a restore's `claim_restore_tx`) are + // serialized: when the update commits after the transaction's + // snapshot and before its write, the transaction conflicts and + // re-runs, reading the row as the update left it; when the + // transaction wrote the document first, the update waits for it + // to end and then applies its filter to the committed document. + // A registration keeps the store its server is configured with. + // The `$inc` of `lock_seq` is a convention, not what serializes + // them: a write that moves the row also advances its lock + // counter. + let mut rewritten = 0; + for chunk in keys.chunks(REWRITE_CHUNK_KEYS) { + let any_key: Vec = chunk.iter().map(key_filter).collect(); + let result = self + .backup_index() + .update_many( + doc! { + "$or": any_key, + "store_kind": from_kind.as_str(), + "store_root": from_root, + "status": { "$ne": BackupStatus::Creating.as_str() }, + }, + doc! { + "$set": { + "store_kind": to.store_kind.as_str(), + "store_root": &to.store_root, + "store_id": &to.store_id, + }, + "$inc": { LOCK_SEQ: 1_i64 }, + }, + ) + .await + .map_err(internal)?; + rewritten += result.modified_count; + } + Ok(rewritten) + }) + } + fn set_backup_deleted<'a>( &'a self, key: &'a BackupKey, @@ -1060,6 +1139,8 @@ impl BackupCatalog for MongoEngine { fn remove_backup_row<'a>( &'a self, key: &'a BackupKey, + store_kind: StoreKind, + store_root: &'a str, ) -> BoxFuture<'a, Result> { Box::pin(async move { let mut filter = key_filter(key); @@ -1067,6 +1148,8 @@ impl BackupCatalog for MongoEngine { "status", doc! { "$in": [BackupStatus::Deleted.as_str(), BackupStatus::Failed.as_str()] }, ); + filter.insert("store_kind", store_kind.as_str()); + filter.insert("store_root", store_root); let result = self .backup_index() .delete_one(filter) @@ -1419,6 +1502,49 @@ impl BackupCatalog for MongoEngine { }) } + fn summarize_store_roots(&self) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + let groups: Vec = self + .backup_index() + .aggregate([doc! { "$group": { + "_id": { + "store_kind": "$store_kind", + "store_root": "$store_root", + "store_id": "$store_id", + "status": "$status", + }, + "rows": { "$sum": 1_i64 }, + } }]) + .await + .map_err(internal)? + .try_collect() + .await + .map_err(internal)?; + let mut counts = Vec::with_capacity(groups.len()); + for group in groups { + let id = group.get_document("_id").map_err(internal)?; + let rows = match group.get("rows") { + Some(Bson::Int64(n)) => *n, + Some(Bson::Int32(n)) => i64::from(*n), + other => { + return Err(StorageError::Internal(format!( + "a root summary count is not an integer: {other:?}" + ))); + } + }; + counts.push(StoreRootCount::from_columns( + &required_str(id, "store_kind")?, + required_str(id, "store_root")?, + required_str(id, "store_id")?, + &required_str(id, "status")?, + rows, + )?); + } + StoreRootCount::sort(&mut counts); + Ok(counts) + }) + } + fn maintenance_claim<'a>( &'a self, holder: &'a str, diff --git a/crates/storage-mongodb/src/backup_live_tests.rs b/crates/storage-mongodb/src/backup_live_tests.rs index 4b2cb377e..3b1f60218 100644 --- a/crates/storage-mongodb/src/backup_live_tests.rs +++ b/crates/storage-mongodb/src/backup_live_tests.rs @@ -130,6 +130,15 @@ impl Fixture { self.engine.catalog_db.drop().await.expect("drop catalog"); self.engine.data_db.drop().await.expect("drop data"); } + + /// [`Self::finish`] after a test body run under `catch_unwind`, whether + /// or not it panicked, and then the panic, if any, goes on. + async fn finish_after(self, outcome: std::thread::Result<()>) { + self.finish().await; + if let Err(panic) = outcome { + std::panic::resume_unwind(panic); + } + } } async fn engine_on(catalog: &str, data: &str, liveness: BackupLiveness) -> MongoEngine { @@ -754,6 +763,448 @@ async fn live_read_store_ids_lists_the_distinct_ids_of_a_root() { fx.finish().await; } +/// A row of `row(ACCOUNT, table, name)` with `status`, under the store of +/// `kind`, `root` and `id`. +fn row_at( + table: &str, + name: &str, + status: BackupStatus, + (kind, root, id): (StoreKind, &str, &str), +) -> BackupRow { + let mut row = row(ACCOUNT, table, name); + row.status = status; + row.store = StoreIdentity { + store_kind: kind, + store_root: root.to_owned(), + store_id: id.to_owned(), + }; + row.size_bytes = 10; + row.item_count = 2; + row +} + +/// The ARNs of `rows` in the listings' (`created_at`, `backup_arn`) order. +fn in_listing_order(rows: &[&BackupRow]) -> Vec { + let mut rows = rows.to_vec(); + rows.sort_by(|a, b| { + a.created_at + .total_cmp(&b.created_at) + .then_with(|| a.backup_arn.cmp(&b.backup_arn)) + }); + rows.into_iter().map(|r| r.backup_arn.clone()).collect() +} + +/// `list_backup_rows_by_root`: every row of the kind and root, whatever its +/// id, of every status or of one, whole and in the listings' order; nothing +/// of a root that differs by one byte (a trailing `/`, one character fewer), +/// of an S3 root that differs by its `?endpoint=` suffix, or of the other +/// kind at the same label. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_list_backup_rows_by_root_returns_every_id_and_status_of_the_root_and_nothing_else() { + let fixture = Fixture::new().await; + let outcome = std::panic::AssertUnwindSafe(async { + let fx = &fixture; + let root = "/srv/backups-old"; + let at = |id| (StoreKind::Filesystem, root, id); + let mut of_root = Vec::new(); + for (i, status) in [ + BackupStatus::Available, + BackupStatus::Deleting, + BackupStatus::Deleted, + BackupStatus::Failed, + ] + .into_iter() + .enumerate() + { + let id = if i % 2 == 0 { "store-1" } else { "store-2" }; + of_root.push(row_at("Music", &format!("r{i}"), status, at(id))); + } + // Inserted newest first: the order asserted is the listing's. + for row in of_root.iter().rev() { + assert!(fx.engine.insert_backup_row(row).await.expect("insert")); + } + let creating = row_at("Music", "job", BackupStatus::Creating, at("store-1")); + let claim = fx.engine.claim_backup(&creating).await.expect("claim"); + let slash = format!("{root}/"); + let s3 = "s3://bucket/prefix"; + let s3_endpoint = "s3://bucket/prefix?endpoint=http://127.0.0.1:9200"; + let others = [ + row_at( + "Music", + "slash", + BackupStatus::Available, + (StoreKind::Filesystem, &slash, "store-1"), + ), + row_at( + "Music", + "shorter", + BackupStatus::Available, + (StoreKind::Filesystem, "/srv/backups-ol", "store-1"), + ), + row_at( + "Music", + "kind", + BackupStatus::Available, + (StoreKind::S3, root, "store-1"), + ), + row_at( + "Music", + "s3", + BackupStatus::Available, + (StoreKind::S3, s3, "store-3"), + ), + row_at( + "Music", + "endpoint", + BackupStatus::Available, + (StoreKind::S3, s3_endpoint, "store-3"), + ), + ]; + for row in &others { + assert!(fx.engine.insert_backup_row(row).await.expect("insert")); + } + let listed = |kind: StoreKind, root: String, status: Option| { + let engine = &fx.engine; + async move { + let rows: Vec = engine + .list_backup_rows_by_root(kind, &root, status) + .try_collect() + .await + .expect("listing"); + rows + } + }; + let arns = |rows: Vec| -> Vec { + rows.into_iter().map(|r| r.backup_arn).collect() + }; + let mut all: Vec<&BackupRow> = of_root.iter().collect(); + all.push(&creating); + let whole = listed(StoreKind::Filesystem, root.to_owned(), None).await; + assert_eq!(arns(whole.clone()), in_listing_order(&all)); + for row in &whole { + assert_eq!( + Some(row), + fx.engine + .read_backup_row(&arn_of(row)) + .await + .expect("read") + .as_ref(), + "the listing returns rows whole" + ); + } + for status in BackupStatus::ALL { + let of_status: Vec<&BackupRow> = + all.iter().copied().filter(|r| r.status == status).collect(); + assert_eq!( + arns(listed(StoreKind::Filesystem, root.to_owned(), Some(status)).await), + in_listing_order(&of_status), + "{status}" + ); + } + for (kind, label, row) in [ + (StoreKind::Filesystem, slash.as_str(), &others[0]), + (StoreKind::Filesystem, "/srv/backups-ol", &others[1]), + (StoreKind::S3, root, &others[2]), + (StoreKind::S3, s3, &others[3]), + (StoreKind::S3, s3_endpoint, &others[4]), + ] { + assert_eq!( + arns(listed(kind, label.to_owned(), None).await), + std::slice::from_ref(&row.backup_arn), + "{kind} {label}" + ); + } + assert!( + listed(StoreKind::Filesystem, "/srv/backups".to_owned(), None) + .await + .is_empty() + ); + drop(claim); + }) + .catch_unwind() + .await; + fixture.finish_after(outcome).await; +} + +/// `rewrite_store`: of the keys given, the rows still of the old kind and +/// root and not CREATING get the new store, with every other column left, +/// and are counted; a CREATING row, a row of another root, a row of the +/// other kind at the same label and a key with no row are left and not +/// counted; a second call counts 0; a row another writer moved to a third +/// root before the call is left there. The `$inc` of `lock_seq` is the one +/// other field the rewrite changes. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_rewrite_store_moves_only_the_old_roots_rows_that_are_not_creating() { + let fixture = Fixture::new().await; + let outcome = std::panic::AssertUnwindSafe(async { + let fx = &fixture; + let old = (StoreKind::Filesystem, "/srv/backups-old", "old-1"); + let to = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/srv/backups-new".to_owned(), + store_id: "new-1".to_owned(), + }; + let mut movable = Vec::new(); + for (i, status) in [ + BackupStatus::Available, + BackupStatus::Deleting, + BackupStatus::Deleted, + BackupStatus::Failed, + ] + .into_iter() + .enumerate() + { + movable.push(row_at("Music", &format!("m{i}"), status, old)); + } + let moved_away = row_at("Music", "away", BackupStatus::Available, old); + let other_root = row_at( + "Music", + "other-root", + BackupStatus::Available, + (StoreKind::Filesystem, "/srv/backups-other", "old-1"), + ); + let other_kind = row_at( + "Music", + "other-kind", + BackupStatus::Available, + (StoreKind::S3, "/srv/backups-old", "old-1"), + ); + for row in movable + .iter() + .chain([&moved_away, &other_root, &other_kind]) + { + assert!(fx.engine.insert_backup_row(row).await.expect("insert")); + } + let creating = row_at("Music", "job", BackupStatus::Creating, old); + let claim = fx.engine.claim_backup(&creating).await.expect("claim"); + let stored = |row: &BackupRow| { + let engine = &fx.engine; + let arn = arn_of(row); + async move { + engine + .read_backup_row(&arn) + .await + .expect("read") + .expect("the row exists") + } + }; + let lock_seq = |row: &BackupRow| { + let engine = &fx.engine; + let id = row.backup_id.clone(); + async move { + engine + .catalog_db + .collection::(BACKUP_INDEX) + .find_one(doc! { "backup_id": &id }) + .await + .expect("read") + .expect("the document exists") + .get_i64("lock_seq") + .expect("lock_seq") + } + }; + let mut before = Vec::new(); + for row in &movable { + before.push((stored(row).await, lock_seq(row).await)); + } + // Another writer moves one row to a third root first. + let third = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/srv/backups-third".to_owned(), + store_id: "third-1".to_owned(), + }; + assert_eq!( + fx.engine + .rewrite_store(old.0, old.1, &[moved_away.key()], &third) + .await + .expect("rewrite"), + 1 + ); + let mut ghost = row_at("Music", "ghost", BackupStatus::Available, old).key(); + ghost.backup_id = extenddb_storage::util::new_backup_id(); + let keys: Vec = movable + .iter() + .chain([&creating, &moved_away, &other_root, &other_kind]) + .map(BackupRow::key) + .chain([ghost]) + .collect(); + assert_eq!( + fx.engine + .rewrite_store(old.0, old.1, &keys, &to) + .await + .expect("rewrite"), + 4, + "the four rows of the old root that are not CREATING" + ); + for (row, (before, seq)) in movable.iter().zip(&before) { + assert_eq!( + stored(row).await, + BackupRow { + store: to.clone(), + ..before.clone() + }, + "only the store changed" + ); + assert_eq!( + lock_seq(row).await, + seq + 1, + "the move advances the row's lock counter" + ); + } + let creating_now = stored(&creating).await; + assert_eq!( + (creating_now.status, creating_now.store.store_root.as_str()), + (BackupStatus::Creating, old.1) + ); + assert_eq!(creating_now.owner_generation, claim.generation()); + assert_eq!(stored(&moved_away).await.store, third); + assert_eq!(stored(&other_root).await.store, other_root.store); + assert_eq!(stored(&other_kind).await.store, other_kind.store); + assert_eq!( + fx.engine + .rewrite_store(old.0, old.1, &keys, &to) + .await + .expect("rewrite"), + 0, + "a second call finds nothing left to move" + ); + let listed: Vec = fx + .engine + .list_backup_rows(None, &to) + .try_collect() + .await + .expect("listing"); + let movable_refs: Vec<&BackupRow> = movable.iter().collect(); + assert_eq!( + listed.into_iter().map(|r| r.backup_arn).collect::>(), + in_listing_order(&movable_refs) + ); + assert_eq!( + fx.engine.read_store_ids(old.0, old.1).await.expect("ids"), + ["old-1"], + "the CREATING row keeps the old root" + ); + assert_eq!( + fx.engine + .rewrite_store(old.0, old.1, &[], &to) + .await + .expect("rewrite"), + 0 + ); + drop(claim); + }) + .catch_unwind() + .await; + fixture.finish_after(outcome).await; +} + +/// `rewrite_store` over more keys than one statement takes moves them all, +/// every chunk counted. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_rewrite_store_moves_every_chunk_of_keys() { + use crate::backup_catalog::REWRITE_CHUNK_KEYS; + let fixture = Fixture::new().await; + let outcome = std::panic::AssertUnwindSafe(async { + let fx = &fixture; + let old = (StoreKind::Filesystem, "/srv/backups-old", "old-1"); + let to = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/srv/backups-new".to_owned(), + store_id: "new-1".to_owned(), + }; + let rows = REWRITE_CHUNK_KEYS + 1; + let mut keys = Vec::with_capacity(rows); + for i in 0..rows { + let row = row_at("Music", &format!("r{i}"), BackupStatus::Available, old); + assert!(fx.engine.insert_backup_row(&row).await.expect("insert")); + keys.push(row.key()); + } + assert_eq!( + fx.engine + .rewrite_store(old.0, old.1, &keys, &to) + .await + .expect("rewrite"), + rows as u64 + ); + let left: Vec = fx + .engine + .list_backup_rows_by_root(old.0, old.1, None) + .try_collect() + .await + .expect("listing"); + assert!(left.is_empty(), "{} rows left", left.len()); + }) + .catch_unwind() + .await; + fixture.finish_after(outcome).await; +} + +/// `summarize_store_roots`: one count per kind, root, id and status with a +/// row, in the one order; two ids of one root, two statuses of one id, two +/// roots and the two kinds are each counted apart. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_summarize_store_roots_counts_the_rows_of_each_store_and_status() { + use extenddb_storage::backup_catalog::StoreRootCount; + let fixture = Fixture::new().await; + let outcome = std::panic::AssertUnwindSafe(async { + let fx = &fixture; + assert!( + fx.engine + .summarize_store_roots() + .await + .expect("summary") + .is_empty() + ); + let a1 = (StoreKind::Filesystem, "/srv/a", "id-1"); + let a2 = (StoreKind::Filesystem, "/srv/a", "id-2"); + let b1 = (StoreKind::Filesystem, "/srv/b", "id-1"); + let s3 = (StoreKind::S3, "s3://bucket/a", "id-1"); + let rows = [ + row_at("Music", "1", BackupStatus::Available, a1), + row_at("Music", "2", BackupStatus::Available, a1), + row_at("Music", "3", BackupStatus::Failed, a1), + row_at("Music", "4", BackupStatus::Available, a2), + row_at("Music", "5", BackupStatus::Deleting, b1), + row_at("Music", "7", BackupStatus::Available, b1), + row_at("Music", "6", BackupStatus::Available, s3), + ]; + for row in &rows { + assert!(fx.engine.insert_backup_row(row).await.expect("insert")); + } + let creating = row_at("Music", "job", BackupStatus::Creating, b1); + let claim = fx.engine.claim_backup(&creating).await.expect("claim"); + let line = |(kind, root, id): (StoreKind, &str, &str), status, rows| StoreRootCount { + store: StoreIdentity { + store_kind: kind, + store_root: root.to_owned(), + store_id: id.to_owned(), + }, + status, + rows, + }; + assert_eq!( + fx.engine.summarize_store_roots().await.expect("summary"), + [ + line(a1, BackupStatus::Available, 2), + line(a1, BackupStatus::Failed, 1), + line(a2, BackupStatus::Available, 1), + line(b1, BackupStatus::Creating, 1), + line(b1, BackupStatus::Available, 1), + line(b1, BackupStatus::Deleting, 1), + line(s3, BackupStatus::Available, 1), + ] + ); + drop(claim); + }) + .catch_unwind() + .await; + fixture.finish_after(outcome).await; +} + /// `insert_backup_row`: the row lands as given, unowned (the document /// carries `owner_generation: 0`, which the claim's upsert filter relies on), /// with `completed_at` and `heartbeat_at` on the server's clock; a second @@ -1534,7 +1985,10 @@ async fn live_start_backup_delete_checks_the_status_and_the_in_use_rule_under_th Err(StorageError::Validation(m)) if m.contains("Backup not found") )); assert!( - !fx.engine.remove_backup_row(&key).await.unwrap(), + !fx.engine + .remove_backup_row(&key, row.store.store_kind, &row.store.store_root) + .await + .unwrap(), "DELETING is not a tombstone" ); assert!(fx.engine.set_backup_deleted(&key).await.unwrap()); @@ -1547,8 +2001,82 @@ async fn live_start_backup_delete_checks_the_status_and_the_in_use_rule_under_th .unwrap(); assert_eq!(stored.status, BackupStatus::Deleted); assert!(stored.completed_at.is_some()); - assert!(fx.engine.remove_backup_row(&key).await.unwrap()); - assert!(!fx.engine.remove_backup_row(&key).await.unwrap()); + // A row another writer moved to another store after the caller read it + // is left by a removal that names the store read. + let moved_to = StoreIdentity { + store_kind: row.store.store_kind, + store_root: format!("{}-moved", row.store.store_root), + store_id: "moved-1".to_owned(), + }; + assert_eq!( + fx.engine + .rewrite_store( + row.store.store_kind, + &row.store.store_root, + std::slice::from_ref(&key), + &moved_to + ) + .await + .unwrap(), + 1 + ); + assert!( + !fx.engine + .remove_backup_row(&key, row.store.store_kind, &row.store.store_root) + .await + .unwrap(), + "a row moved to another store is left" + ); + assert_eq!( + fx.engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .map(|r| (r.status, r.store)), + Some((BackupStatus::Deleted, moved_to.clone())) + ); + // The same root under the other kind is another store: a removal that + // names the kind it read leaves the row. + let other_kind = StoreIdentity { + store_kind: if moved_to.store_kind == StoreKind::S3 { + StoreKind::Filesystem + } else { + StoreKind::S3 + }, + ..moved_to.clone() + }; + assert_eq!( + fx.engine + .rewrite_store( + moved_to.store_kind, + &moved_to.store_root, + std::slice::from_ref(&key), + &other_kind + ) + .await + .unwrap(), + 1 + ); + assert!( + !fx.engine + .remove_backup_row(&key, moved_to.store_kind, &moved_to.store_root) + .await + .unwrap(), + "a row moved to the other kind is left" + ); + let moved_to = other_kind; + assert!( + fx.engine + .remove_backup_row(&key, moved_to.store_kind, &moved_to.store_root) + .await + .unwrap() + ); + assert!( + !fx.engine + .remove_backup_row(&key, moved_to.store_kind, &moved_to.store_root) + .await + .unwrap() + ); assert!( fx.engine .read_backup_row(&arn_of(&row)) diff --git a/crates/storage-postgres/src/backup_catalog.rs b/crates/storage-postgres/src/backup_catalog.rs index 83453dff2..732ddeb5b 100644 --- a/crates/storage-postgres/src/backup_catalog.rs +++ b/crates/storage-postgres/src/backup_catalog.rs @@ -48,7 +48,7 @@ use extenddb_storage::backup_catalog::{ BackupCatalog, BackupKey, BackupRow, BackupStatus, DeleteStart, ForgetRow, Heartbeat, JobClaim, JobRow, JobStatus, MaintenanceClaim, RestoreJobRow, RestoreKind, RestoreRegistered, RestoreRegistrationRow, RestoreTargetKey, StaleClock, StatusFields, StoreIdentity, StoreKind, - TimeBounds, UNOWNED_GENERATION, WriteOutcome, new_generation, + StoreRootCount, TimeBounds, UNOWNED_GENERATION, WriteOutcome, new_generation, }; use extenddb_storage::error::StorageError; use futures::future::BoxFuture; @@ -604,6 +604,29 @@ impl BackupCatalog for PostgresEngine { }) } + fn list_backup_rows_by_root<'a>( + &'a self, + store_kind: StoreKind, + store_root: &'a str, + status: Option, + ) -> BoxStream<'a, Result> { + // The `(store_kind, store_root)` prefix of + // `idx_backup_index_store_status`; the status is a filter on it. + rows_stream(async move { + sqlx::query_as(&format!( + "SELECT {ROW_COLUMNS} FROM backup_index \ + WHERE store_kind = $1 AND store_root = $2 \ + AND ($3::TEXT IS NULL OR status = $3) ORDER BY created_at, backup_arn" + )) + .bind(store_kind.as_str()) + .bind(store_root) + .bind(status.map(BackupStatus::as_str)) + .fetch_all(&self.pool) + .await + .map_err(db_err) + }) + } + fn newest_available_backup_row<'a>( &'a self, store: &'a StoreIdentity, @@ -688,6 +711,46 @@ impl BackupCatalog for PostgresEngine { }) } + fn rewrite_store<'a>( + &'a self, + from_kind: StoreKind, + from_root: &'a str, + keys: &'a [BackupKey], + to: &'a StoreIdentity, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + // Batches of REWRITE_BATCH keys, each one statement in its own + // transaction under the two bounds, so no statement nears the + // statement timeout whatever the key count; a batch that fails + // leaves the batches before it committed and the rest left. + let mut rewritten = 0; + for batch in keys.chunks(REWRITE_BATCH) { + let mut attempt = 0; + rewritten += loop { + attempt += 1; + match self.rewrite_store_tx(from_kind, from_root, batch, to).await { + Ok(count) => break count, + Err(e) + if attempt < REWRITE_ATTEMPTS + && crate::data::lock_not_available(&e) => + { + // A row lock held past the timeout (a + // DeleteBackup's or a registration's + // transaction, or the backend of a client that + // died mid-statement): the transaction rolled + // back, and the next attempt waits again after + // a pause. + tokio::time::sleep(Duration::from_millis(500 * u64::from(attempt))) + .await; + } + Err(e) => return Err(db_err(e)), + } + }; + } + Ok(rewritten) + }) + } + fn set_backup_deleted<'a>( &'a self, key: &'a BackupKey, @@ -710,14 +773,18 @@ impl BackupCatalog for PostgresEngine { fn remove_backup_row<'a>( &'a self, key: &'a BackupKey, + store_kind: StoreKind, + store_root: &'a str, ) -> BoxFuture<'a, Result> { Box::pin(async move { let affected = sqlx::query( "DELETE FROM backup_index WHERE account_id = $1 AND backup_id = $2 \ - AND status IN ('DELETED', 'FAILED')", + AND status IN ('DELETED', 'FAILED') AND store_kind = $3 AND store_root = $4", ) .bind(&key.account_id) .bind(&key.backup_id) + .bind(store_kind.as_str()) + .bind(store_root) .execute(&self.pool) .await .map_err(db_err)? @@ -1129,6 +1196,26 @@ impl BackupCatalog for PostgresEngine { }) } + fn summarize_store_roots(&self) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + let grouped: Vec<(String, String, String, String, i64)> = sqlx::query_as( + "SELECT store_kind, store_root, store_id, status, count(*) FROM backup_index \ + GROUP BY store_kind, store_root, store_id, status", + ) + .fetch_all(&self.pool) + .await + .map_err(db_err)?; + let mut counts = grouped + .into_iter() + .map(|(kind, root, id, status, rows)| { + StoreRootCount::from_columns(&kind, root, id, &status, rows) + }) + .collect::, _>>()?; + StoreRootCount::sort(&mut counts); + Ok(counts) + }) + } + fn maintenance_claim<'a>( &'a self, holder: &'a str, @@ -1234,7 +1321,157 @@ const MAINTENANCE_LOCK_KEY: &str = "backup_maintenance"; const MAINTENANCE_TAKE_TRIES: u32 = 6; const MAINTENANCE_TAKE_PAUSE: Duration = Duration::from_millis(2); +/// `rewrite_store`'s `lock_timeout`. The rewrite waits on the row locks a +/// DeleteBackup's or a registration's `FOR UPDATE` transaction takes, each +/// a few statements long, so a wait past this is a lock held by something +/// else: the backend of a client that died with its transaction open, or a +/// stalled session. The statement then fails with 55P03, its transaction +/// rolls back, and the rewrite tries again ([`REWRITE_ATTEMPTS`]). +#[doc(hidden)] +pub const REWRITE_LOCK_TIMEOUT: Duration = Duration::from_secs(3); + +/// `rewrite_store`'s `statement_timeout`: how long one batch's statement may +/// run in all, its lock waits included; past it the statement fails, rolls +/// back, and is not tried again, with the batches before it committed. +/// One statement over every key does not fit under a fixed bound: in plain +/// SQL on PostgreSQL 16 with migration 0006's indexes, over a catalog of +/// 400,000 rows with 1,449-byte descriptions, one statement over 200,000 +/// keys took 16.4 and 16.9 s, so the rewrite runs in batches of +/// [`REWRITE_BATCH`] keys. +#[doc(hidden)] +pub const REWRITE_STATEMENT_TIMEOUT: Duration = Duration::from_secs(30); + +/// How many keys one `rewrite_store` statement covers: one batch is one +/// transaction the client commits, under [`REWRITE_LOCK_TIMEOUT`] and +/// [`REWRITE_STATEMENT_TIMEOUT`]. On the catalog of +/// [`REWRITE_STATEMENT_TIMEOUT`]'s measurement a batch of 5,000 keys took +/// 0.23 to 0.29 s (six batches, each committed) and one of 20,000 keys 0.9 +/// to 1.1 s, so a batch stays about a hundred times inside the bound, and +/// 200,000 keys take 40 batches. +#[doc(hidden)] +pub const REWRITE_BATCH: usize = 5_000; + +/// How many times `rewrite_store` runs one batch's transaction when the +/// statement meets [`REWRITE_LOCK_TIMEOUT`]; the pauses between grow by +/// half a second. +#[doc(hidden)] +pub const REWRITE_ATTEMPTS: u32 = 3; + impl crate::PostgresEngine { + /// One attempt of one batch of `rewrite_store`: one statement over the + /// batch's keys inside a transaction this call commits, under + /// [`REWRITE_LOCK_TIMEOUT`] and [`REWRITE_STATEMENT_TIMEOUT`]. The row + /// lock the statement takes orders it against a registration's and a + /// delete's `FOR UPDATE`. CREATING rows are excluded; the statement + /// takes no advisory lock. Because the commit is the client's own, a + /// client that dies while the statement waits or runs rewrites nothing + /// of the batch: the server rolls the transaction back when it finds + /// the connection closed, at the latest when the statement ends. + async fn rewrite_store_tx( + &self, + from_kind: StoreKind, + from_root: &str, + keys: &[BackupKey], + to: &StoreIdentity, + ) -> Result { + let mut conn = self.pool.acquire().await?; + Self::rewrite_store_on(&mut conn, from_kind, from_root, keys, to, None).await + } + + /// [`Self::rewrite_store_tx`] on `conn`: the transaction begins and + /// commits there, so a test can read the connection's settings after + /// it ([`Self::rewrite_store_settings_after`]). With `inside`, `SHOW + /// lock_timeout` and `SHOW statement_timeout` are read in the + /// transaction once the bounds are set, and pushed there in that order. + async fn rewrite_store_on( + conn: &mut sqlx::PgConnection, + from_kind: StoreKind, + from_root: &str, + keys: &[BackupKey], + to: &StoreIdentity, + inside: Option<&mut Vec>, + ) -> Result { + use sqlx::Connection as _; + let accounts: Vec<&str> = keys.iter().map(|k| k.account_id.as_str()).collect(); + let ids: Vec<&str> = keys.iter().map(|k| k.backup_id.as_str()).collect(); + let mut tx = conn.begin().await?; + for (setting, bound) in [ + ("lock_timeout", REWRITE_LOCK_TIMEOUT), + ("statement_timeout", REWRITE_STATEMENT_TIMEOUT), + ] { + sqlx::query("SELECT set_config($1, $2, true)") + .bind(setting) + .bind(format!("{}ms", bound.as_millis())) + .execute(&mut *tx) + .await?; + } + if let Some(inside) = inside { + for name in ["lock_timeout", "statement_timeout"] { + let value: String = sqlx::query_scalar(&format!("SHOW {name}")) + .fetch_one(&mut *tx) + .await?; + inside.push(value); + } + } + let rewritten = sqlx::query( + "UPDATE backup_index SET store_kind = $1, store_root = $2, store_id = $3 \ + WHERE (account_id, backup_id) IN \ + (SELECT * FROM unnest($4::TEXT[], $5::TEXT[])) \ + AND store_kind = $6 AND store_root = $7 AND status <> 'CREATING'", + ) + .bind(to.store_kind.as_str()) + .bind(&to.store_root) + .bind(&to.store_id) + .bind(&accounts) + .bind(&ids) + .bind(from_kind.as_str()) + .bind(from_root) + .execute(&mut *tx) + .await? + .rows_affected(); + tx.commit().await?; + Ok(rewritten) + } + + /// One rewrite's transaction on one pooled connection, with `SHOW + /// lock_timeout` and `SHOW statement_timeout` read inside it once its + /// bounds are set, then the same two on that connection after the + /// commit: for the test that checks the rewrite runs under its bounds, + /// and that they end with its transaction and do not stay on the pooled + /// connection. The answer: the rows rewritten, the two settings inside, + /// the two after. + /// + /// # Errors + /// + /// The rewrite's or a `SHOW`'s error. + #[cfg(feature = "test-hooks")] + #[doc(hidden)] + pub async fn rewrite_store_settings_after( + &self, + from_kind: StoreKind, + from_root: &str, + keys: &[BackupKey], + to: &StoreIdentity, + ) -> Result<(u64, [String; 2], [String; 2]), StorageError> { + let mut conn = self.pool.acquire().await.map_err(db_err)?; + let mut inside = Vec::new(); + let rewritten = + Self::rewrite_store_on(&mut conn, from_kind, from_root, keys, to, Some(&mut inside)) + .await + .map_err(db_err)?; + let mut shown = Vec::new(); + for name in ["lock_timeout", "statement_timeout"] { + let value: String = sqlx::query_scalar(&format!("SHOW {name}")) + .fetch_one(&mut *conn) + .await + .map_err(db_err)?; + shown.push(value); + } + let after: [String; 2] = shown.try_into().expect("two settings"); + let inside: [String; 2] = inside.try_into().expect("two settings"); + Ok((rewritten, inside, after)) + } + /// DeleteBackup's first step, and with `remove` the outright removal of /// `forget_available_backup_row`, as one transaction: lock the row, read /// its status, run the in-use rule, then write DELETING or, with diff --git a/crates/storage-postgres/src/lib.rs b/crates/storage-postgres/src/lib.rs index 6a8948405..b4b6578ad 100755 --- a/crates/storage-postgres/src/lib.rs +++ b/crates/storage-postgres/src/lib.rs @@ -45,6 +45,10 @@ mod warm_throughput; mod worker_store; mod workers; +#[doc(hidden)] +pub use backup_catalog::{ + REWRITE_ATTEMPTS, REWRITE_BATCH, REWRITE_LOCK_TIMEOUT, REWRITE_STATEMENT_TIMEOUT, +}; #[cfg(feature = "test-hooks")] #[doc(hidden)] pub use backup_engine::claim_session_settings; diff --git a/crates/storage-postgres/tests/backup_restore.rs b/crates/storage-postgres/tests/backup_restore.rs index 2a58b5ff0..fc53a8517 100644 --- a/crates/storage-postgres/tests/backup_restore.rs +++ b/crates/storage-postgres/tests/backup_restore.rs @@ -3616,7 +3616,10 @@ mod catalog_index { Err(StorageError::Validation(m)) if m.contains("Backup not found") )); assert!( - !s.engine.remove_backup_row(&key).await.unwrap(), + !s.engine + .remove_backup_row(&key, row.store.store_kind, &row.store.store_root) + .await + .unwrap(), "DELETING is not a tombstone" ); assert!(s.engine.set_backup_deleted(&key).await.unwrap()); @@ -3629,8 +3632,82 @@ mod catalog_index { .unwrap(); assert_eq!(stored.status, BackupStatus::Deleted); assert!(stored.completed_at.is_some()); - assert!(s.engine.remove_backup_row(&key).await.unwrap()); - assert!(!s.engine.remove_backup_row(&key).await.unwrap()); + // A row another writer moved to another store after the caller read it + // is left by a removal that names the store read. + let moved_to = StoreIdentity { + store_kind: row.store.store_kind, + store_root: format!("{}-moved", row.store.store_root), + store_id: "moved-1".to_owned(), + }; + assert_eq!( + s.engine + .rewrite_store( + row.store.store_kind, + &row.store.store_root, + std::slice::from_ref(&key), + &moved_to + ) + .await + .unwrap(), + 1 + ); + assert!( + !s.engine + .remove_backup_row(&key, row.store.store_kind, &row.store.store_root) + .await + .unwrap(), + "a row moved to another store is left" + ); + assert_eq!( + s.engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .map(|r| (r.status, r.store)), + Some((BackupStatus::Deleted, moved_to.clone())) + ); + // The same root under the other kind is another store: a removal that + // names the kind it read leaves the row. + let other_kind = StoreIdentity { + store_kind: if moved_to.store_kind == StoreKind::S3 { + StoreKind::Filesystem + } else { + StoreKind::S3 + }, + ..moved_to.clone() + }; + assert_eq!( + s.engine + .rewrite_store( + moved_to.store_kind, + &moved_to.store_root, + std::slice::from_ref(&key), + &other_kind + ) + .await + .unwrap(), + 1 + ); + assert!( + !s.engine + .remove_backup_row(&key, moved_to.store_kind, &moved_to.store_root) + .await + .unwrap(), + "a row moved to the other kind is left" + ); + let moved_to = other_kind; + assert!( + s.engine + .remove_backup_row(&key, moved_to.store_kind, &moved_to.store_root) + .await + .unwrap() + ); + assert!( + !s.engine + .remove_backup_row(&key, moved_to.store_kind, &moved_to.store_root) + .await + .unwrap() + ); assert!( s.engine .read_backup_row(&arn_of(&row)) @@ -4042,6 +4119,729 @@ mod catalog_index { s.cleanup().await; } + /// A row of `row(ACCOUNT, table, name)` with `status`, under the store + /// of `kind`, `root` and `id`. + fn row_at( + table: &str, + name: &str, + status: BackupStatus, + (kind, root, id): (StoreKind, &str, &str), + ) -> BackupRow { + let mut row = row(ACCOUNT, table, name); + row.status = status; + row.store = StoreIdentity { + store_kind: kind, + store_root: root.to_owned(), + store_id: id.to_owned(), + }; + row.size_bytes = 10; + row.item_count = 2; + row + } + + /// The ARNs of `rows` in the listings' (`created_at`, `backup_arn`) + /// order. + fn in_listing_order(rows: &[&BackupRow]) -> Vec { + let mut rows = rows.to_vec(); + rows.sort_by(|a, b| { + a.created_at + .total_cmp(&b.created_at) + .then_with(|| a.backup_arn.cmp(&b.backup_arn)) + }); + rows.into_iter().map(|r| r.backup_arn.clone()).collect() + } + + /// `list_backup_rows_by_root`: every row of the kind and root, whatever + /// its id, of every status or of one, whole and in the listings' order; + /// nothing of a root that differs by one byte (a trailing `/`, one + /// character fewer), of an S3 root that differs by its `?endpoint=` + /// suffix, or of the other kind at the same label. + #[tokio::test] + async fn list_backup_rows_by_root_returns_every_id_and_status_of_the_root_and_nothing_else() { + if base_conn().is_none() { + eprintln!( + "SKIP list_backup_rows_by_root_returns_every_id_and_status_of_the_root_and_nothing_else: no PostgreSQL" + ); + return; + } + let s = scratch().await; + let root = "/srv/backups-old"; + let at = |id| (StoreKind::Filesystem, root, id); + let mut of_root = Vec::new(); + for (i, status) in [ + BackupStatus::Available, + BackupStatus::Deleting, + BackupStatus::Deleted, + BackupStatus::Failed, + ] + .into_iter() + .enumerate() + { + let id = if i % 2 == 0 { "store-1" } else { "store-2" }; + of_root.push(row_at("Music", &format!("r{i}"), status, at(id))); + } + // Inserted newest first: the order asserted is the listing's. + for row in of_root.iter().rev() { + assert!(s.engine.insert_backup_row(row).await.expect("insert")); + } + let creating = row_at("Music", "job", BackupStatus::Creating, at("store-1")); + let claim = s.engine.claim_backup(&creating).await.expect("claim"); + let slash = format!("{root}/"); + let s3 = "s3://bucket/prefix"; + let s3_endpoint = "s3://bucket/prefix?endpoint=http://127.0.0.1:9200"; + let others = [ + row_at( + "Music", + "slash", + BackupStatus::Available, + (StoreKind::Filesystem, &slash, "store-1"), + ), + row_at( + "Music", + "shorter", + BackupStatus::Available, + (StoreKind::Filesystem, "/srv/backups-ol", "store-1"), + ), + row_at( + "Music", + "kind", + BackupStatus::Available, + (StoreKind::S3, root, "store-1"), + ), + row_at( + "Music", + "s3", + BackupStatus::Available, + (StoreKind::S3, s3, "store-3"), + ), + row_at( + "Music", + "endpoint", + BackupStatus::Available, + (StoreKind::S3, s3_endpoint, "store-3"), + ), + ]; + for row in &others { + assert!(s.engine.insert_backup_row(row).await.expect("insert")); + } + let listed = |kind: StoreKind, root: String, status: Option| { + let engine = &s.engine; + async move { + let rows: Vec = engine + .list_backup_rows_by_root(kind, &root, status) + .try_collect() + .await + .expect("listing"); + rows + } + }; + let arns = |rows: Vec| -> Vec { + rows.into_iter().map(|r| r.backup_arn).collect() + }; + let mut all: Vec<&BackupRow> = of_root.iter().collect(); + all.push(&creating); + let whole = listed(StoreKind::Filesystem, root.to_owned(), None).await; + assert_eq!(arns(whole.clone()), in_listing_order(&all)); + for row in &whole { + assert_eq!( + Some(row), + s.engine + .read_backup_row(&arn_of(row)) + .await + .expect("read") + .as_ref(), + "the listing returns rows whole" + ); + } + for status in BackupStatus::ALL { + let of_status: Vec<&BackupRow> = + all.iter().copied().filter(|r| r.status == status).collect(); + assert_eq!( + arns(listed(StoreKind::Filesystem, root.to_owned(), Some(status)).await), + in_listing_order(&of_status), + "{status}" + ); + } + for (kind, label, row) in [ + (StoreKind::Filesystem, slash.as_str(), &others[0]), + (StoreKind::Filesystem, "/srv/backups-ol", &others[1]), + (StoreKind::S3, root, &others[2]), + (StoreKind::S3, s3, &others[3]), + (StoreKind::S3, s3_endpoint, &others[4]), + ] { + assert_eq!( + arns(listed(kind, label.to_owned(), None).await), + std::slice::from_ref(&row.backup_arn), + "{kind} {label}" + ); + } + assert!( + listed(StoreKind::Filesystem, "/srv/backups".to_owned(), None) + .await + .is_empty() + ); + drop(claim); + s.cleanup().await; + } + + /// `rewrite_store`: of the keys given, the rows still of the old kind + /// and root and not CREATING get the new store, with every other column + /// left, and are counted; a CREATING row, a row of another root, a row + /// of the other kind at the same label and a key with no row are left + /// and not counted; a second call counts 0; a row another writer moved + /// to a third root before the call is left there. + #[tokio::test] + async fn rewrite_store_moves_only_the_old_roots_rows_that_are_not_creating() { + if base_conn().is_none() { + eprintln!( + "SKIP rewrite_store_moves_only_the_old_roots_rows_that_are_not_creating: no PostgreSQL" + ); + return; + } + let s = scratch().await; + let old = (StoreKind::Filesystem, "/srv/backups-old", "old-1"); + let to = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/srv/backups-new".to_owned(), + store_id: "new-1".to_owned(), + }; + let mut movable = Vec::new(); + for (i, status) in [ + BackupStatus::Available, + BackupStatus::Deleting, + BackupStatus::Deleted, + BackupStatus::Failed, + ] + .into_iter() + .enumerate() + { + movable.push(row_at("Music", &format!("m{i}"), status, old)); + } + let moved_away = row_at("Music", "away", BackupStatus::Available, old); + let other_root = row_at( + "Music", + "other-root", + BackupStatus::Available, + (StoreKind::Filesystem, "/srv/backups-other", "old-1"), + ); + let other_kind = row_at( + "Music", + "other-kind", + BackupStatus::Available, + (StoreKind::S3, "/srv/backups-old", "old-1"), + ); + for row in movable + .iter() + .chain([&moved_away, &other_root, &other_kind]) + { + assert!(s.engine.insert_backup_row(row).await.expect("insert")); + } + let creating = row_at("Music", "job", BackupStatus::Creating, old); + let claim = s.engine.claim_backup(&creating).await.expect("claim"); + let stored = |row: &BackupRow| { + let engine = &s.engine; + let arn = arn_of(row); + async move { + engine + .read_backup_row(&arn) + .await + .expect("read") + .expect("the row exists") + } + }; + let mut before = Vec::new(); + for row in &movable { + before.push(stored(row).await); + } + // Another writer moves one row to a third root first. + let third = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/srv/backups-third".to_owned(), + store_id: "third-1".to_owned(), + }; + assert_eq!( + s.engine + .rewrite_store(old.0, old.1, &[moved_away.key()], &third) + .await + .expect("rewrite"), + 1 + ); + let mut ghost = row_at("Music", "ghost", BackupStatus::Available, old).key(); + ghost.backup_id = extenddb_storage::util::new_backup_id(); + let keys: Vec = movable + .iter() + .chain([&creating, &moved_away, &other_root, &other_kind]) + .map(BackupRow::key) + .chain([ghost]) + .collect(); + assert_eq!( + s.engine + .rewrite_store(old.0, old.1, &keys, &to) + .await + .expect("rewrite"), + 4, + "the four rows of the old root that are not CREATING" + ); + for (row, before) in movable.iter().zip(&before) { + assert_eq!( + stored(row).await, + BackupRow { + store: to.clone(), + ..before.clone() + }, + "only the store changed" + ); + } + let creating_now = stored(&creating).await; + assert_eq!( + (creating_now.status, creating_now.store.store_root.as_str()), + (BackupStatus::Creating, old.1) + ); + assert_eq!(creating_now.owner_generation, claim.generation()); + assert_eq!(stored(&moved_away).await.store, third); + assert_eq!(stored(&other_root).await.store, other_root.store); + assert_eq!(stored(&other_kind).await.store, other_kind.store); + assert_eq!( + s.engine + .rewrite_store(old.0, old.1, &keys, &to) + .await + .expect("rewrite"), + 0, + "a second call finds nothing left to move" + ); + let listed: Vec = s + .engine + .list_backup_rows(None, &to) + .try_collect() + .await + .expect("listing"); + let movable_refs: Vec<&BackupRow> = movable.iter().collect(); + assert_eq!( + listed.into_iter().map(|r| r.backup_arn).collect::>(), + in_listing_order(&movable_refs) + ); + assert_eq!( + s.engine.read_store_ids(old.0, old.1).await.expect("ids"), + ["old-1"], + "the CREATING row keeps the old root" + ); + assert_eq!( + s.engine + .rewrite_store(old.0, old.1, &[], &to) + .await + .expect("rewrite"), + 0 + ); + drop(claim); + s.cleanup().await; + } + + /// The store root of each row of `rows`, as the catalog holds it now. + async fn roots_of(s: &Scratch, rows: &[&BackupRow]) -> Vec { + let mut roots = Vec::new(); + for row in rows { + let root: String = + sqlx::query_scalar("SELECT store_root FROM backup_index WHERE backup_id = $1") + .bind(&row.backup_id) + .fetch_one(&s.catalog) + .await + .expect("read a row's root"); + roots.push(root); + } + roots + } + + /// How many backends of the scratch catalog run the rewrite's + /// statement, waiting on a lock when `waiting`. + async fn rewrites_running(s: &Scratch, waiting: bool) -> i64 { + sqlx::query_scalar( + "SELECT count(*) FROM pg_stat_activity WHERE datname = $1 \ + AND query LIKE 'UPDATE backup_index SET store_kind%' \ + AND state <> 'idle' AND (NOT $2 OR wait_event_type = 'Lock')", + ) + .bind(&s.db_names[0]) + .bind(waiting) + .fetch_one(&s.admin) + .await + .expect("read pg_stat_activity") + } + + /// Hold `FOR UPDATE` on the `backup_index` row of `row` in a + /// transaction of the scratch catalog's pool, until it ends. + async fn hold_row_lock( + s: &Scratch, + row: &BackupRow, + ) -> sqlx::Transaction<'static, sqlx::Postgres> { + let mut tx = s.catalog.begin().await.expect("begin"); + sqlx::query("SELECT 1 FROM backup_index WHERE backup_id = $1 FOR UPDATE") + .bind(&row.backup_id) + .execute(&mut *tx) + .await + .expect("lock the row"); + tx + } + + /// A rewrite whose client dies while its statement waits on a row lock + /// rewrites nothing, even when the lock is granted after the death: the + /// statement runs in a transaction the client commits, and a dead client + /// commits nothing. The client here is an engine on its own runtime, + /// shut down while its statement waits; the lock is released well + /// within `REWRITE_LOCK_TIMEOUT`, so the statement does get its lock. + #[tokio::test] + async fn a_rewrite_whose_client_dies_mid_statement_rewrites_nothing() { + if base_conn().is_none() { + eprintln!( + "SKIP a_rewrite_whose_client_dies_mid_statement_rewrites_nothing: no PostgreSQL" + ); + return; + } + let s = scratch().await; + let old = (StoreKind::Filesystem, "/srv/backups-old", "old-1"); + let to = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/srv/backups-new".to_owned(), + store_id: "new-1".to_owned(), + }; + let rows = [ + row_at("Music", "free", BackupStatus::Available, old), + row_at("Music", "locked", BackupStatus::Available, old), + ]; + for row in &rows { + assert!(s.engine.insert_backup_row(row).await.expect("insert")); + } + let lock = hold_row_lock(&s, &rows[1]).await; + let (stop, stopped) = std::sync::mpsc::channel::<()>(); + let client = { + let url = s.catalog_url.clone(); + let keys: Vec<_> = rows.iter().map(BackupRow::key).collect(); + let to = to.clone(); + std::thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .enable_all() + .build() + .expect("a runtime"); + runtime.spawn(async move { + let engine = PostgresEngine::new( + &PostgresConfig { + connection_string: url, + pool_size: 2, + max_item_size_bytes: 400_000, + }, + REGION, + ) + .await + .expect("a second engine"); + let _ = engine.rewrite_store(old.0, old.1, &keys, &to).await; + panic!("the rewrite returned while its row was locked"); + }); + stopped.recv().expect("the stop"); + // Every task and connection of the client goes with it. + runtime.shutdown_timeout(std::time::Duration::from_secs(5)); + }) + }; + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); + while rewrites_running(&s, true).await == 0 { + assert!( + std::time::Instant::now() < deadline, + "the rewrite never waited" + ); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + stop.send(()).expect("stop the client"); + client.join().expect("the client thread"); + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + lock.commit().await.expect("release the row lock"); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); + while rewrites_running(&s, false).await > 0 { + assert!( + std::time::Instant::now() < deadline, + "the statement never ended" + ); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + assert_eq!( + roots_of(&s, &rows.iter().collect::>()).await, + [old.1, old.1], + "no key is rewritten" + ); + s.cleanup().await; + } + + /// A rewrite that meets a row lock held past `REWRITE_LOCK_TIMEOUT` + /// rolls back and runs again: a lock released after the first timeout + /// lets the second attempt move every key; a lock held past every + /// attempt is an error after `REWRITE_ATTEMPTS` waits, and no key moves, + /// the unlocked one included. + #[tokio::test] + async fn a_rewrite_that_meets_a_held_row_lock_runs_again_a_bounded_number_of_times() { + use extenddb_storage_postgres::{REWRITE_ATTEMPTS, REWRITE_LOCK_TIMEOUT}; + if base_conn().is_none() { + eprintln!( + "SKIP a_rewrite_that_meets_a_held_row_lock_runs_again_a_bounded_number_of_times: no PostgreSQL" + ); + return; + } + let s = scratch().await; + let old = (StoreKind::Filesystem, "/srv/backups-old", "old-1"); + let to = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/srv/backups-new".to_owned(), + store_id: "new-1".to_owned(), + }; + let rows = [ + row_at("Music", "free", BackupStatus::Available, old), + row_at("Music", "locked", BackupStatus::Available, old), + ]; + for row in &rows { + assert!(s.engine.insert_backup_row(row).await.expect("insert")); + } + let keys: Vec<_> = rows.iter().map(BackupRow::key).collect(); + let refs: Vec<&BackupRow> = rows.iter().collect(); + + let lock = hold_row_lock(&s, &rows[1]).await; + let started = std::time::Instant::now(); + let (rewritten, ()) = + tokio::join!(s.engine.rewrite_store(old.0, old.1, &keys, &to), async { + tokio::time::sleep(REWRITE_LOCK_TIMEOUT + std::time::Duration::from_millis(250)) + .await; + lock.commit().await.expect("release the row lock"); + }); + assert_eq!(rewritten.expect("the second attempt"), 2); + assert!( + started.elapsed() > REWRITE_LOCK_TIMEOUT, + "{:?}", + started.elapsed() + ); + assert_eq!(roots_of(&s, &refs).await, [to.store_root.as_str(); 2]); + + let lock = hold_row_lock(&s, &rows[1]).await; + let started = std::time::Instant::now(); + let refused = tokio::time::timeout( + std::time::Duration::from_secs(60), + s.engine.rewrite_store( + to.store_kind, + &to.store_root, + &keys, + &StoreIdentity { + store_kind: old.0, + store_root: old.1.to_owned(), + store_id: old.2.to_owned(), + }, + ), + ) + .await + .expect("the attempts are bounded") + .expect_err("the lock outlasts every attempt"); + // Three attempts with their two pauses (0.5 s, 1 s) take about 10.5 s; + // no pause would end near 9 s, and a fourth attempt near 15 s. + let took = started.elapsed(); + assert!( + took > REWRITE_LOCK_TIMEOUT * REWRITE_ATTEMPTS + std::time::Duration::from_secs(1), + "{took:?}: the attempts paused between them" + ); + assert!( + took < REWRITE_LOCK_TIMEOUT * REWRITE_ATTEMPTS + std::time::Duration::from_secs(3), + "{took:?}: no more than {REWRITE_ATTEMPTS} attempts" + ); + assert!(refused.to_string().contains("lock timeout"), "{refused}"); + assert_eq!( + roots_of(&s, &refs).await, + [to.store_root.as_str(); 2], + "no key moves" + ); + lock.rollback().await.expect("release the row lock"); + + // The bounds are the transaction's own: set inside it (each read + // there), and the connection the rewrite ran on reads the server's + // defaults once it has committed. + let (rewritten, inside, after) = s + .engine + .rewrite_store_settings_after( + to.store_kind, + &to.store_root, + &keys, + &StoreIdentity { + store_kind: old.0, + store_root: old.1.to_owned(), + store_id: old.2.to_owned(), + }, + ) + .await + .expect("a rewrite with no lock held"); + assert_eq!(rewritten, 2); + assert_eq!( + inside, + [ + format!("{}s", REWRITE_LOCK_TIMEOUT.as_secs()), + format!( + "{}s", + extenddb_storage_postgres::REWRITE_STATEMENT_TIMEOUT.as_secs() + ), + ], + "the rewrite runs under its lock and statement bounds" + ); + let mut defaults = Vec::new(); + for name in ["lock_timeout", "statement_timeout"] { + let value: String = sqlx::query_scalar(&format!("SHOW {name}")) + .fetch_one(&s.catalog) + .await + .expect("SHOW on another session"); + defaults.push(value); + } + assert_ne!( + defaults, inside, + "the server's defaults are not the rewrite's bounds" + ); + assert_eq!( + after.to_vec(), + defaults, + "the rewrite's settings stay with its transaction" + ); + s.cleanup().await; + } + + /// `rewrite_store` over more keys than one batch (`REWRITE_BATCH`) + /// commits each batch in its own transaction: with a row lock on the + /// last key held past every attempt, the call is an error and the first + /// batch's rows are moved, the last row left under the old root; run + /// again once the lock is gone, the call moves that row and counts it + /// alone. + #[tokio::test] + async fn a_rewrite_of_more_keys_than_one_batch_commits_each_batch_on_its_own() { + use extenddb_storage_postgres::REWRITE_BATCH; + use futures::StreamExt as _; + if base_conn().is_none() { + eprintln!( + "SKIP a_rewrite_of_more_keys_than_one_batch_commits_each_batch_on_its_own: no PostgreSQL" + ); + return; + } + let s = scratch().await; + let old = (StoreKind::Filesystem, "/srv/backups-old", "old-1"); + let to = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/srv/backups-new".to_owned(), + store_id: "new-1".to_owned(), + }; + let rows: Vec = (0..=REWRITE_BATCH) + .map(|n| row_at("Music", &format!("b{n}"), BackupStatus::Available, old)) + .collect(); + let inserted: Vec = futures::stream::iter(&rows) + .map(|row| s.engine.insert_backup_row(row)) + .buffer_unordered(16) + .try_collect() + .await + .expect("insert"); + assert!(inserted.iter().all(|new| *new)); + let keys: Vec<_> = rows.iter().map(BackupRow::key).collect(); + let last = rows.last().expect("a row"); + let per_root = || async { + sqlx::query_as::<_, (String, i64)>( + "SELECT store_root, count(*) FROM backup_index GROUP BY store_root \ + ORDER BY store_root", + ) + .fetch_all(&s.catalog) + .await + .expect("count the rows per root") + }; + + let lock = hold_row_lock(&s, last).await; + let refused = s + .engine + .rewrite_store(old.0, old.1, &keys, &to) + .await + .expect_err("the last batch's lock outlasts every attempt"); + assert!(refused.to_string().contains("lock timeout"), "{refused}"); + lock.rollback().await.expect("release the row lock"); + assert_eq!( + per_root().await, + [ + (to.store_root.clone(), i64::try_from(REWRITE_BATCH).unwrap()), + (old.1.to_owned(), 1), + ], + "the first batch committed before the last one failed" + ); + assert_eq!(roots_of(&s, &[last]).await, [old.1]); + + assert_eq!( + s.engine + .rewrite_store(old.0, old.1, &keys, &to) + .await + .expect("rewrite"), + 1, + "only the row left is rewritten" + ); + assert_eq!( + per_root().await, + [(to.store_root.clone(), i64::try_from(rows.len()).unwrap())] + ); + s.cleanup().await; + } + + /// `summarize_store_roots`: one count per kind, root, id and status with + /// a row, in the one order; two ids of one root, two statuses of one id, + /// two roots and the two kinds are each counted apart. + #[tokio::test] + async fn summarize_store_roots_counts_the_rows_of_each_store_and_status() { + use extenddb_storage::backup_catalog::StoreRootCount; + if base_conn().is_none() { + eprintln!( + "SKIP summarize_store_roots_counts_the_rows_of_each_store_and_status: no PostgreSQL" + ); + return; + } + let s = scratch().await; + assert!( + s.engine + .summarize_store_roots() + .await + .expect("summary") + .is_empty() + ); + let a1 = (StoreKind::Filesystem, "/srv/a", "id-1"); + let a2 = (StoreKind::Filesystem, "/srv/a", "id-2"); + let b1 = (StoreKind::Filesystem, "/srv/b", "id-1"); + let s3 = (StoreKind::S3, "s3://bucket/a", "id-1"); + let rows = [ + row_at("Music", "1", BackupStatus::Available, a1), + row_at("Music", "2", BackupStatus::Available, a1), + row_at("Music", "3", BackupStatus::Failed, a1), + row_at("Music", "4", BackupStatus::Available, a2), + row_at("Music", "5", BackupStatus::Deleting, b1), + row_at("Music", "7", BackupStatus::Available, b1), + row_at("Music", "6", BackupStatus::Available, s3), + ]; + for row in &rows { + assert!(s.engine.insert_backup_row(row).await.expect("insert")); + } + let creating = row_at("Music", "job", BackupStatus::Creating, b1); + let claim = s.engine.claim_backup(&creating).await.expect("claim"); + let line = |(kind, root, id): (StoreKind, &str, &str), status, rows| StoreRootCount { + store: StoreIdentity { + store_kind: kind, + store_root: root.to_owned(), + store_id: id.to_owned(), + }, + status, + rows, + }; + assert_eq!( + s.engine.summarize_store_roots().await.expect("summary"), + [ + line(a1, BackupStatus::Available, 2), + line(a1, BackupStatus::Failed, 1), + line(a2, BackupStatus::Available, 1), + line(b1, BackupStatus::Creating, 1), + line(b1, BackupStatus::Available, 1), + line(b1, BackupStatus::Deleting, 1), + line(s3, BackupStatus::Available, 1), + ] + ); + drop(claim); + s.cleanup().await; + } + /// `insert_backup_row`: the row lands as given, unowned, with /// `completed_at` and `heartbeat_at` on the database's clock; a second /// insert, an insert over a CREATING row a claim holds, and an insert of @@ -4734,7 +5534,7 @@ mod catalog_index { let (_tx, mut interrupt) = Interrupt::new(); let started = std::time::Instant::now(); assert_eq!( - wait_for_quiet(&s.engine, &store(), &held, wait, &mut interrupt, announce).await, + wait_for_quiet(&s.engine, &[store()], &held, wait, &mut interrupt, announce).await, WaitOutcome::Quiet, "a FAILED restore record is the reconciler's duty, not a running job" ); @@ -4774,7 +5574,7 @@ mod catalog_index { failed.backup_arn )]; assert_eq!( - wait_for_quiet(&s.engine, &store(), &held, wait, &mut interrupt, announce).await, + wait_for_quiet(&s.engine, &[store()], &held, wait, &mut interrupt, announce).await, WaitOutcome::Busy(expected.clone()), "a FAILED row with a live heartbeat holds the wait and is named" ); @@ -4790,7 +5590,7 @@ mod catalog_index { .await .expect("age"); assert_eq!( - wait_for_quiet(&s.engine, &store(), &held, wait, &mut interrupt, announce).await, + wait_for_quiet(&s.engine, &[store()], &held, wait, &mut interrupt, announce).await, WaitOutcome::Quiet, "a FAILED row whose heartbeat has aged is not waited on" ); diff --git a/crates/storage-sqlite/src/backup.rs b/crates/storage-sqlite/src/backup.rs index 699023000..c96556f12 100644 --- a/crates/storage-sqlite/src/backup.rs +++ b/crates/storage-sqlite/src/backup.rs @@ -4831,7 +4831,10 @@ mod restore_tests { Err(StorageError::Validation(m)) if m.contains("Backup not found") )); assert!( - !engine.remove_backup_row(&key).await.unwrap(), + !engine + .remove_backup_row(&key, row.store.store_kind, &row.store.store_root) + .await + .unwrap(), "DELETING is not a tombstone" ); assert!(engine.set_backup_deleted(&key).await.unwrap()); @@ -4843,8 +4846,82 @@ mod restore_tests { .unwrap(); assert_eq!(stored.status, BackupStatus::Deleted); assert!(stored.completed_at.is_some()); - assert!(engine.remove_backup_row(&key).await.unwrap()); - assert!(!engine.remove_backup_row(&key).await.unwrap()); + // A row another writer moved to another store after the caller read it + // is left by a removal that names the store read. + let moved_to = StoreIdentity { + store_kind: row.store.store_kind, + store_root: format!("{}-moved", row.store.store_root), + store_id: "moved-1".to_owned(), + }; + assert_eq!( + engine + .rewrite_store( + row.store.store_kind, + &row.store.store_root, + std::slice::from_ref(&key), + &moved_to + ) + .await + .unwrap(), + 1 + ); + assert!( + !engine + .remove_backup_row(&key, row.store.store_kind, &row.store.store_root) + .await + .unwrap(), + "a row moved to another store is left" + ); + assert_eq!( + engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .map(|r| (r.status, r.store)), + Some((BackupStatus::Deleted, moved_to.clone())) + ); + // The same root under the other kind is another store: a removal that + // names the kind it read leaves the row. + let other_kind = StoreIdentity { + store_kind: if moved_to.store_kind == StoreKind::S3 { + StoreKind::Filesystem + } else { + StoreKind::S3 + }, + ..moved_to.clone() + }; + assert_eq!( + engine + .rewrite_store( + moved_to.store_kind, + &moved_to.store_root, + std::slice::from_ref(&key), + &other_kind + ) + .await + .unwrap(), + 1 + ); + assert!( + !engine + .remove_backup_row(&key, moved_to.store_kind, &moved_to.store_root) + .await + .unwrap(), + "a row moved to the other kind is left" + ); + let moved_to = other_kind; + assert!( + engine + .remove_backup_row(&key, moved_to.store_kind, &moved_to.store_root) + .await + .unwrap() + ); + assert!( + !engine + .remove_backup_row(&key, moved_to.store_kind, &moved_to.store_root) + .await + .unwrap() + ); assert!( engine .read_backup_row(&arn_of(&row)) @@ -5260,6 +5337,427 @@ mod restore_tests { ); } + /// A row of `row(ACCOUNT, table, name)` with `status`, under the + /// store of `kind`, `root` and `id`. + fn row_at( + table: &str, + name: &str, + status: BackupStatus, + (kind, root, id): (StoreKind, &str, &str), + ) -> BackupRow { + let mut row = row(ACCOUNT, table, name); + row.status = status; + row.store = StoreIdentity { + store_kind: kind, + store_root: root.to_owned(), + store_id: id.to_owned(), + }; + row.size_bytes = 10; + row.item_count = 2; + row + } + + /// The ARNs of `rows` in the listings' (`created_at`, `backup_arn`) + /// order. + fn in_listing_order(rows: &[&BackupRow]) -> Vec { + let mut rows = rows.to_vec(); + rows.sort_by(|a, b| { + a.created_at + .total_cmp(&b.created_at) + .then_with(|| a.backup_arn.cmp(&b.backup_arn)) + }); + rows.into_iter().map(|r| r.backup_arn.clone()).collect() + } + + /// `list_backup_rows_by_root`: every row of the kind and root, + /// whatever its id, of every status or of one, whole and in the + /// listings' order; nothing of a root that differs by one byte (a + /// trailing `/`, one character fewer), of an S3 root that differs by + /// its `?endpoint=` suffix, or of the other kind at the same label. + #[tokio::test] + async fn list_backup_rows_by_root_returns_every_id_and_status_of_the_root_and_nothing_else() + { + let engine = engine().await; + let root = "/srv/backups-old"; + let at = |id| (StoreKind::Filesystem, root, id); + let mut of_root = Vec::new(); + for (i, status) in [ + BackupStatus::Available, + BackupStatus::Deleting, + BackupStatus::Deleted, + BackupStatus::Failed, + ] + .into_iter() + .enumerate() + { + let id = if i % 2 == 0 { "store-1" } else { "store-2" }; + of_root.push(row_at("Music", &format!("r{i}"), status, at(id))); + } + // Inserted newest first: the order asserted is the listing's. + for row in of_root.iter().rev() { + assert!(engine.insert_backup_row(row).await.expect("insert")); + } + let creating = row_at("Music", "job", BackupStatus::Creating, at("store-1")); + let _claim = engine.claim_backup(&creating).await.expect("claim"); + let slash = format!("{root}/"); + let s3 = "s3://bucket/prefix"; + let s3_endpoint = "s3://bucket/prefix?endpoint=http://127.0.0.1:9200"; + let others = [ + row_at( + "Music", + "slash", + BackupStatus::Available, + (StoreKind::Filesystem, &slash, "store-1"), + ), + row_at( + "Music", + "shorter", + BackupStatus::Available, + (StoreKind::Filesystem, "/srv/backups-ol", "store-1"), + ), + row_at( + "Music", + "kind", + BackupStatus::Available, + (StoreKind::S3, root, "store-1"), + ), + row_at( + "Music", + "s3", + BackupStatus::Available, + (StoreKind::S3, s3, "store-3"), + ), + row_at( + "Music", + "endpoint", + BackupStatus::Available, + (StoreKind::S3, s3_endpoint, "store-3"), + ), + ]; + for row in &others { + assert!(engine.insert_backup_row(row).await.expect("insert")); + } + let listed = |kind: StoreKind, root: String, status: Option| { + let engine = &engine; + async move { + let rows: Vec = engine + .list_backup_rows_by_root(kind, &root, status) + .try_collect() + .await + .expect("listing"); + rows + } + }; + let arns = |rows: Vec| -> Vec { + rows.into_iter().map(|r| r.backup_arn).collect() + }; + let mut all: Vec<&BackupRow> = of_root.iter().collect(); + all.push(&creating); + let whole = listed(StoreKind::Filesystem, root.to_owned(), None).await; + assert_eq!(arns(whole.clone()), in_listing_order(&all)); + for row in &whole { + assert_eq!( + Some(row), + engine + .read_backup_row(&arn_of(row)) + .await + .expect("read") + .as_ref(), + "the listing returns rows whole" + ); + } + for status in BackupStatus::ALL { + let of_status: Vec<&BackupRow> = + all.iter().copied().filter(|r| r.status == status).collect(); + assert_eq!( + arns(listed(StoreKind::Filesystem, root.to_owned(), Some(status)).await), + in_listing_order(&of_status), + "{status}" + ); + } + for (kind, label, row) in [ + (StoreKind::Filesystem, slash.as_str(), &others[0]), + (StoreKind::Filesystem, "/srv/backups-ol", &others[1]), + (StoreKind::S3, root, &others[2]), + (StoreKind::S3, s3, &others[3]), + (StoreKind::S3, s3_endpoint, &others[4]), + ] { + assert_eq!( + arns(listed(kind, label.to_owned(), None).await), + std::slice::from_ref(&row.backup_arn), + "{kind} {label}" + ); + } + assert!( + listed(StoreKind::Filesystem, "/srv/backups".to_owned(), None) + .await + .is_empty() + ); + } + + /// `rewrite_store`: of the keys given, the rows still of the old kind + /// and root and not CREATING get the new store, with every other + /// column left, and are counted; a CREATING row, a row of another + /// root, a row of the other kind at the same label and a key with no + /// row are left and not counted; a second call counts 0; a row + /// another writer moved to a third root before the call is left + /// there. + #[tokio::test] + async fn rewrite_store_moves_only_the_old_roots_rows_that_are_not_creating() { + let engine = engine().await; + let old = (StoreKind::Filesystem, "/srv/backups-old", "old-1"); + let to = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/srv/backups-new".to_owned(), + store_id: "new-1".to_owned(), + }; + let mut movable = Vec::new(); + for (i, status) in [ + BackupStatus::Available, + BackupStatus::Deleting, + BackupStatus::Deleted, + BackupStatus::Failed, + ] + .into_iter() + .enumerate() + { + movable.push(row_at("Music", &format!("m{i}"), status, old)); + } + let moved_away = row_at("Music", "away", BackupStatus::Available, old); + let other_root = row_at( + "Music", + "other-root", + BackupStatus::Available, + (StoreKind::Filesystem, "/srv/backups-other", "old-1"), + ); + let other_kind = row_at( + "Music", + "other-kind", + BackupStatus::Available, + (StoreKind::S3, "/srv/backups-old", "old-1"), + ); + for row in movable + .iter() + .chain([&moved_away, &other_root, &other_kind]) + { + assert!(engine.insert_backup_row(row).await.expect("insert")); + } + let creating = row_at("Music", "job", BackupStatus::Creating, old); + let claim = engine.claim_backup(&creating).await.expect("claim"); + let stored = |row: &BackupRow| { + let engine = &engine; + let arn = arn_of(row); + async move { + engine + .read_backup_row(&arn) + .await + .expect("read") + .expect("the row exists") + } + }; + let mut before = Vec::new(); + for row in &movable { + before.push(stored(row).await); + } + // Another writer moves one row to a third root first. + let third = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/srv/backups-third".to_owned(), + store_id: "third-1".to_owned(), + }; + assert_eq!( + engine + .rewrite_store(old.0, old.1, &[moved_away.key()], &third) + .await + .expect("rewrite"), + 1 + ); + let mut ghost = row_at("Music", "ghost", BackupStatus::Available, old).key(); + ghost.backup_id = extenddb_storage::util::new_backup_id(); + let keys: Vec = movable + .iter() + .chain([&creating, &moved_away, &other_root, &other_kind]) + .map(BackupRow::key) + .chain([ghost]) + .collect(); + assert_eq!( + engine + .rewrite_store(old.0, old.1, &keys, &to) + .await + .expect("rewrite"), + 4, + "the four rows of the old root that are not CREATING" + ); + for (row, before) in movable.iter().zip(&before) { + assert_eq!( + stored(row).await, + BackupRow { + store: to.clone(), + ..before.clone() + }, + "only the store changed" + ); + } + let creating_now = stored(&creating).await; + assert_eq!( + (creating_now.status, creating_now.store.store_root.as_str()), + (BackupStatus::Creating, old.1) + ); + assert_eq!(creating_now.owner_generation, claim.generation()); + assert_eq!(stored(&moved_away).await.store, third); + assert_eq!(stored(&other_root).await.store, other_root.store); + assert_eq!(stored(&other_kind).await.store, other_kind.store); + assert_eq!( + engine + .rewrite_store(old.0, old.1, &keys, &to) + .await + .expect("rewrite"), + 0, + "a second call finds nothing left to move" + ); + let listed: Vec = engine + .list_backup_rows(None, &to) + .try_collect() + .await + .expect("listing"); + let movable_refs: Vec<&BackupRow> = movable.iter().collect(); + assert_eq!( + listed.into_iter().map(|r| r.backup_arn).collect::>(), + in_listing_order(&movable_refs) + ); + assert_eq!( + engine.read_store_ids(old.0, old.1).await.expect("ids"), + ["old-1"], + "the CREATING row keeps the old root" + ); + assert_eq!( + engine + .rewrite_store(old.0, old.1, &[], &to) + .await + .expect("rewrite"), + 0 + ); + } + + /// `rewrite_store` moves every key or none: a statement that fails + /// on a later key (a trigger here raises on the last one) rolls back + /// the keys already rewritten in the same call; once the trigger is + /// gone the same call moves them all. + #[tokio::test] + async fn a_rewrite_that_fails_on_a_later_key_moves_no_row() { + let engine = engine().await; + let old = (StoreKind::Filesystem, "/srv/backups-old", "old-1"); + let to = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/srv/backups-new".to_owned(), + store_id: "new-1".to_owned(), + }; + let rows: Vec = ["a", "b", "c"] + .into_iter() + .map(|name| row_at("Music", name, BackupStatus::Available, old)) + .collect(); + for row in &rows { + assert!(engine.insert_backup_row(row).await.expect("insert")); + } + sqlx::query(&format!( + "CREATE TRIGGER refuse_the_last_key BEFORE UPDATE OF store_root ON backup_index \ + WHEN OLD.backup_id = '{}' BEGIN SELECT RAISE(ABORT, 'refused'); END", + rows[2].backup_id + )) + .execute(&engine.pool) + .await + .expect("create the trigger"); + let keys: Vec<_> = rows.iter().map(BackupRow::key).collect(); + let refused = engine + .rewrite_store(old.0, old.1, &keys, &to) + .await + .expect_err("the last key's statement fails"); + assert!(refused.to_string().contains("refused"), "{refused}"); + let roots = || async { + let mut roots = Vec::new(); + for row in &rows { + roots.push( + engine + .read_backup_row(&arn_of(row)) + .await + .expect("read") + .expect("the row exists") + .store + .store_root, + ); + } + roots + }; + assert_eq!(roots().await, [old.1; 3], "no row moved"); + sqlx::query("DROP TRIGGER refuse_the_last_key") + .execute(&engine.pool) + .await + .expect("drop the trigger"); + assert_eq!( + engine + .rewrite_store(old.0, old.1, &keys, &to) + .await + .expect("rewrite"), + 3 + ); + assert_eq!(roots().await, [to.store_root.as_str(); 3]); + } + + /// `summarize_store_roots`: one count per kind, root, id and status + /// with a row, in the one order; two ids of one root, two statuses of + /// one id, two roots and the two kinds are each counted apart. + #[tokio::test] + async fn summarize_store_roots_counts_the_rows_of_each_store_and_status() { + use extenddb_storage::backup_catalog::StoreRootCount; + let engine = engine().await; + assert!( + engine + .summarize_store_roots() + .await + .expect("summary") + .is_empty() + ); + let a1 = (StoreKind::Filesystem, "/srv/a", "id-1"); + let a2 = (StoreKind::Filesystem, "/srv/a", "id-2"); + let b1 = (StoreKind::Filesystem, "/srv/b", "id-1"); + let s3 = (StoreKind::S3, "s3://bucket/a", "id-1"); + let rows = [ + row_at("Music", "1", BackupStatus::Available, a1), + row_at("Music", "2", BackupStatus::Available, a1), + row_at("Music", "3", BackupStatus::Failed, a1), + row_at("Music", "4", BackupStatus::Available, a2), + row_at("Music", "5", BackupStatus::Deleting, b1), + row_at("Music", "7", BackupStatus::Available, b1), + row_at("Music", "6", BackupStatus::Available, s3), + ]; + for row in &rows { + assert!(engine.insert_backup_row(row).await.expect("insert")); + } + let creating = row_at("Music", "job", BackupStatus::Creating, b1); + let _claim = engine.claim_backup(&creating).await.expect("claim"); + let line = |(kind, root, id): (StoreKind, &str, &str), status, rows| StoreRootCount { + store: StoreIdentity { + store_kind: kind, + store_root: root.to_owned(), + store_id: id.to_owned(), + }, + status, + rows, + }; + assert_eq!( + engine.summarize_store_roots().await.expect("summary"), + [ + line(a1, BackupStatus::Available, 2), + line(a1, BackupStatus::Failed, 1), + line(a2, BackupStatus::Available, 1), + line(b1, BackupStatus::Creating, 1), + line(b1, BackupStatus::Available, 1), + line(b1, BackupStatus::Deleting, 1), + line(s3, BackupStatus::Available, 1), + ] + ); + } + /// `insert_backup_row`: the row lands as given, unowned, with /// `completed_at` and `heartbeat_at` on the database's clock; a /// second insert, an insert over a CREATING row a claim holds, and an diff --git a/crates/storage-sqlite/src/backup_catalog.rs b/crates/storage-sqlite/src/backup_catalog.rs index 7e492387f..a037d0175 100644 --- a/crates/storage-sqlite/src/backup_catalog.rs +++ b/crates/storage-sqlite/src/backup_catalog.rs @@ -54,7 +54,7 @@ use extenddb_storage::backup_catalog::{ BackupCatalog, BackupKey, BackupRow, BackupStatus, DeleteStart, ForgetRow, Heartbeat, JobClaim, JobRow, JobStatus, MaintenanceClaim, RestoreJobRow, RestoreKind, RestoreRegistered, RestoreRegistrationRow, RestoreTargetKey, StaleClock, StatusFields, StoreIdentity, StoreKind, - TimeBounds, UNOWNED_GENERATION, WriteOutcome, new_generation, + StoreRootCount, TimeBounds, UNOWNED_GENERATION, WriteOutcome, new_generation, }; use extenddb_storage::error::StorageError; use futures::future::BoxFuture; @@ -898,6 +898,29 @@ impl BackupCatalog for SqliteEngine { }) } + fn list_backup_rows_by_root<'a>( + &'a self, + store_kind: StoreKind, + store_root: &'a str, + status: Option, + ) -> BoxStream<'a, Result> { + // `=` on TEXT compares bytes (the BINARY collation the columns + // carry), so a root matches as stored. + rows_stream(async move { + sqlx::query_as(&format!( + "SELECT {ROW_COLUMNS} FROM backup_index \ + WHERE store_kind = ?1 AND store_root = ?2 \ + AND (?3 IS NULL OR status = ?3) ORDER BY created_at, backup_arn" + )) + .bind(store_kind.as_str()) + .bind(store_root) + .bind(status.map(BackupStatus::as_str)) + .fetch_all(&self.pool) + .await + .map_err(db_err) + }) + } + fn newest_available_backup_row<'a>( &'a self, store: &'a StoreIdentity, @@ -984,6 +1007,48 @@ impl BackupCatalog for SqliteEngine { }) } + fn rewrite_store<'a>( + &'a self, + from_kind: StoreKind, + from_root: &'a str, + keys: &'a [BackupKey], + to: &'a StoreIdentity, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + if keys.is_empty() { + return Ok(0); + } + // One transaction under the writer lock: every key or none. + let _writer = self.write_lock.lock().await; + let mut tx = self + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(db_err)?; + let mut rewritten = 0; + for key in keys { + rewritten += sqlx::query( + "UPDATE backup_index SET store_kind = ?, store_root = ?, store_id = ? \ + WHERE account_id = ? AND backup_id = ? AND store_kind = ? \ + AND store_root = ? AND status <> 'CREATING'", + ) + .bind(to.store_kind.as_str()) + .bind(&to.store_root) + .bind(&to.store_id) + .bind(&key.account_id) + .bind(&key.backup_id) + .bind(from_kind.as_str()) + .bind(from_root) + .execute(&mut *tx) + .await + .map_err(db_err)? + .rows_affected(); + } + tx.commit().await.map_err(db_err)?; + Ok(rewritten) + }) + } + fn set_backup_deleted<'a>( &'a self, key: &'a BackupKey, @@ -1007,15 +1072,19 @@ impl BackupCatalog for SqliteEngine { fn remove_backup_row<'a>( &'a self, key: &'a BackupKey, + store_kind: StoreKind, + store_root: &'a str, ) -> BoxFuture<'a, Result> { Box::pin(async move { let _writer = self.write_lock.lock().await; let affected = sqlx::query( "DELETE FROM backup_index WHERE account_id = ? AND backup_id = ? \ - AND status IN ('DELETED', 'FAILED')", + AND status IN ('DELETED', 'FAILED') AND store_kind = ? AND store_root = ?", ) .bind(&key.account_id) .bind(&key.backup_id) + .bind(store_kind.as_str()) + .bind(store_root) .execute(&self.pool) .await .map_err(db_err)? @@ -1463,6 +1532,26 @@ impl BackupCatalog for SqliteEngine { }) } + fn summarize_store_roots(&self) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + let grouped: Vec<(String, String, String, String, i64)> = sqlx::query_as( + "SELECT store_kind, store_root, store_id, status, count(*) FROM backup_index \ + GROUP BY store_kind, store_root, store_id, status", + ) + .fetch_all(&self.pool) + .await + .map_err(db_err)?; + let mut counts = grouped + .into_iter() + .map(|(kind, root, id, status, rows)| { + StoreRootCount::from_columns(&kind, root, id, &status, rows) + }) + .collect::, _>>()?; + StoreRootCount::sort(&mut counts); + Ok(counts) + }) + } + fn maintenance_claim<'a>( &'a self, holder: &'a str, diff --git a/crates/storage/src/backup_catalog.rs b/crates/storage/src/backup_catalog.rs index 7a8a354b6..6cb790f95 100644 --- a/crates/storage/src/backup_catalog.rs +++ b/crates/storage/src/backup_catalog.rs @@ -153,6 +153,95 @@ pub struct StoreIdentity { pub store_id: String, } +/// What [`BackupCatalog::remove_orphan_status_entries`] did with a root's +/// status entries. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct StatusEntrySweep { + /// Entries deleted: their row is gone or names another store. + pub removed: u64, + /// Entries whose row is of the root under another status, rewritten + /// under the row's status (the stale entry deleted). + pub repaired: u64, +} + +/// One line of [`BackupCatalog::summarize_store_roots`]: the catalog holds +/// `rows` rows of `status` under `store`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StoreRootCount { + pub store: StoreIdentity, + pub status: BackupStatus, + pub rows: u64, +} + +impl StoreRootCount { + /// A count as a grouping query returns it: the `store_kind`, + /// `store_root`, `store_id` and `status` columns and the count. + /// + /// # Errors + /// + /// `Internal` for a kind or status no release writes. + pub fn from_columns( + store_kind: &str, + store_root: String, + store_id: String, + status: &str, + rows: i64, + ) -> Result { + let kind = StoreKind::parse(store_kind).ok_or_else(|| { + StorageError::Internal(format!( + "backup rows of root {store_root:?} carry an unknown store kind {store_kind:?}" + )) + })?; + Ok(Self { + store: StoreIdentity { + store_kind: kind, + store_root, + store_id, + }, + status: BackupStatus::parse(status)?, + rows: u64::try_from(rows).unwrap_or_default(), + }) + } + + /// The order [`BackupCatalog::summarize_store_roots`] returns on every + /// backend: by kind, root and id, compared as bytes, then by status in + /// [`BackupStatus::ALL`]'s order, whatever the database's collation. + pub fn sort(counts: &mut [Self]) { + counts.sort_by(|a, b| { + fn key(c: &StoreRootCount) -> (&str, &[u8], &[u8], Option) { + ( + c.store.store_kind.as_str(), + c.store.store_root.as_bytes(), + c.store.store_id.as_bytes(), + BackupStatus::ALL.iter().position(|s| *s == c.status), + ) + } + key(a).cmp(&key(b)) + }); + } + + /// One count per (store, status) of `rows`, sorted: the tally a backend + /// that cannot group in its query (Cassandra) makes of what it read. + #[must_use] + pub fn tally(rows: impl IntoIterator) -> Vec { + let mut counts: std::collections::HashMap<(StoreIdentity, BackupStatus), u64> = + std::collections::HashMap::new(); + for key in rows { + *counts.entry(key).or_default() += 1; + } + let mut counts: Vec = counts + .into_iter() + .map(|((store, status), rows)| Self { + store, + status, + rows, + }) + .collect(); + Self::sort(&mut counts); + counts + } +} + /// The status column of a backup row. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum BackupStatus { @@ -164,6 +253,16 @@ pub enum BackupStatus { } impl BackupStatus { + /// Every status, in the order a row moves through them (CREATING first, + /// FAILED last): the order a per-status listing or report uses. + pub const ALL: [Self; 5] = [ + Self::Creating, + Self::Available, + Self::Deleting, + Self::Deleted, + Self::Failed, + ]; + /// The value stored in the `status` column. #[must_use] pub const fn as_str(self) -> &'static str { @@ -945,6 +1044,29 @@ pub trait BackupCatalog: Send + Sync { store: &'a StoreIdentity, ) -> BoxStream<'a, Result>; + /// Every row whose `store_kind` and `store_root` equal these, whatever + /// its `store_id`, with `status` or of every status for `None`, in the + /// (`created_at`, `backup_arn`) order [`Self::list_backup_rows`] uses: + /// `backup sync --move-store` and `--forget-store`, which get a root and + /// no id, and the report of the rows of other roots. `store_root` + /// matches as stored, byte for byte: a root that differs by a trailing + /// `/`, or an S3 root that differs by its `?endpoint=` suffix, is + /// another root. The rows are read whole, as `list_backup_rows` reads + /// them. On Cassandra the root's five status partitions are read + /// whatever `status` is, each row behind an entry is re-read once, and + /// it is kept when its kind and root still match (and its status, with + /// `Some(status)`), whatever partition its entry is in: a row whose + /// entry is under an earlier status (a status change whose entry batch + /// failed, or whose writer died between the row and the batch) is + /// listed under its own status. A row with no status entry at all (an + /// insert that stopped before its entries) is not listed. + fn list_backup_rows_by_root<'a>( + &'a self, + store_kind: StoreKind, + store_root: &'a str, + status: Option, + ) -> BoxStream<'a, Result>; + /// The newest AVAILABLE row of `store` (the last in the /// (`created_at`, `backup_arn`) order [`Self::list_backup_rows`] lists /// in), or `None` when the store has none. The store's manifest probe @@ -985,23 +1107,48 @@ pub trait BackupCatalog: Send + Sync { ) -> BoxFuture<'a, Result>; /// DELETING to DELETED, conditional on the row being DELETING; `false` - /// when it was not. + /// when it was not. Besides the deletion's last step, `backup sync + /// --forget-store` and `--forget-backup` remove a DELETING row with this + /// write and then [`Self::remove_backup_row`], under the maintenance + /// claim. No DELETED-prefix or tombstone duty acts on the row between + /// the two, by the order of the listings: a duty that listed its rows + /// before the claim was taken listed this one DELETING, not DELETED, and + /// a duty that lists it DELETED did so after the claim was taken, and + /// reads the claim held after its re-read of the row. A DELETING row of + /// the configured store has its deletion finished first, by the forget + /// itself or by a reconciler under the claim: its bytes were asked to be + /// deleted. A DELETING row of another store keeps its bytes unless a + /// server configured with that store finishes the deletion. fn set_backup_deleted<'a>( &'a self, key: &'a BackupKey, ) -> BoxFuture<'a, Result>; - /// Remove a DELETED or FAILED tombstone; `false` when the row is not one. + /// Remove a DELETED or FAILED tombstone that is still of `store_kind` + /// and `store_root`, the store the caller read the row with; `false` + /// when the row is not one, or names another store. The tombstone + /// duty's write, and `backup sync --forget-store`'s and + /// `--forget-backup`'s for a DELETED or FAILED row (and for a DELETING + /// row after [`Self::set_backup_deleted`]). The store condition keeps a + /// duty of a server still configured with an old root from removing a + /// row `rewrite_store` moved after the duty read it: the configured + /// store's own duties then finish that row (on Cassandra the lightweight + /// transaction carries the condition, and a `NotMatched` answers + /// `false`). fn remove_backup_row<'a>( &'a self, key: &'a BackupKey, + store_kind: StoreKind, + store_root: &'a str, ) -> BoxFuture<'a, Result>; /// Remove an AVAILABLE row outright, leaving no DELETING or DELETED /// tombstone, so no deletion duty ever acts on the backup's objects: /// `backup sync --prune-missing`'s write for a row whose manifest the /// configured root does not hold, whose bytes may be under another - /// host's root of the same path. Under the row's lock, as + /// host's root of the same path, and `--forget-backup`'s and + /// `--forget-store`'s for an AVAILABLE row, whose bytes the operator + /// keeps. Under the row's lock, as /// [`Self::start_backup_delete`] takes it, the status check and the /// in-use rule in one step with the removal: a row a CREATING restore /// target reads is left ([`ForgetRow::InUse`]), and so is a row another @@ -1015,12 +1162,90 @@ pub trait BackupCatalog: Send + Sync { key: &'a BackupKey, ) -> BoxFuture<'a, Result>; + /// `backup sync --move-store`'s write: each row of `keys` that is still + /// of `from_kind` and `from_root` and is not CREATING gets `to`'s kind, + /// root and id; every other column is left. Returns how many rows were + /// rewritten. A key with no row, or whose row no longer matches (another + /// writer moved it to another root, or it is CREATING: a job may still + /// write its bytes under the old root), is left and not counted, so a + /// second call with the same keys counts 0 once `to` is another root. + /// A DELETING, DELETED or + /// FAILED row is rewritten like an AVAILABLE one. No claim and no fence + /// are taken: no object is touched. + /// + /// SQLite rewrites every key in one transaction, so a failed call + /// rewrites none. PostgreSQL rewrites in batches of keys, each batch one + /// statement in a transaction of its own (so no statement nears its + /// bound whatever the key count), MongoDB in chunks of keys, each chunk + /// one statement, and Cassandra key by key, so on those three a call + /// that stops leaves the rows split between the two roots: every step + /// is idempotent, and the caller re-lists the old root and runs again + /// for what is left. Cassandra keeps no `backup_id` uniqueness, so the + /// caller, not this method, refuses a move that would give an account + /// two rows of one backup id. + /// + /// What a client that dies in the call (a killed command) leaves: + /// - PostgreSQL: the batches before the one in flight, and of that one + /// nothing, unless its commit was already sent. Each batch's + /// statement runs in a transaction the client commits, under a + /// `lock_timeout` (a lock timeout rolls back and runs the batch + /// again, a bounded number of times) and a `statement_timeout`, so a + /// statement still waiting or running when the client dies commits + /// nothing: the server rolls it back once it finds the connection + /// closed, at the latest when those bounds end it, and no rewrite + /// lands after the maintenance claim's advisory lock went with the + /// client. + /// - SQLite: nothing; the transaction is in the process and rolls back + /// with it. + /// - MongoDB: the chunks before the one in flight are rewritten, and + /// the one in flight in whole or in part (each document's update is + /// atomic, and the server may finish the statement after the client + /// is gone). The claim is a heartbeat, live until it is stale, which + /// outlasts the statement. + /// - Cassandra: the keys before the one in flight are rewritten, and + /// the one in flight is left at one of the points the module doc + /// names (its new entry written, or its row moved with the old entry + /// left); a lightweight transaction in flight may still commit. The + /// claim is a heartbeat, as on MongoDB. + /// + /// In each case the re-run of the move re-lists the old root and moves + /// what is left. + /// + /// # Errors + /// + /// The backend's error for a failed read or write. On Cassandra an + /// answer lost in transit is settled by a serial read of the row: a row + /// that carries `to` whole (its kind, root and store id) counts as + /// rewritten, a row of the old root is sent again (up to three attempts + /// in all), a row of any other store (`to`'s root under another store + /// id included) is left and not counted, and an outcome still unknown + /// after the last attempt is an error. + fn rewrite_store<'a>( + &'a self, + from_kind: StoreKind, + from_root: &'a str, + keys: &'a [BackupKey], + to: &'a StoreIdentity, + ) -> BoxFuture<'a, Result>; + + /// Whether [`Self::rewrite_store`] writes key by key, several requests + /// per key with nothing checked between keys (Cassandra), so a caller + /// that must stop between keys (the move, once its maintenance claim + /// reads lost) passes one key per call. `false` by default: the other + /// backends rewrite a call's keys in one statement per batch or chunk. + fn rewrite_store_runs_key_by_key(&self) -> bool { + false + } + /// Remove a CREATING row whose owner is dead: `heartbeat_at` older than /// `stale_after` on the catalog's clock. `false` when the row is not /// CREATING, is gone, or has a live owner. The maintenance command's /// write for a CREATING row its wait saw no live owner for, after it has /// deleted the row's prefix; the row's own job, were it alive, would - /// find its claim lost at its next heartbeat. + /// find its claim lost at its next heartbeat. Also `backup sync + /// --forget-store`'s and `--forget-backup`'s write for a CREATING row, + /// which delete no prefix: `false` with the row still CREATING is a live + /// owner, and they refuse the row. fn remove_dead_creating_backup_row<'a>( &'a self, key: &'a BackupKey, @@ -1254,6 +1479,48 @@ pub trait BackupCatalog: Send + Sync { store_root: &'a str, ) -> BoxFuture<'a, Result, StorageError>>; + /// How many rows the catalog holds per store (kind, root and id) and + /// status, every store and status with at least one row, in + /// [`StoreRootCount::sort`]'s order: plain `backup sync`'s report of the + /// rows of roots other than the configured one, and the roots + /// `--move-store` and `--forget-store` name when given one with no + /// rows. Counts only; no row is read whole. On Cassandra the counts are + /// of the status entries (`backup_index_by_status`, one partition per + /// root and status), not verified against their rows, so an entry a + /// stopped writer left behind counts as [`Self::read_store_ids`] counts + /// it ([`Self::summary_counts_status_entries`]). + fn summarize_store_roots(&self) -> BoxFuture<'_, Result, StorageError>>; + + /// Whether [`Self::summarize_store_roots`] counts status entries rather + /// than rows (Cassandra), so a caller that reports the counts verifies + /// them against the rows' listing. `false` by default: the other + /// backends count rows. + fn summary_counts_status_entries(&self) -> bool { + false + } + + /// The sweep of the status entries under `store_root` whose entry + /// names `store_kind`, each verified against its row as it is read: + /// an entry whose row is gone or names another store is deleted, and + /// one whose row is of the root under another status is written again + /// under the row's status and then deleted (repaired). `backup sync + /// --forget-store`'s sweep of the root it forgets, under the + /// maintenance claim. Only Cassandra keeps status entries beside the + /// rows (`backup_index_by_status`): a deleter whose entry batch lands + /// after a `rewrite_store` of its row leaves an entry with no row of + /// the root behind it, and a status change whose entry batch failed, or + /// whose writer died between its row and its batch, leaves the row's + /// entry under the earlier status. Every other backend has none, which + /// is what the default body says. + fn remove_orphan_status_entries<'a>( + &'a self, + store_kind: StoreKind, + store_root: &'a str, + ) -> BoxFuture<'a, Result> { + let _ = (store_kind, store_root); + Box::pin(async { Ok(StatusEntrySweep::default()) }) + } + /// Take the catalog-level maintenance claim for `holder` (a label naming /// the command and the host it runs on, for the operator who finds it /// held). The claim is one record, `backup_maintenance`, born owned @@ -1845,4 +2112,46 @@ mod tests { has no connection slot left for the job); retry later" ); } + + /// The tally counts each (kind, root, id, status) apart, and the order + /// is the bytes of kind, root and id, then the statuses in the order a + /// row moves through them, not their names' order. + #[test] + fn the_root_tally_counts_each_store_and_status_apart_in_one_order() { + let store = |kind: StoreKind, root: &str, id: &str| StoreIdentity { + store_kind: kind, + store_root: root.to_owned(), + store_id: id.to_owned(), + }; + let a1 = store(StoreKind::Filesystem, "/a", "id-1"); + let a2 = store(StoreKind::Filesystem, "/a", "id-2"); + let lower = store(StoreKind::Filesystem, "/a/b", "id-1"); + let s3 = store(StoreKind::S3, "s3://bucket/p", "id-1"); + let counts = StoreRootCount::tally([ + (s3.clone(), BackupStatus::Available), + (a1.clone(), BackupStatus::Failed), + (a1.clone(), BackupStatus::Available), + (a2.clone(), BackupStatus::Available), + (a1.clone(), BackupStatus::Available), + (lower.clone(), BackupStatus::Creating), + (a1.clone(), BackupStatus::Creating), + ]); + let line = |store: &StoreIdentity, status, rows| StoreRootCount { + store: store.clone(), + status, + rows, + }; + assert_eq!( + counts, + [ + line(&a1, BackupStatus::Creating, 1), + line(&a1, BackupStatus::Available, 2), + line(&a1, BackupStatus::Failed, 1), + line(&a2, BackupStatus::Available, 1), + line(&lower, BackupStatus::Creating, 1), + line(&s3, BackupStatus::Available, 1), + ] + ); + assert!(StoreRootCount::tally([]).is_empty()); + } } diff --git a/docs/adr/0019-cassandra-logical-backup-restore.md b/docs/adr/0019-cassandra-logical-backup-restore.md index 7e247743c..29696813e 100644 --- a/docs/adr/0019-cassandra-logical-backup-restore.md +++ b/docs/adr/0019-cassandra-logical-backup-restore.md @@ -1,6 +1,6 @@ # ADR-0019: Logical backup and restore for Cassandra -**Status:** Accepted +**Status:** Superseded in part by [the backup and restore design](../design/15-backup-restore.md) (section 10) **Date:** 2026-08-03 ## Context diff --git a/docs/adr/README.md b/docs/adr/README.md index 1fe3fe104..82c2f34ea 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -22,6 +22,8 @@ ADRs are *records*, not proposals. If you are still soliciting input, write an 3. Open a PR. Discussion happens inline. 4. On merge, the ADR is Accepted. Subsequent decisions that override it should create a new ADR and update the original's Status to "Superseded by ADR-NNNN". + A decision that a later design document replaces in part updates its Status + to "Superseded in part by [document]", and that document states which parts. ADRs are never edited after acceptance except to update Status. To change a decision, write a new ADR. @@ -45,7 +47,7 @@ decision, write a new ADR. | [0016](0016-cassandra-index-table-primary-key-structure.md) | Cassandra: index table primary-key structure | Accepted | | [0017](0017-cassandra-transaction-implementation.md) | Cassandra: TransactWriteItems / TransactGetItems implementation | Accepted | | [0018](0018-cassandra-stream-implementation.md) | Cassandra: DynamoDB Streams implementation | Accepted | -| [0019](0019-cassandra-logical-backup-restore.md) | Cassandra: logical backup and restore | Accepted | +| [0019](0019-cassandra-logical-backup-restore.md) | Cassandra: logical backup and restore | Superseded in part by [the backup and restore design](../design/15-backup-restore.md) (section 10) | | [0020](0020-cassandra-occ-update-item.md) | Cassandra: optimistic concurrency control for UpdateItem and PutItem | Accepted | | [0021](0021-cassandra-lwt-delete-atomicity-gap.md) | Cassandra: LWT delete atomicity gap and required worker/saga pattern | Accepted | | [0022](0022-trust-policy-deny-over-allow.md) | Role trust-policy evaluation follows IAM Deny-over-Allow order | Accepted | diff --git a/docs/build-docs.py b/docs/build-docs.py index 1a43bae74..41409e13e 100755 --- a/docs/build-docs.py +++ b/docs/build-docs.py @@ -89,6 +89,8 @@ "docs/local-postgres-setup.md"), (18, "event-ticketing-demo", "Event Ticketing Platform Demo", "getting-started", "docs/manuals/13-event-ticketing-demo.md"), + (19, "backup-and-restore", "Backup and Restore", "usage", + "docs/manuals/14-backup-and-restore.md"), ] diff --git a/docs/design/15-backup-restore-bench-pr11.md b/docs/design/15-backup-restore-bench-pr11.md new file mode 100644 index 000000000..dd121a9a0 --- /dev/null +++ b/docs/design/15-backup-restore-bench-pr11.md @@ -0,0 +1,453 @@ +# PR 11 backup and restore record: the open cells (2026-11-05) + +This is the record the testing summary of `15-backup-restore.md` cites for +the cells the PR 8 and PR 9 records left open: a 1M-item restore on MongoDB +and on Cassandra, and the backup and restore of a 1M-item table carrying a +vector index on PostgreSQL and on SQLite. The script (`bench-pr11.py`) and +the raw samples (`/proc//status` every 200 ms per phase, the servers' +logs, one JSON result per run) are kept beside the PR outside the +repository, on the measuring host; the figures below are what they +produced. The earlier records are `15-backup-restore-bench-pr8.md` (the +backup) and `15-backup-restore-bench-pr9.md` (the restore). + +## Method + +The PR 9 record's method. The release binaries (`cargo build --release -p +extenddb` for PostgreSQL, `--no-default-features --features sqlite`, +`mongodb` and `cassandra` for the others), built from this change's first +version, whose Rust sources are those of PR 10 at 358ff8bd (on main +94b5f689), running `serve --foreground` with a filesystem backup store on +local disk. Every other build this record names (30553b38, 04c29155, +f78e9bc1) predates the change's rebase onto main 887b6e26 too, whose 76 +commits since 94b5f689 changed the backup and restore sources the figures +measure (#700 and #740 as merged, and #474); "Reproduced on the rebased +tree" below gives the 100k cells run again on release builds of eb9e48de, +the rebased change. Each run: a fresh deployment (`init`, the port +patched, `[backup] max_concurrent_jobs = 2`, and, after provisioning, +`control_plane_delay_seconds = 0.05` and `index_propagation_delay_ms = 0` +set with `extenddb settings set`), the table loaded through the wire, then +for each measured phase the server stopped and started again so the +process's high-water mark covers that phase alone; `VmHWM` read one second +after the start (the idle figure) and one second after the phase ended (the +peak), sampled every 200 ms in between; "over idle" is the difference. A +backup ends when `DescribeBackup` reports `AVAILABLE`, a restore when +`DescribeTable` reports the table and every vector index `ACTIVE`, each +timed from the request as the client saw it; the job's own log line gives +the same figure within 0.3 s (on MongoDB the job's line is its `DONE`, +below). After each restore the target was counted with a paged `Scan` +(`Select = COUNT`), equal to the items written in every run, and on a vector +table searched with `SearchVectors` for item 7's own vector, which came +first in every run. + +The table: a 200-byte `S` pad of one repeated character (`"x" * 200`) under +a composite key (`pk` cycling over 1,000 values, `sk` the item number), no +secondary index, loaded with BatchWriteItem from four client processes of +eight threads each (the load is not measured). The vector shape adds an +`emb` attribute to every item, a list of 128 numbers (seeded uniform values +in [-1, 1], six decimals), and a `COSINE` vector index on it with no hash +element; its data files are about 2.5 KB of line text per item, 9.5 times +the plain shape's 265 bytes, and 532 bytes compressed against 3.0 to 5.9 +bytes for the plain shape (90 to 176 times; 176 for the 1M key-ordered +backup, 3.0 bytes per item, 145 at 100k, and 90 for MongoDB's unordered 1M +backup: the plain pad is one repeated character, so its 265-byte lines +compress to 3 to 6 bytes). The vector cells' durations are not the index's +cost alone. + +MongoDB restores its own backup, whose data files are unordered (the job's +log line says so). Cassandra's own CreateBackup is +in-catalog, so the Cassandra cell restores the SQLite build's store-backed +backup of the same table (key-ordered lines): the prefix is copied under the +root the Cassandra deployment owns and adopted with `extenddb backup sync`, +as `devtools/backup-restore-matrix` does, and the server is started again +before the restore. + +Host: 16 cores (Xeon Platinum 8124M), 30 GiB, Linux 5.10 (amzn2int); +PostgreSQL 16.15 with pgvector 0.8.7 in a container on loopback (55432); the +SQLite the binary bundles (libsqlite3-sys 0.30.1); MongoDB 7.0.43, +single-node replica set in a container on loopback; Cassandra 4.1.12, a +single node in a container on loopback started as the Cassandra integration +workflow starts its service (`MAX_HEAP_SIZE=1024M`, `HEAP_NEWSIZE=128M`, +`PasswordAuthenticator`), used by these runs alone. What was not held +constant: the host ran other work; the 1-minute load average at each phase's +start is beside each figure (1.3 to 6.4, and 12.6 at the start of the +PostgreSQL 1M vector backup, which began as its table's load ended and ended +at 1.7), so the durations are upper bounds, and each figure is one run. The +peaks varied by up to about 20% between runs of one cell, idle or loaded: +the SQLite 100k vector restore's peak over idle was 344.6 to 409.0 MiB over +nine runs (this record's cell and the runs below, two of them pinned to two +cores, idle and beside four busy loops), and the PostgreSQL 100k restore of +the same items with no vector index 60.0 to 71.4 MiB over five. Everything is on +loopback and on a filesystem store: nothing here says what a network +database or an S3 store's upload or download does to the duration, and the +spool's disk is not in these figures. Every run is under glibc's default +arenas (glibc 2.26) except the four rows marked `MALLOC_ARENA_MAX=1`. + +## Restores + +| backend | shape | data files | idle `VmHWM` | peak `VmHWM` | peak − idle | `ACTIVE` after | downloaded | load | +|---|---|---|---|---|---|---|---|---| +| MongoDB | 100k × 200 B, unordered lines | 1 | 23.5 MiB | 58.6 MiB | **35.1 MiB** | 3.92 s | 573,993 B | 2.7 | +| MongoDB | 1M × 200 B, unordered lines | 4 | 23.5 MiB | 75.3 MiB | **51.8 MiB** | 33.81 s | 5,932,752 B | 3.4 | +| Cassandra | 100k × 200 B, key-ordered lines | 1 | 22.5 MiB | 42.4 MiB | **19.9 MiB** | 6.12 s | 366,631 B | 1.3 | +| Cassandra | 1M × 200 B, key-ordered lines | 4 | 23.2 MiB | 56.3 MiB | **33.1 MiB** | 48.99 s | 3,017,864 B | 3.2 | +| PostgreSQL | 100k, 128-dimension vector | 4 | 21.4 MiB | 122.0 MiB | **100.6 MiB** | 25.15 s | 53,202,407 B | 4.9 | +| PostgreSQL | 1M, 128-dimension vector | 38 | 21.5 MiB | 132.6 MiB | **111.1 MiB** | 246.41 s | 531,977,239 B | 1.7 | +| SQLite | 100k, 128-dimension vector | 4 | 22.8 MiB | 376.8 MiB | **354.0 MiB** | 21.78 s | 53,202,407 B | 3.1 | +| SQLite | 1M, 128-dimension vector | 38 | 21.7 MiB | 573.8 MiB | **552.1 MiB** | 214.23 s | 531,977,239 B | 1.9 | +| SQLite | 100k, 128-dimension vector, `MALLOC_ARENA_MAX=1` | 4 | 22.8 MiB | 218.1 MiB | **195.3 MiB** | 27.93 s | 53,202,407 B | 2.8 | +| SQLite | 1M, 128-dimension vector, `MALLOC_ARENA_MAX=1` | 38 | 23.1 MiB | 227.0 MiB | **203.9 MiB** | 282.86 s | 531,977,239 B | 2.4 | + +The restore request answered in 0.014 to 0.181 s in every run (0.05 s on +MongoDB, 0.17 to 0.18 s on Cassandra, 0.06 s on PostgreSQL, 0.014 to +0.016 s on SQLite). On MongoDB the job's `DONE` ends the item load (33.6 s +at 1M), and the backend's backfill worker flips the table `ACTIVE` at its +next cycle: it runs every `GSI_BACKFILL_INTERVAL`, 5 s +(`crates/storage-mongodb/src/ttl_worker.rs`), and the end of a restore does +not wake it. A MongoDB restore's duration is therefore the job's plus up to +5 s: 3.92 s here, and 8.94 and 8.92 s in two runs at 100k whose job took +3.9 s (below). On Cassandra the 1M high-water mark reached 90% of its +final value at 11.5 s of 49.9, on MongoDB at 18.9 s of 34.7; those times +include the idle figure, and over idle they are 11.7 s and 21.1 s. + +From 100k to 1M the peak over idle moved +16.7 MiB on MongoDB, +13.2 MiB on +Cassandra and +10.5 MiB on PostgreSQL with a vector index. On PostgreSQL the +1M run's mark reached 90% of its final value at 51 s of 247 (57.8 s over +idle) and moved by 1 MiB after 150 s. + +SQLite with a vector index holds more, and under the default arenas its peak +is higher at 1M than at 100k (+198 MiB, 354 to 552 MiB over idle); the 1M +run levelled off after about 150 s, and no other size was measured. The 1M +run's `VmHWM` was 363 MiB at 15 s, 484 MiB at 90 s and 574 MiB at 150 s, +flat from there to the end at 215 s, with `VmRSS` 516 MiB at the end. Under +`MALLOC_ARENA_MAX=1` the same restore holds 195 MiB over idle at 100k and +204 MiB at 1M (+8.6 MiB; `VmHWM` 193 MiB at 20 s and 225 MiB at 60 s, +within 2 MiB of that to the end), so the growth is the allocator's, the +pattern the PR 9 record explains for its worst-expansion shape (items +allocated on the decoder's thread and freed on the backend's write, the +per-thread arenas keeping that memory). What stays is the restore's terms +on this shape: a write batch of up to `RESTORE_BATCH_ITEMS` (10,000) items, +here about 25 MB of line text with each item's 128 numbers decoded beside +it, and, as the PR 9 record found for indexed tables, the write +connection's page cache (`cache_size`, 32 MiB per connection) over the base +and index tables. The decoded size of one such item was not measured. One +arena cost this restore 32% in duration (282.9 s against 214.2 s, one run +each); with two 100k restores running at once it cost 131% (57.5 s against +24.9 s for the pair, 278.9 against 696.7 MiB over idle; the later runs +below). + +## Backups of the same tables + +| backend | shape | idle `VmHWM` | peak `VmHWM` | peak − idle | `AVAILABLE` after | backup size | load | +|---|---|---|---|---|---|---|---| +| MongoDB | 100k × 200 B | 24.5 MiB | 33.8 MiB | **9.3 MiB** | 1.45 s | 573,993 B | 3.0 | +| MongoDB | 1M × 200 B | 24.3 MiB | 40.3 MiB | **16.0 MiB** | 13.49 s | 5,932,752 B | 3.8 | +| PostgreSQL | 100k, 128-dimension vector | 20.8 MiB | 103.8 MiB | **83.0 MiB** | 24.19 s | 53,202,407 B | 6.4 | +| PostgreSQL | 1M, 128-dimension vector | 21.4 MiB | 125.6 MiB | **104.2 MiB** | 243.17 s | 531,977,239 B | 12.6 | +| SQLite | 100k, 128-dimension vector | 21.4 MiB | 119.2 MiB | **97.8 MiB** | 21.42 s | 53,202,407 B | 3.0 | +| SQLite | 1M, 128-dimension vector | 21.5 MiB | 159.1 MiB | **137.6 MiB** | 209.72 s | 531,977,239 B | 3.8 | +| SQLite | 100k, 128-dimension vector, `MALLOC_ARENA_MAX=1` | 22.3 MiB | 75.6 MiB | **53.3 MiB** | 22.19 s | 53,202,407 B | 3.3 | +| SQLite | 1M, 128-dimension vector, `MALLOC_ARENA_MAX=1` | 22.3 MiB | 80.1 MiB | **57.8 MiB** | 222.48 s | 531,977,239 B | 3.1 | + +The backup's growth from 100k to 1M is +6.7 MiB on MongoDB, +21.2 MiB on +PostgreSQL and +39.8 MiB on SQLite with a vector index under the default +arenas (+4.5 MiB under one arena), each under the PR 8 record's bound of +112 MiB. The MongoDB rows are of a table without secondary indexes, so they +do not compare with the PR 8 record's MongoDB rows (two GSIs, 500k in the +1M row's place); they are the first 1M-item MongoDB backup measured. A +backup stores no index contents (the design's testing summary), so the +vector rows measure the backup of the items that carry the vectors. + +## Durations (the RTO baseline the testing summary names) + +On this host, loopback, filesystem store, one run each: 1M items of 200 +bytes restore in 33.8 s on MongoDB (its own backup, unordered lines; another +run may take up to one 5 s cycle of the backfill worker more) and in +49.0 s on Cassandra (a key-ordered backup adopted from SQLite); 100k in +3.9 s (3.9 to 8.9 s over five runs: the job's time plus up to one cycle) +and 6.1 s. A 1M-item table whose items carry a 128-dimension vector, +with a vector index, backs up in 243 s on PostgreSQL and 210 s on SQLite +and restores in 246 s and 214 s; under `MALLOC_ARENA_MAX=1` the SQLite +backup took 222 s and the SQLite restore 283 s. At 100k: 24 s and 21 to +22 s to back up, 25 s and 22 to 28 s to restore. The 1M MongoDB backup took 13.5 s. The PR 9 record's 1M +restores of the plain shape were 9.5 s on PostgreSQL and 26.8 s on SQLite +on the same host class under another build and load, so they compare with +these as an order of magnitude only. + +After each vector restore one `SearchVectors` over the whole table (no hash +element, so every vector is scored) answered in 0.24 s on PostgreSQL and in +5.5 to 15.8 s on SQLite at 1M over seven runs (14.7 s in this run, 5.8 s in +the one-arena run, 5.5 s in a later run, 6.9 s and 5.7 s, one arena, on +04c29155, and 15.8 and 15.2 s in the f78e9bc1 runs), 0.04 s and 0.6 s +at 100k: SQLite's vector search is an exact scan +(`crates/storage-sqlite/src/vector_search.rs`), which an index with a hash +element bounds to one partition. + +## Later runs on the same host + +These runs used release builds of 30553b38 and, for the second 1,536-number +one-arena restore and the third two-GSI WAL run, of 04c29155, two earlier +versions of this change on PR 10 at 358ff8bd (main 94b5f689), before the +rebase onto main 887b6e26: 30553b38's Rust sources are 358ff8bd's, and +04c29155's differ from them only in the help text of two `init` options. On +the host above, with a separate harness: a fresh deployment per +run, a filesystem store, `max_concurrent_jobs = 2`, the item shapes above +loaded from four processes of eight threads, the server's default settings +(no `settings set`), the server restarted before each measured phase, +`VmHWM` read right before the request and at the end, `/proc//status` +sampled every 200 ms. Every restore was counted and every vector restore +searched for item 7's vector, as above. The 1-minute load average at each +phase's start is in the last column. + +They reproduce the cells above in kind (within 12% on the peak, within 20% +on the durations): the SQLite 100k vector restore 373.3 and 376.5 MiB over +idle in 23.3 and 22.7 s (load 3.0, 2.9), 192.3 MiB in 28.6 s under one arena +(2.8); the SQLite 1M vector backup 141.2 MiB in 210.7 s (3.1), its restore +522.8 MiB in 218.7 s (2.5) and 204.3 MiB in 286.7 s under one arena (1.7); +the PostgreSQL 100k vector backup 83.0 MiB in 26.5 s (10.8) and restore 112.5 +MiB in 29.7 s (8.1); the MongoDB 100k restore 36.2 and 33.2 MiB in 3.9 s; +the Cassandra 100k restore 19.7 and 20.1 MiB in 6.4 and 4.9 s. The vector +data files were byte-identical with the runs above (53,202,407 bytes at +100k). + +What the vector restore's memory belongs to (SQLite unless stated): + +| restore | default arenas | `MALLOC_ARENA_MAX=1` | load | +|---|---|---|---| +| 100k, 128 numbers per item, vector index | 373.3 / 376.5 MiB, 23.3 / 22.7 s | 192.3 MiB, 28.6 s | 3.0, 2.9 / 2.8 | +| 100k, the same items, no vector index | 407.5 MiB, 18.3 s | 184.0 MiB, 20.5 s | 3.4 / 2.6 | +| 20,000 items of 1,536 numbers, vector index | 668.6 MiB, 44.7 s | 301.0 MiB, 48.9 s; 315.9 MiB, 48.4 s | 2.3 / 1.6, 2.8 | +| two 100k restores at once (128 numbers, vector index) | 696.7 MiB, 24.9 s | 278.9 MiB, 57.5 s | 2.0 / 2.7 | +| 100k, 128 numbers, pinned to two cores: idle / beside four busy loops | 409.0 MiB, 22.5 s / 376.0 MiB, 60.6 s | | 2.2 / 2.7 | +| PostgreSQL 100k, 128 numbers, vector index | 112.5 MiB, 29.7 s | 48.8 MiB, 31.5 s | 8.1 / 6.2 | +| PostgreSQL 100k, the same items, no vector index | 60.0 MiB, 15.6 s | | 7.0 | + +The backup of the 1,536-number table held 520.9 MiB over idle and took +47.0 s (load 3.3); its data files carry 26,987 bytes of line text per item. On +SQLite the items without the index held as much as those with it, so the +vector index is not what the memory belongs to. The string control of "Runs +on release builds of f78e9bc1" shows what is: the restore's write batch +(up to +`RESTORE_BATCH_ITEMS`, 10,000 items, or `RESTORE_BATCH_BYTES`, 32 MiB of +item text, `crates/storage/src/restore_engine.rs`) and the decode chunks in +front of it (up to 1,000 items each), which grow with the item's size up to +the batch bound whatever the attribute types. Lists of numbers decode larger +than strings of the same text size, and the term grows with the numbers per +item. The vector index is about 20% of the 100k restore's duration (18.3 +against 22.7 to 23.3 s). On PostgreSQL the index's share is given by the pairs below. The second +1,536-number one-arena run (315.9 MiB) was taken on the same build to +confirm the first. + +Two more pairs on PostgreSQL, on release builds of 04c29155 with a third +harness written from the Method paragraph (a fresh deployment per cell, +`max_concurrent_jobs = 2`, the two `settings set` lines, the server +restarted before each phase, `VmHWM` read one second after the start and +sampled every 200 ms), each 100k vector restore followed at once by the +restore of the same items with no vector index: 99.9 MiB in 24.84 s (load +6.8) against 66.5 MiB in 16.26 s (6.0), and 98.5 MiB in 25.23 s (5.8) +against 71.4 MiB in 15.72 s (3.0). The vector index's share comes from +pairs whose two runs share a harness: these two, the later runs' pair in the +table above (112.5 MiB in 29.7 s against 60.0 MiB in 15.6 s) and the last +section's two (104.5 against 63.9 MiB in 26.99 against 18.05 s, and 99.8 +against 69.8 MiB in 28.30 against 17.34 s). The five put the index at 33.4, +27.1, 52.5, 40.6 and 30.0 MiB and at 35, 38, 47, 33 and 39% of the restore's +duration: 27 to 53 MiB and 33 to 47%. This record's cell has no no-index run +from its own harness, and its pair with the later runs' no-index run (100.6 +against 60.0 MiB), whose harness read the idle figure differently and kept +the server's default settings, is left out. On SQLite the same harness measured the 100k vector restore at +395.8 MiB in 21.86 s (load 4.1) and 358.2 MiB in 21.96 s (2.7), and the same items +with no index at 397.3 MiB in 18.01 s (5.5): the index 18% of the duration. + +SQLite's WAL beside a backup, on the same build: the writer of the manual's +figure (`PutItem` overwrites of 1,000 200-byte items from four threads, no +secondary index) beside the 100k vector backup grew a 51 MiB `-wal` file to +58.7 MiB over 15,051 writes, the frames reusing the file until they pass its +end. The same writer on a table with two GSIs (hash keys `g0` and `g1`, each +changed by every overwrite, `ALL` projection, the default +`index_propagation_delay_ms`) grew it from 58.7 to 1,277.6 MiB over 13,953 +writes in 23.4 s (91.6 KB per write) and, on fresh deployments, from 4.1 to +1,294.1 MiB over 13,996 writes in 25.6 s (96.6 KB, load 4.2) and from 4.1 to +1,261.8 MiB over 13,891 writes in 26.0 s (94.9 KB, load 3.7). With no backup +the same writer for 15 s left it at 4.1 MiB. Each file kept its size after +the server stopped. An asynchronous GSI write is queued in `gsi_pending` +inside the put's transaction and applied and removed by later transactions, +so one such overwrite appends about 22 to 24 pages of 4 KiB during a backup +(about 20 in a fourth run, below). + +## Figures measured for the manual + +The two database-cost figures of `docs/manuals/14-backup-and-restore.md` +("What a backup costs the database") for a table with no secondary index. +Release builds of 358ff8bd's Rust sources (those of 30553b38) on the host +above, a fresh deployment per run, a filesystem store, +`max_concurrent_jobs = 2`, the Method paragraph's two `settings set` lines (the reproductions +below used the defaults and measured the same in kind). The backup's source +is this record's 100k-item, 128-dimension vector table, loaded through the +wire, the server restarted after the load. The writer: four client threads +of `PutItem` overwrites of 1,000 items (a hash key, a 200-byte `S` pad that +stays the same, and an `N` attribute set to the writer's request counter, so +that each put changes the item) in a second table with no secondary index, +started with the `CreateBackup` +request and stopped when `DescribeBackup` reported `AVAILABLE`; the control +is the same writer for the same time with no backup. + +- PostgreSQL 16.15. The relation measured is the writer table's + `pg_total_relation_size` (with its indexes and TOAST), after a `VACUUM` + before the run. A 25.2 s backup beside 12,156 overwrites grew it from 0.4 to + 4.4 MiB, about 345 bytes per write. A `VACUUM (VERBOSE)` of that table 10 s + into the backup, 4,947 writes in, reported "4948 are dead but not yet + removable"; `pg_stat_user_tables` counted 12,114 dead after the backup, and + the first `VACUUM` after it left "0 are dead but not yet removable". The + control, after a `VACUUM FULL`: 17,478 overwrites in 25.2 s, 0.4 to 0.5 MiB. + Load 5.4 at the backup's start, 4.3 at its end. + +- SQLite. The file measured is the database's `-wal`, sampled every 200 ms. + The control first: 15,762 writes in 22.2 s left it at 4.0 MiB. Then a 22.8 s + backup beside 15,472 writes grew it from 4.0 to 61.1 MiB, about 3.9 KB per + write; it was 61.1 MiB 5 s after the backup, after 10.2 s more of the writer + with no backup, and after the server stopped. Load 3.2 at the backup's end. + +Reproduced on release builds of 04c29155 with the third harness above and +the server's default settings: + +- PostgreSQL: a 28.1 s backup beside 12,623 overwrites grew the relation + from 0.49 to 4.91 MiB (367 bytes per write); `VACUUM (VERBOSE)` 10 s in: + "3952 are dead but not yet removable"; the `VACUUM` after the backup: 0. The + same writer with no backup for the same time (17,042 writes), the relation + not compacted in between, left it at 4.91 MiB: the space was reused. Load + 9.2 at the end. + +- SQLite, the table with no index: 15,267 writes during a 22.5 s backup grew + the `-wal` file from 4.1 to 63.0 MiB (4.0 KB per write); 15,165 writes with + no backup left it at 4.1 MiB; 63.0 MiB 5 s after the backup and after a + `SIGTERM` stop. Load 5.2. + +- SQLite, the two-GSI writer of "Later runs": 14,212 writes during a 25.9 s + backup grew it from 4.2 to 1,117.7 MiB (82.2 KB per write, about 20 frames), + kept after the stop. Load 3.9. + +## Runs on release builds of f78e9bc1 + +Release builds of f78e9bc1 (04c29155's Rust sources, on PR 10 at 358ff8bd) +on the host above, with a +fourth harness written from the Method paragraph: a fresh deployment per +invocation, a filesystem store, `max_concurrent_jobs = 2`, the two +`settings set` lines, the shapes above loaded from four processes of eight +threads, the server stopped and started before each phase, `VmHWM` read one +second after the start and one second after the phase and sampled every +200 ms, every restore counted with a paged `Scan` (equal to the items in +every run) and every vector restore searched for item 7's vector (first in +every run). The host ran other work, and two cells often ran at once; the +1-minute load at each phase's start is in the last column. The string shape +replaces the 128-number list with a random 2,290-character `S` attribute and +has no vector index: 2,569 bytes of line text per item in the decompressed +data files, against 2,503 for the 128-number items. + +| restore | default arenas | `MALLOC_ARENA_MAX=1` | load | +|---|---|---|---| +| SQLite 100k, 128 numbers, vector index | 367.2 MiB, 22.26 s; 344.6 MiB, 36.58 s | 185.4 MiB, 37.13 s | 13.1, 5.4 / 7.8 | +| SQLite 100k, the same items, no vector index | 399.2 MiB, 20.53 s; 424.5 MiB, 22.36 s | 186.2 MiB, 23.25 s | 7.0, 7.3 / 6.5 | +| SQLite 100k, a 2,290-character string per item, no index | 253.4 MiB, 10.78 s; 252.5 MiB, 9.20 s | 129.6 MiB, 11.66 s; 129.8 MiB, 9.44 s | 6.2, 2.8 / 6.2, 2.7 | +| SQLite 1M, 128 numbers, vector index | 500.8 MiB, 226.47 s | 208.0 MiB | 4.5 / 1.6 | +| PostgreSQL 100k, 128 numbers, vector index | 104.5 MiB, 26.99 s; 99.8 MiB, 28.30 s | 52.0 MiB, 35.40 s | 14.5, 6.0 / 8.5 | +| PostgreSQL 100k, the same items, no vector index | 63.9 MiB, 18.05 s; 69.8 MiB, 17.34 s | | 7.1, 6.1 | +| MongoDB 100k × 200 B | 35.5 MiB, 8.94 s; 35.4 MiB, 8.92 s | | 7.3, not read | + +The 1M one-arena restore's duration is left out: it started while the +server was still dropping the previous 1M target. + +The string items carry no number and held 253 MiB over idle (130 MiB under +one arena), 6.7 times the PR 9 record's 38 MiB for 100k items of 200 bytes. +So on SQLite a restore of items of more than a few hundred bytes holds its +write batch and decode chunks ("Later runs"), which grow with the item's +size up to the batch bound whatever the attribute types. At the same text +size the 128-number items held more, 344.6 to 424.5 MiB at 100k over the +measured runs with or without the index and 184.0 to 195.3 MiB under one +arena, since each number decodes to an attribute value with a string of its +own. Under one arena the 1M restore measured 203.9 to 208.0 MiB over three +runs, so that figure is flat from 100k to 1M within about 25 MiB over the +measured runs. + +On MongoDB each of the two 100k restores logged the job's `DONE` "in 3.9 s; +index backfill pending", and the backfill worker's `ACTIVE` line came about +4.8 s later: the worker's cycle ("Restores"). The whole-table +`SearchVectors` over the 1M restored vectors took 15.78 s after the +default-arena restore (load 2.7) and 15.23 s after the one-arena one. + +## Reproduced on the rebased tree + +Release builds of eb9e48de, this change after its rebase onto main +887b6e26 (on PR 10 at 84bc2639; its Rust sources differ from 84bc2639's +only in the help text of two `init` options), on the host above, with a +fifth harness written from the Method paragraph: one deployment per backend +(a fresh `init`, its phases run in sequence on it), a filesystem store, +`max_concurrent_jobs = 2`, the two `settings set` lines, the table loaded +through the wire from 32 threads, the server stopped and started before +each phase, `VmHWM` read one second after the health check and one second +after the phase and sampled every 200 ms. Every restore counted 100,000 +items with a paged `Scan`. Only the 100k cells were run; no 1M cell was +repeated on this build. The host ran other builds: the 1-minute load was +5.3 to 11.0, against 1.3 to 8.5 at the earlier cells (0.8 to 5.0 times +the earlier cell's, cell by cell). Peak over idle, duration and load, +against the cell above with the same shape: + +| cell | above | eb9e48de | +|---|---|---| +| SQLite 100k vector, backup | 97.8 MiB, 21.42 s (3.0) | 89.6 MiB, 23.38 s (7.1) | +| SQLite 100k vector, restore | 354.0 MiB, 21.78 s (3.1) | 351.8 MiB, 28.83 s (6.65) | +| SQLite 100k vector, restore, `MALLOC_ARENA_MAX=1` | 195.3 MiB, 27.93 s (2.8) | 182.6 MiB, 32.72 s (10.95) | +| SQLite 100k vector, backup, `MALLOC_ARENA_MAX=1` | 53.3 MiB, 22.19 s (3.3) | 52.4 MiB, 22.98 s (9.49) | +| SQLite 100k string, no index, restore | 253.4 and 252.5 MiB, 10.78 and 9.20 s | 236.9 MiB, 9.33 s (5.4) | +| the same, `MALLOC_ARENA_MAX=1` | 129.6 and 129.8 MiB, 11.66 and 9.44 s | 129.9 MiB, 9.34 s (5.32) | +| PostgreSQL 100k vector, backup | 83.0 MiB, 24.19 s (6.4) | 76.0 MiB, 29.26 s (7.49) | +| PostgreSQL 100k vector, restore | 100.6 MiB, 25.15 s (4.9) | 106.1 MiB, 28.58 s (7.44) | +| PostgreSQL 100k vector, restore, `MALLOC_ARENA_MAX=1` | 48.8 and 52.0 MiB, 31.5 and 35.4 s | 51.9 MiB, 32.67 s (6.75) | +| MongoDB 100k × 200 B, backup | 9.3 MiB, 1.45 s (3.0) | 9.7 MiB, 1.44 s (9.39) | +| MongoDB 100k × 200 B, restore of its own backup | 35.1 MiB, 3.92 s (2.7) | 34.4 MiB, 4.08 s (9.12); 36.2 MiB, 3.88 s (8.79) | +| Cassandra 100k × 200 B, restore of a SQLite backup | 19.9 MiB, 6.12 s (1.3) | 19.8 MiB, 7.38 s (5.88); 19.5 MiB, 5.93 s (6.55) | + +Every cell reproduces in kind. The largest gap in peak is 8.4%, the two +100k vector backups (97.8 against 89.6 MiB on SQLite, 83.0 against +76.0 MiB on PostgreSQL), both lower on eb9e48de; the restores are within +6.5%. That is inside the spread of about 20% between runs of one cell +("Later runs on the same host"). The largest gap in duration is 32%, the +SQLite 100k vector restore (28.83 against 21.78 s, at load 6.65 against +3.1). The string control reproduces too: 237 MiB over idle (130 MiB under +one arena) against 352 MiB (183 MiB) for the 128-number items on the same +build. + +The figures measured for the manual, on the same build, beside the 100k +vector backup: + +- PostgreSQL: a 27.9 s backup beside 13,758 overwrites grew the writer + table's relation from 0.41 to 4.70 MiB, 327 bytes per write; a `VACUUM + (VERBOSE)` 10 s in, 4,871 writes in, reported "4866 are dead but not yet + removable", and the one after the backup 0. The control after a `VACUUM + FULL`: 20,714 writes in 27.9 s, 0.38 to 0.62 MiB. Load 7.75. +- SQLite, counted in WAL frames, since the `-wal` file was already larger + than the runs on the table with no index needed. That table: the + frames went from 343 to 16,548 over 16,223 writes beside a 27.3 s backup, + about one page per write (the control: 0.06 per write). With the pad and + every other attribute left unchanged by each put, they went from 165 to + 174 over 18,163 writes beside a 24.1 s backup: a put that leaves the + item's bytes unchanged writes almost no page. The two-GSI table, each put + changing both index keys: from 717 to 270,431 frames over 15,967 writes + beside a 27.7 s backup, 16.9 per write, and the file grew from 753.9 to + 1,062.6 MiB. Load 6.3 to 8.6. + +## The sizing sentence + +A restore of 200-byte items held under 60 MiB over idle on MongoDB and under +40 MiB on Cassandra at 1M items. With a 128-dimension vector on every item, +PostgreSQL held about 110 MiB over idle to restore and about 105 MiB to back +up at 1M, growing about 10 and 20 MiB from 100k. SQLite held 552 MiB over +idle to restore and 138 MiB to back up at 1M under the default arenas, +higher than at 100k, and 204 and 58 MiB under `MALLOC_ARENA_MAX=1`, flat +from 100k to 1M within about 25 MiB over the measured runs. The runs above +show that on SQLite this memory is the restore's write batch and decode +chunks, not the vector index, and that it grows with the item's size up to +the 32 MiB batch bound whatever the attribute types: size a SQLite host that +restores tables of items of more than a few hundred bytes from the measured +points under one arena (130 MiB per running restore for 100k items of +2.5 KB strings, 184 to 208 MiB for items of 128 numbers, 301 to 316 MiB for +20,000 items of 1,536 numbers), times `max_concurrent_jobs`, more for larger +items, and treat the default-arena figures as unbounded by this record. diff --git a/docs/design/15-backup-restore.md b/docs/design/15-backup-restore.md index 06ac759b1..b1898dbf6 100644 --- a/docs/design/15-backup-restore.md +++ b/docs/design/15-backup-restore.md @@ -169,7 +169,7 @@ spool_path = "" # restore download area for an S3 store; d On SQLite (file-backed) a missing `[backup]` section is equivalent to `kind = "filesystem"` with `path` = `backups/` beside the database file; a `validate` failure of that defaulted root is a logged warning that leaves the store unconfigured, not a startup error, so an upgrade never stops a server that never configured backups, while an explicitly configured root that fails `validate` is still a startup error. The warning holds only while the catalog has no backup rows for the defaulted root (`read_store_ids`); once it has, the root has been used and is handled as an explicitly configured one (revision item 29). On PostgreSQL and MongoDB a missing `[backup.store]` leaves the off-host path unconfigured and startup logs that; `kind = "disabled"` leaves it unconfigured on purpose on any backend. Without a configured store, CreateBackup on every backend uses the legacy in-catalog create (`LegacyBackupReader::create`, section 3) and logs a deprecation warning naming `[backup.store]` on each call, until PR 14 removes that path; after it CreateBackup without a configured store returns a ValidationException naming `[backup.store]`. Legacy backups stay readable, listable, restorable and deletable either way. `sqlite-memory` (any `:memory:` database path) has no database file and so no default root: CreateBackup there takes the off-host path only with `[backup.store]` set and the legacy create without it. Cassandra has no `SnapshotEngine`, so it uses the legacy create even with a store configured (section 3); a configured store serves restores of off-host backups into Cassandra and nothing else there. -Memory per running job, which operators size hosts from. A backup's terms, for a row of `r` bytes of item text whose decoded `Item` is `d` bytes (`d` is about 1.2 × `r` for a row of many short strings, #394's worst shape, where every `N` is a 133-character string): the read path holds the page's decoded rows, `d` each, while they wait for the pipeline, at most the rows of the budget PR 8 adds to the PostgreSQL and SQLite streams (`SNAPSHOT_FETCH_BYTES`, 64 MiB, a constant beside `DEFAULT_FETCH_BATCH`, not a `[backup]` key in v1) plus one row, and one row's parse, `r + 2d`, since the deserializer builds an intermediate value before the item; PostgreSQL pulls the page's rows from the connection one at a time and parses each as it arrives, so it holds no page text beyond the row being parsed, and SQLite pulls a row at a time with the driver's worker at most five rows ahead of the pull, `r` each (`row_buffer_size` on the pool, four, plus the row the worker holds while its `send` waits; 160 MiB at the line cap; the default of 50 made the run-ahead, not the page, the largest term on a table of large rows). The pipeline holds the feeder's item, the item channel's items (`ITEM_CHANNEL`, four) and the encoder's item, `d` each; the encoder's line buffer (one line, at most 32 MiB, the line cap) and gzip window; the feeder's measurement of an item over the default `max_item_size_bytes` holds no line (it counts the line's bytes through a writer that stops at the cap), so the terms are the same on an instance that raises the limit, at the largest stored row; the chunk sink's 1 MiB buffer and the chunk channel (four 1 MiB chunks); the upload window (`upload_part_size_bytes * max_concurrent_uploads` on an S3 store, one write buffer on a filesystem store); and the manifest's data-file entries until the manifest is written. No term grows with `data_file_target_bytes` or with the table, except on `sqlite-memory`, whose open copies the whole table (section 3); every term but the budget and the chunks grows with the row, which `max_item_size_bytes` bounds (the line cap bounds the line the job writes and names the item it refuses; the row is read and decoded before the feeder measures it). The sizing rule, per running job, is the terms at the line cap times the allocator's factor, times `max_concurrent_jobs`; the rule is stated at the default `max_item_size_bytes`, where no stored item's line exceeds the cap (27.7 MB is the largest line a 400 KB item expands to, `MAX_LINE_BYTES`'s doc); on an instance that raises it the page's decoded rows, one row's parse and the pipeline's items are at the largest stored row, which only `max_item_size_bytes` bounds (about 68 times the item's size as a line, for #394's worst shape). The terms at the line cap (`r` = 32 MiB, `d` = 1.2 `r`): the page's rows, 3 `d` = 115 MiB; one row's parse, `r` + 2 `d` = 109 MiB; the pipeline's six items, 6 `d` = 230 MiB; the line buffer, 32 MiB; the chunks, 5 MiB; on SQLite the run-ahead, 5 `r` = 160 MiB: about 490 MiB on PostgreSQL and 650 MiB on SQLite, an upper bound, since the run-ahead and a full pipeline are not both at their largest at one instant. The allocator's factor is glibc's: the row's 200,000 short strings are allocated on the tokio worker that parses the row and freed on the blocking thread that encodes it, and with glibc's default per-thread arenas that memory stays in the arenas between pages, so the process's high-water mark grows with the number of large rows the job reads and varies by about 200 MiB between identical runs; under `MALLOC_ARENA_MAX=1` (one arena, set in the server's environment on a host that backs up tables of large items; the admin guide's `max_concurrent_jobs` row carries it and the deployment guide's service unit carries the line commented out with that condition, since one arena serializes the worker threads' allocations on one lock and costs the request path 3.5 to 9.2% on SQLite and about 2% on PostgreSQL (the scans, 2.1%), measured on GetItem, Query and Scan with a Python client, the record) the peak is the terms' and does not grow with the table. Measured on PR 8's 16-core host at 27.7 MB rows (`r` = 26.4 MiB, three rows to a page; the terms' sum 545 MiB on SQLite), 1,000 small rows then 12, 24 and 48 such rows, each series six runs back to back on one SQLite binary (`docs/design/15-backup-restore-bench-pr8.md`; revision item 31): with the default arenas the peak over idle was 466, 588 and 760 MiB on a quiet host and 707, 700 and 1,281 MiB on the final build with the kill-point runs beside it (0.86 to 2.35 of the terms, growing with the rows and varying by hundreds of MiB between series; runs of the PR's first build on a loaded host reached 1,096 MiB at 48 rows; no plateau was measured), and under `MALLOC_ARENA_MAX=1` 481, 452 and 480 MiB and then 452, 480 and 455 MiB (0.83 to 0.88 of the terms, flat from 12 to 48 rows in both series; the SQLite ratios are of the peak over idle), and on PostgreSQL at 12 rows 340 MiB `VmHWM` under one arena against 750 to 788 MiB under the default arenas (0.82 of the terms at that row on PostgreSQL, 413 MiB, which has no run-ahead; that ratio is of the `VmHWM`, the process's whole high-water mark, and about 0.47 over its idle of 144 MiB); so the bound an operator can hold the process to is the terms at the line cap times 1.2 per job under `MALLOC_ARENA_MAX=1`, and under the default arenas a figure that grows with the large rows a job reads, 2.35 of the terms at 48 rows being the largest measured. A one-item channel saved about 50 MiB of the peak and cost a 1M-row backup 75% of its duration, so the channel stays at four. With 200-byte rows every term that grows with the row is under 1 MiB, the chunk channel and the sink hold their 5 MiB whatever the row, and the high-water mark of a 1M-row backup is within 10 MiB of a 100k-row backup's on PostgreSQL and SQLite (the bound the acceptance asserts is 112 MiB: the budget, the line cap and the channels) and the 500k-row backup's within 5 MiB on MongoDB, where the 500k run stands in for the 1M run (its load alone took 27 minutes through the wire). A running restore holds the parsed manifest, the decoder's line (at most `MAX_LINE_BYTES`) and the deserializer's intermediate value for it, the decoded items of `LOAD_CHUNK_BYTES` (32 MiB) of data-file text plus one line per chunk (the chunk closes with the item that reaches the bound) at four chunk positions (two in the channel to `bulk_load`, the one being filled, the one being drained), and the backend's write batch, times `max_concurrent_jobs`; the chunk bound counts the bytes the decoder read for the chunk's items, the unit the backup side budgets in, not the service's item sizing, and a chunk also closes at `LOAD_CHUNK_ITEMS` (1,000) items. The backend's write batch: at most 32 MiB of item text on SQLite and Cassandra (10,000 items when they are smaller), held beside the decoded items of that text, 4 MiB per INSERT statement on PostgreSQL (500 rows when they are smaller; one transaction per file; the item and its text, two copies), 1,000 documents per `insert_many` on MongoDB (the driver splits the call; the documents are held until it returns, bounded in bytes only by the data file, since `bulk_load` is per file). A `bulk_load` is one data file at a time, so the chunks in flight never hold more than one file's items, at most the source's `data_file_target_bytes` plus one line of text. None of these grows with the table: they grow with the item, which `max_item_size_bytes` bounds, and SQLite's `COUNT(*)` after the last file costs the counting connection's page cache, bounded by `cache_size` (32 MiB; 22 to 28 MiB measured at 1M and at 3M rows, the record). The input order of the backup's lines is a term of SQLite's beside these: a key-ordered backup (the server's own, of a hash-only or single-range-key table) appends to the base b-tree's right edge, an unordered one (every MongoDB-sourced backup and every multi-part-key table) dirties pages across the whole tree, and the write connection's page cache plus the allocator's arenas hold the difference (the record's input-order table: 1M items of 200 bytes 87 to 98 MiB key-ordered against 144 to 164 MiB unordered on SQLite, in 27 to 56 s against 66 to 90 s; PostgreSQL 16 MiB either way). A table with secondary or vector indexes adds no term of the restore's: on SQLite it adds the backend's own per-connection page cache for the index b-trees (`cache_size`, 32 MiB), bounded by that setting and growing sub-linearly with the table (the record's index table), on PostgreSQL a few MiB. Measured (`docs/design/15-backup-restore-bench-pr9.md`; revision item 33), `VmHWM` over idle from a fresh start, 24 worst-shape items of 27.7 MB: 848 MiB on SQLite and 778 MiB on PostgreSQL under the default arenas (848 and 1,013 MiB at 12 and 48 items on SQLite: the peak grows with the large items a restore reads, as the backup's does, and no bound was measured), 392 MiB on SQLite and 427 MiB on PostgreSQL under `MALLOC_ARENA_MAX=1`; 1,200 items of 350 KB in one file 270 MiB on SQLite and 156 MiB on PostgreSQL; 1M items of 200 bytes under a composite key with no secondary index, the backup's lines in key order, 89 MiB on SQLite and 16 MiB on PostgreSQL (100k: 38 and 10.7 MiB), restored in 26.8 s and 9.5 s; the same items under the same key with two full-projection GSIs 143 to 225 MiB on SQLite at 100,000 items in three runs and 165 to 172 MiB at 300,000, and 23 to 34 MiB on PostgreSQL at both: the excess on SQLite is the backend's own per-connection page cache (`cache_size`, 32 MiB) holding the index tables' b-tree pages plus the allocator's arenas (about a third less under `MALLOC_ARENA_MAX=1`), not the restore's buffers, which the 32 MiB batch bound holds whatever the indexes; it is bounded by `cache_size` and grows sub-linearly with the table. A restore of a table of large items is of the same order as its backup, not less; the sizing rule is the record's: about 450 MiB per running restore of large items under one arena, times `max_concurrent_jobs`. Each running backup also holds one connection from the PostgreSQL data pool for its duration, and on the separate layout one catalog-pool connection while its snapshot opens (section 3); the backend admits at most half the pool (after the raise below) as open snapshots and refuses more with `LimitExceeded`; a running restore holds one data connection and has no slot of its own after PR 6; and each PostgreSQL job holds one dedicated claim connection outside the pool. `pool_size` sizes both pools, and the rule is `pool_size >= max(MIN_POOL_SIZE, 2 * max_concurrent_jobs + 10)`: twice the jobs, so the half-pool bound admits every job `max_concurrent_jobs` allows, plus ten for requests; 18 at the defaults, under the default `pool_size` of 20. It applies whether or not `[backup.store]` is configured, because legacy restores are jobs too. Startup raises a smaller `pool_size` to that value with a warning naming both numbers, as it already raises a `pool_size` under `MIN_POOL_SIZE` (10) today. On SQLite the snapshot slot (`backup_server_slot_count`, `pool_size / 2 - 1` capped at 16, shared by the in-catalog create and the snapshot open) stays the backend's admission bound, as PostgreSQL's half-pool slot does, and a server raises `[storage.sqlite] pool_size` to `2 * (max_concurrent_jobs + 1)` (at most 34) with a warning naming both numbers so the slot refuses no job the permit admitted; 10 at the defaults, the default pool; above 16 jobs the slot refuses the seventeenth. `backup sync`, which is not a job, keeps of the configured root's rows (every status) each row's status, store id and table and the key of each AVAILABLE row of the configured store id, dropping the rest of each row as it is read, and holds the objects of the one prefix the listing is in with their sizes and the prefix of each manifest it listed; every backend's `list_backup_rows` collects its whole result before it yields the first row, so the read of the rows is sync's largest term on a catalog with many rows of the root (tombstones within their retention included): about 22.5 KiB of peak RSS per row of a table with five GSIs (1,452 bytes of `source_table_details_json`; 610 MiB at 25,000 rows and 1,144 MiB at 50,000, against 47 MiB with none, and 148 MiB at 50,000 rows whose JSON is `{}`, so nearly all of it is the parsed description), measured on a SQLite debug build on a 16-core x86_64 Linux host under glibc's default arenas, the same peak `backup list` reaches over those rows; the filesystem store builds its listing of the whole root in one `Vec` before it yields the first key (about 72 bytes plus the key per object, the store trait's doc), and the S3 store lists a page of 1,000 keys at a time. Measured on a filesystem store with 100 objects per prefix and 60-byte keys: 45 MiB of peak RSS for an empty root, 66 MiB for 100,000 objects and 98 MiB for 300,000, about 190 bytes per object, nearly all of it the store's listing (revision item 34). The report keeps an entry per prefix for the whole run (it lists every prefix the catalog already has a row for among the skipped ones), so at one object per prefix the per-prefix terms double that: about 350 bytes per prefix, 62 MiB of peak RSS for 50,000 such prefixes and 112 MiB for 200,000 against 45 MiB for an empty root (and about 165 bytes per object at 100 objects per prefix), measured on a SQLite debug build of PR 10 with a filesystem root and no rows. On an in-memory SQLite server that cost is paid in the serving process at every start. +Memory per running job, which operators size hosts from. A backup's terms, for a row of `r` bytes of item text whose decoded `Item` is `d` bytes (`d` is about 1.2 × `r` for a row of many short strings, #394's worst shape, where every `N` is a 133-character string): the read path holds the page's decoded rows, `d` each, while they wait for the pipeline, at most the rows of the budget PR 8 adds to the PostgreSQL and SQLite streams (`SNAPSHOT_FETCH_BYTES`, 64 MiB, a constant beside `DEFAULT_FETCH_BATCH`, not a `[backup]` key in v1) plus one row, and one row's parse, `r + 2d`, since the deserializer builds an intermediate value before the item; PostgreSQL pulls the page's rows from the connection one at a time and parses each as it arrives, so it holds no page text beyond the row being parsed, and SQLite pulls a row at a time with the driver's worker at most five rows ahead of the pull, `r` each (`row_buffer_size` on the pool, four, plus the row the worker holds while its `send` waits; 160 MiB at the line cap; the default of 50 made the run-ahead, not the page, the largest term on a table of large rows). The pipeline holds the feeder's item, the item channel's items (`ITEM_CHANNEL`, four) and the encoder's item, `d` each; the encoder's line buffer (one line, at most 32 MiB, the line cap) and gzip window; the feeder's measurement of an item over the default `max_item_size_bytes` holds no line (it counts the line's bytes through a writer that stops at the cap), so the terms are the same on an instance that raises the limit, at the largest stored row; the chunk sink's 1 MiB buffer and the chunk channel (four 1 MiB chunks); the upload window (`upload_part_size_bytes * max_concurrent_uploads` on an S3 store, one write buffer on a filesystem store); and the manifest's data-file entries until the manifest is written. No term grows with `data_file_target_bytes` or with the table, except on `sqlite-memory`, whose open copies the whole table (section 3); every term but the budget and the chunks grows with the row, which `max_item_size_bytes` bounds (the line cap bounds the line the job writes and names the item it refuses; the row is read and decoded before the feeder measures it). The sizing rule, per running job, is the terms at the line cap times the allocator's factor, times `max_concurrent_jobs`; the rule is stated at the default `max_item_size_bytes`, where no stored item's line exceeds the cap (27.7 MB is the largest line a 400 KB item expands to, `MAX_LINE_BYTES`'s doc); on an instance that raises it the page's decoded rows, one row's parse and the pipeline's items are at the largest stored row, which only `max_item_size_bytes` bounds (about 68 times the item's size as a line, for #394's worst shape). The terms at the line cap (`r` = 32 MiB, `d` = 1.2 `r`): the page's rows, 3 `d` = 115 MiB; one row's parse, `r` + 2 `d` = 109 MiB; the pipeline's six items, 6 `d` = 230 MiB; the line buffer, 32 MiB; the chunks, 5 MiB; on SQLite the run-ahead, 5 `r` = 160 MiB: about 490 MiB on PostgreSQL and 650 MiB on SQLite, an upper bound, since the run-ahead and a full pipeline are not both at their largest at one instant. The allocator's factor is glibc's: the row's 200,000 short strings are allocated on the tokio worker that parses the row and freed on the blocking thread that encodes it, and with glibc's default per-thread arenas that memory stays in the arenas between pages, so the process's high-water mark grows with the number of large rows the job reads and varies by about 200 MiB between identical runs; under `MALLOC_ARENA_MAX=1` (one arena, set in the server's environment on a host that backs up tables of large items; the admin guide's `max_concurrent_jobs` row carries it and the deployment guide's service unit carries the line commented out with that condition, since one arena serializes the worker threads' allocations on one lock and costs the request path 3.5 to 9.2% on SQLite and about 2% on PostgreSQL (the scans, 2.1%), measured on GetItem, Query and Scan with a Python client, the record) the peak is the terms' and does not grow with the table. Measured on PR 8's 16-core host at 27.7 MB rows (`r` = 26.4 MiB, three rows to a page; the terms' sum 545 MiB on SQLite), 1,000 small rows then 12, 24 and 48 such rows, each series six runs back to back on one SQLite binary (`docs/design/15-backup-restore-bench-pr8.md`; revision item 31): with the default arenas the peak over idle was 466, 588 and 760 MiB on a quiet host and 707, 700 and 1,281 MiB on the final build with the kill-point runs beside it (0.86 to 2.35 of the terms, growing with the rows and varying by hundreds of MiB between series; runs of the PR's first build on a loaded host reached 1,096 MiB at 48 rows; no plateau was measured), and under `MALLOC_ARENA_MAX=1` 481, 452 and 480 MiB and then 452, 480 and 455 MiB (0.83 to 0.88 of the terms, flat from 12 to 48 rows in both series; the SQLite ratios are of the peak over idle), and on PostgreSQL at 12 rows 340 MiB `VmHWM` under one arena against 750 to 788 MiB under the default arenas (0.82 of the terms at that row on PostgreSQL, 413 MiB, which has no run-ahead; that ratio is of the `VmHWM`, the process's whole high-water mark, and about 0.47 over its idle of 144 MiB); so the bound an operator can hold the process to is the terms at the line cap times 1.2 per job under `MALLOC_ARENA_MAX=1`, and under the default arenas a figure that grows with the large rows a job reads, 2.35 of the terms at 48 rows being the largest measured. A one-item channel saved about 50 MiB of the peak and cost a 1M-row backup 75% of its duration, so the channel stays at four. With 200-byte rows every term that grows with the row is under 1 MiB, the chunk channel and the sink hold their 5 MiB whatever the row, and the high-water mark of a 1M-row backup is within 10 MiB of a 100k-row backup's on PostgreSQL and SQLite (the bound the acceptance asserts is 112 MiB: the budget, the line cap and the channels) and the 500k-row backup's within 5 MiB on MongoDB, where the 500k run stands in for the 1M run (its load alone took 27 minutes through the wire). A running restore holds the parsed manifest, the decoder's line (at most `MAX_LINE_BYTES`) and the deserializer's intermediate value for it, the decoded items of `LOAD_CHUNK_BYTES` (32 MiB) of data-file text plus one line per chunk (the chunk closes with the item that reaches the bound) at four chunk positions (two in the channel to `bulk_load`, the one being filled, the one being drained), and the backend's write batch, times `max_concurrent_jobs`; the chunk bound counts the bytes the decoder read for the chunk's items, the unit the backup side budgets in, not the service's item sizing, and a chunk also closes at `LOAD_CHUNK_ITEMS` (1,000) items. The backend's write batch: at most 32 MiB of item text on SQLite and Cassandra (10,000 items when they are smaller), held beside the decoded items of that text, 4 MiB per INSERT statement on PostgreSQL (500 rows when they are smaller; one transaction per file; the item and its text, two copies), 1,000 documents per `insert_many` on MongoDB (the driver splits the call; the documents are held until it returns, bounded in bytes only by the data file, since `bulk_load` is per file). A `bulk_load` is one data file at a time, so the chunks in flight never hold more than one file's items, at most the source's `data_file_target_bytes` plus one line of text. None of these grows with the table: they grow with the item, which `max_item_size_bytes` bounds, and SQLite's `COUNT(*)` after the last file costs the counting connection's page cache, bounded by `cache_size` (32 MiB; 22 to 28 MiB measured at 1M and at 3M rows, the record). The input order of the backup's lines is a term of SQLite's beside these: a key-ordered backup (the server's own, of a hash-only or single-range-key table) appends to the base b-tree's right edge, an unordered one (every MongoDB-sourced backup and every multi-part-key table) dirties pages across the whole tree, and the write connection's page cache plus the allocator's arenas hold the difference (the record's input-order table: 1M items of 200 bytes 87 to 98 MiB key-ordered against 144 to 164 MiB unordered on SQLite, in 27 to 56 s against 66 to 90 s; PostgreSQL 16 MiB either way). A table with secondary indexes adds no term of the restore's: on SQLite it adds the backend's own per-connection page cache for the index b-trees (`cache_size`, 32 MiB), bounded by that setting and growing sub-linearly with the table (the record's index table), on PostgreSQL a few MiB. On SQLite the restore's write batch (`RESTORE_BATCH_ITEMS`, 10,000 items, or `RESTORE_BATCH_BYTES`, 32 MiB of item text) and the decode chunks in front of it grow with the item's size up to the batch bound, whatever the attribute types, so a restore of items of more than a few hundred bytes holds more: 253 MiB over idle for 100k items of about 2.5 KB of strings, 130 MiB under one arena (two runs each), and more for lists of numbers of the same text size, which decode larger (345 to 424 MiB, 184 to 195 MiB under one arena, at 100k items of 128 numbers, with or without a vector index); at 1M items of 128 numbers 552 MiB under the default arenas and 204 MiB under one arena, and 301 to 316 MiB under one arena for 20,000 items of 1,536 numbers (`15-backup-restore-bench-pr11.md`). A PostgreSQL vector index adds 27 to 53 MiB to the restore at 100k items (five pairs of runs with and without it). Measured (`docs/design/15-backup-restore-bench-pr9.md`; revision item 33), `VmHWM` over idle from a fresh start, 24 worst-shape items of 27.7 MB: 848 MiB on SQLite and 778 MiB on PostgreSQL under the default arenas (848 and 1,013 MiB at 12 and 48 items on SQLite: the peak grows with the large items a restore reads, as the backup's does, and no bound was measured), 392 MiB on SQLite and 427 MiB on PostgreSQL under `MALLOC_ARENA_MAX=1`; 1,200 items of 350 KB in one file 270 MiB on SQLite and 156 MiB on PostgreSQL; 1M items of 200 bytes under a composite key with no secondary index, the backup's lines in key order, 89 MiB on SQLite and 16 MiB on PostgreSQL (100k: 38 and 10.7 MiB), restored in 26.8 s and 9.5 s; the same items under the same key with two full-projection GSIs 143 to 225 MiB on SQLite at 100,000 items in three runs and 165 to 172 MiB at 300,000, and 23 to 34 MiB on PostgreSQL at both: the excess on SQLite is the backend's own per-connection page cache (`cache_size`, 32 MiB) holding the index tables' b-tree pages plus the allocator's arenas (about a third less under `MALLOC_ARENA_MAX=1`), not the restore's buffers, which the 32 MiB batch bound holds whatever the indexes; it is bounded by `cache_size` and grows sub-linearly with the table. A restore of a table of large items is of the same order as its backup, not less; the sizing rule is the record's: about 450 MiB per running restore of large items under one arena, times `max_concurrent_jobs`. Each running backup also holds one connection from the PostgreSQL data pool for its duration, and on the separate layout one catalog-pool connection while its snapshot opens (section 3); the backend admits at most half the pool (after the raise below) as open snapshots and refuses more with `LimitExceeded`; a running restore holds one data connection and has no slot of its own after PR 6; and each PostgreSQL job holds one dedicated claim connection outside the pool. `pool_size` sizes both pools, and the rule is `pool_size >= max(MIN_POOL_SIZE, 2 * max_concurrent_jobs + 10)`: twice the jobs, so the half-pool bound admits every job `max_concurrent_jobs` allows, plus ten for requests; 18 at the defaults, under the default `pool_size` of 20. It applies whether or not `[backup.store]` is configured, because legacy restores are jobs too. Startup raises a smaller `pool_size` to that value with a warning naming both numbers, as it already raises a `pool_size` under `MIN_POOL_SIZE` (10) today. On SQLite the snapshot slot (`backup_server_slot_count`, `pool_size / 2 - 1` capped at 16, shared by the in-catalog create and the snapshot open) stays the backend's admission bound, as PostgreSQL's half-pool slot does, and a server raises `[storage.sqlite] pool_size` to `2 * (max_concurrent_jobs + 1)` (at most 34) with a warning naming both numbers so the slot refuses no job the permit admitted; 10 at the defaults, the default pool; above 16 jobs the slot refuses the seventeenth. `backup sync`, which is not a job, keeps of the configured root's rows (every status) each row's status, store id and table and the key of each AVAILABLE row of the configured store id, dropping the rest of each row as it is read, and holds the objects of the one prefix the listing is in with their sizes and the prefix of each manifest it listed; every backend's `list_backup_rows` collects its whole result before it yields the first row, so the read of the rows is sync's largest term on a catalog with many rows of the root (tombstones within their retention included): about 22.5 KiB of peak RSS per row of a table with five GSIs (1,452 bytes of `source_table_details_json`; 610 MiB at 25,000 rows and 1,144 MiB at 50,000, against 47 MiB with none, and 148 MiB at 50,000 rows whose JSON is `{}`, so nearly all of it is the parsed description), measured on a SQLite debug build on a 16-core x86_64 Linux host under glibc's default arenas, the same peak `backup list` reaches over those rows; `backup sync --move-store` keeps the old root's rows without their parsed description and the configured root's (account, backup id) pairs, and reads one manifest and one prefix listing at a time, so its peak is that same read of the rows, made twice: of the old root's rows and of the configured root's, read whole for the two-rows check (and again by the plain sync after the move), each growing with the description: a move of 20,000 rows of a table with five GSIs (the same 1,452-byte description) into an empty store peaked at 569 MiB on a SQLite debug build and 500 MiB on a PostgreSQL debug build, against 51 and 49 MiB at 50 rows, 37 s each with filesystem roots on a local disk, and a move of 50 rows on the PostgreSQL debug build at 49 MiB into an empty store and 231 MiB into a store of 8,000 rows (1,569-byte descriptions; 1.8 and 5.8 s), about 23 KiB per row of the configured root, on that host (revision item 36); after the rewrite the move reads each moved row by its ARN, 16 at a time, and does not list the configured store again; the filesystem store builds its listing of the whole root in one `Vec` before it yields the first key (about 72 bytes plus the key per object, the store trait's doc), and the S3 store lists a page of 1,000 keys at a time. Measured on a filesystem store with 100 objects per prefix and 60-byte keys: 45 MiB of peak RSS for an empty root, 66 MiB for 100,000 objects and 98 MiB for 300,000, about 190 bytes per object, nearly all of it the store's listing (revision item 34). The report keeps an entry per prefix for the whole run (it lists every prefix the catalog already has a row for among the skipped ones), so at one object per prefix the per-prefix terms double that: about 350 bytes per prefix, 62 MiB of peak RSS for 50,000 such prefixes and 112 MiB for 200,000 against 45 MiB for an empty root (and about 165 bytes per object at 100 objects per prefix), measured on a SQLite debug build of PR 10 with a filesystem root and no rows. On an in-memory SQLite server that cost is paid in the serving process at every start. A configuration error that needs no request to the store (a part size, prefix or key-length problem, an insecure endpoint, a root that fails the owner-only rule or whose ancestors others can write to, a filesystem root that is or contains the database directory, a spool rule) is a startup error naming the path or bucket, except for the defaulted SQLite root above while the catalog has no rows for it, where it and a failure of the root's identity check are a warning that leaves the store unconfigured. A filesystem root's `validate` also writes a probe file, a request to the store: an I/O failure there or in resolving the root is a store condition that degrades the store (`failed validation`), as the same condition one reconciler cycle later does, not a startup error. A filesystem store's `validate` and identity check run before the listener, bounded by a timeout, so a mount that stops answering after the root is opened leaves the server starting with the store `checking` (a root that does not answer at startup holds the server before the listener, as the database directory would); an S3 store's are requests to the endpoint and run after the listener is up, in the reconciler's first cycle, with the store `checking` until the first result reports, so a stalled endpoint never holds the data plane (revision item 29). Every later check runs under the same bound (5 s for a filesystem root; 30 s for a bucket, under the SDK's per-attempt timeout so no retry outlives the check) and against the shutdown token: a store that has stopped answering costs a reconciler cycle its bound and nothing more, the cycle's other duties run, the check is recorded as a transport failure, and neither the drain at shutdown nor the process's exit waits on it (a filesystem root's abandoned call stays parked on a thread until the mount answers, no later check starts while it is, the root's calls run one at a time so that calls started before the first reached its bound wait for it under the bound and then have the bound for their own call, rather than parking a thread each, and the binary ends its runtime with a 5 s bound). A transport failure (no region, no credentials, an unreachable endpoint, a check that did not finish within its bound), a missing or denied bucket found after the listener, and a failure of section 4's identity check or manifest probe, do not stop startup, because a store problem must not take the data plane out of rotation (section 9): the failure is logged with its cause, the store component is reported unhealthy with the problem's category as its reason, the operations that touch the store refuse with a ValidationException that names the store's kind and the category and not its root (CreateBackup on PostgreSQL, SQLite and MongoDB, and RestoreTableFromBackup and DeleteBackup of an off-host backup; Describe and List, legacy backups and the in-catalog CreateBackup are unaffected), the reconciler acts on no row of the store (its legacy duty, which touches no store, still runs) and resumes no restore, and the check runs again each reconciler cycle until it passes. @@ -226,11 +226,11 @@ Which document and which row each PR changes; what each row says is in the ownin | upgrade manual: PR 14's migration refusal, `--force-drop-legacy-backups`, CreateBackup without a store after PR 14 | PR 14 | | upgrade manual: the #618 regression on the off-host path and the way out; the `[backup.store]` paragraph's asynchronous create and the restore of a store-backed backup deferred to PR 9a | PR 8 | | `docs/troubleshooting.md` DeleteTable entry and admin guide `control_plane_delay_seconds` row | #395 (in its diff) | -| ADR-0019 and the Cassandra RFC's "Backup and restore" section marked as superseded; the admin and upgrade manuals' backup sections | PR 11 | +| ADR-0019 and the Cassandra RFC's "Backup and restore" section marked as superseded in part; the admin and upgrade manuals' backup sections | PR 11 | | `docs/design/README.md` line 22 | PR 2 | | RFC-0003 backup criterion | a separate docs change (section 3) | -- This document is the design record; `docs/manuals/14-backup-and-restore.md` is derived from it for operators. Related records: ADR-0019 (Cassandra logical backup) and the "Backup and restore" section of `docs/rfcs/draft-cassandra-backend.md`, which still describe backup and restore as synchronous and are superseded on that point by #384 and this document; the admin and upgrade manuals' backup sections (#384) describe the in-catalog path this series replaces and are rewritten in PR 11. +- This document is the design record; `docs/manuals/14-backup-and-restore.md` is derived from it for operators. Related records: ADR-0019 (Cassandra logical backup) and the "Backup and restore" section of `docs/rfcs/draft-cassandra-backend.md` describe the first Cassandra implementation, with a synchronous restore that dropped the secondary indexes. Both are superseded in part (PR 11): the RFC section carries a note at its top, and the ADR its Status line, "Superseded in part by" this document (section 10), since `docs/adr/README.md` allows no other edit of an accepted ADR; what that Status covers is stated here. What still holds of ADR-0019: CreateBackup on Cassandra writes the in-catalog backup it describes, before it returns. What changed: RestoreTableFromBackup runs as a background job that answers `CREATING` and restores the secondary indexes (#384), a reconciler resolves an interrupted restore, a backup in a configured backup store restores into Cassandra, and the continuous-backup status is no longer stored. The reconciler also removes an in-catalog backup whose CreateBackup did not finish and that no process owns, so the ADR's Negative consequence that interrupted operations need a future reconciliation worker no longer holds for the create. The admin guide keeps the `[backup]` and `[backup.store]` reference and the `extenddb backup` commands, and the upgrade manual what an upgrade changes; both point at the manual for the rest (PR 11). ## Phasing and work breakdown @@ -307,7 +307,8 @@ pub trait BackupCatalog: Send + Sync { // DeleteBackup (5.2): the in-use check and the AVAILABLE-to-DELETING write in one transaction under the row's // Deleting | InUse(target name) | Status(the row's status when it is not AVAILABLE); PR 6 fn set_backup_deleted(&self, key: &BackupKey) -> BoxFuture>; // DELETING to DELETED, conditional; PR 6 - fn remove_backup_row(&self, key: &BackupKey) -> BoxFuture>; + fn remove_backup_row(&self, key: &BackupKey, store_kind: StoreKind, store_root: &str) -> BoxFuture>; + // a DELETED or FAILED tombstone still of the store the caller read it with (revision item 36) fn backup_in_use(&self, key: &BackupKey) -> BoxFuture, StorageError>>; // the name of a CREATING restore target registered from the backup, if any: the in-use rule of 5.2 that // 5.3 step 1 applies before the permit (on Cassandra read from `active_restores`, section 4); PR 6 @@ -445,10 +446,10 @@ Kill points (testing summary): every restore kill point, the fenced-out restore ### PR 10: `backup sync`, its options, `remove-target`, `migrate-legacy` -Goal: the catalog-loss recovery and the maintenance commands of section 4. As implemented (revision item 34) the section ships as three PRs: #746 plain `backup sync`, `--prune-missing`, `--adopt-store`, the `sqlite-memory` startup sync and `insert_backup_row`, with the acceptance cells those carry; #820 `--move-store`, `--forget-store` and `--forget-backup` with `list_backup_rows_by_root` and `rewrite_store`, and the report of the rows of other roots, with their cells; #821 `migrate-legacy` with its cells; `remove-target` shipped with PR 9 (item 33). Its line names the interfaces and carries the acceptance: +Goal: the catalog-loss recovery and the maintenance commands of section 4. As implemented (revision item 34) the section ships as three PRs: #746 plain `backup sync`, `--prune-missing`, `--adopt-store`, the `sqlite-memory` startup sync and `insert_backup_row`, with the acceptance cells those carry; #820 `--move-store`, `--forget-store` and `--forget-backup` with `list_backup_rows_by_root` and `rewrite_store`, and the report of the rows of other roots, with their cells (as implemented, revision item 36: `rewrite_store` takes the old kind and root, and a third method, `summarize_store_roots`, gives the report its stores); #821 `migrate-legacy` with its cells; `remove-target` shipped with PR 9 (item 33). Its line names the interfaces and carries the acceptance: - PR 10: `extenddb backup sync` per section 4 with store-root scoping, the store id, the maintenance claim with its printed wait bound, tombstone and identity checks, the row check before the manifest read and the insert-if-absent, `--prune-missing`, `--forget-store`, `--forget-backup`, `--move-store` (with `list_backup_rows_by_root` and `rewrite_store`), `--adopt-store`, the SQLite serve lock on every command that writes (`extenddb backup remove-target --account
`, section 4, shipped with PR 9 rather than here, item 33: a crash before the first progress record on MongoDB or Cassandra leaves a `target_not_empty` target that no other command removes) and the dead-owner resolutions such a command runs first, and `extenddb backup migrate-legacy |--all` with its re-read, validation and count checks and its refusals (section 3). Acceptance: drop the catalog rows, run sync, Describe and List and Restore work; a prefix without a manifest, with a bad sidecar, with a mismatched ARN, under `changelog/`, or with a CREATING, DELETING or FAILED row is not adopted; rows for another store root are reported and untouched, and the reconciler deletes none of their prefixes; after a store tree is copied to a new root, `--move-store` re-points its rows and Describe and Restore work against the new root; `--move-store` resolves the old root's CREATING and DELETING rows, stops on a row whose data files were not all copied, and refuses a DELETING, DELETED or FAILED row whose copied prefix holds a verified manifest; `--forget-backup` refuses a CREATING row with a live owner and a backup from which a restore target is CREATING (a RUNNING restore's target, a FAILED restore's target not yet removed, and on MongoDB a DONE restore's target awaiting the backfill), and says so and does nothing for a legacy ARN; a second maintenance command while one holds the claim fails at once; on SQLite every writing command refuses to run beside a server; `migrate-legacy` on a backup the PR 0 binary created produces a prefix Describe, List and Restore serve under the original ARN, and the legacy rows are gone; with the volume unmounted (an empty root, no id or another id) the store is unhealthy, sync without the option that resolves it refuses, and no row changes; `--adopt-store` of a root another catalog owns makes this catalog the owner, and the other catalog finds the store unhealthy at its next check; with host-local roots on two instances of one catalog no reconciler on either deletes a prefix because its own root lacks the bytes, and `--prune-missing` on the instance that lacks them removes the row and deletes nothing: the backup is BackupNotFound on every instance until a sync on the instance that holds the bytes adopts it again (revision item 34); `migrate-legacy` refuses a backup with a 13-digit id, one holding a value #618 let in and a pre-0.0.4 SQLite backup, keeps their legacy rows and names the reason; without `--prune-missing` a row whose manifest is gone is reported and stays AVAILABLE; after `--forget-store` and the store's return, sync adopts its prefixes; a CreateBackup started on another instance during `--move-store` either finishes before the rewrite or is refused; after `--move-store A` stops on a row with one data file missing under the configured root and `--forget-backup` lets the move finish, sync reports that prefix as incomplete and does not adopt it, and adopts it once the file is copied; a maintenance command started while a DELETING row and a RUNNING restore with a dead owner exist proceeds once the reconciler, which runs those two duties under the claim, has finished the one and failed the other. Depends on PR 6, 8. -Invariants honoured: 4(c) (store identity, scoping, adoption only through `--adopt-store` for a durable catalog and, for an in-memory one, at its startup from another in-memory catalog), 4(f) (every writing command takes the maintenance claim), 4(g) (what sync adopts and never adopts; the opt-ins), 4(d) (the ARN checked against the path and `source_table`), and 4(a)'s rule that no deletion duty acts on the bytes of a row the operator forgot. +Invariants honoured: 4(c) (store identity, scoping, adoption only through `--adopt-store` for a durable catalog and, for an in-memory one, at its startup from another in-memory catalog), 4(f) (every writing command takes the maintenance claim), 4(g) (what sync adopts and never adopts; the opt-ins), 4(d) (the ARN checked against the path and `source_table`), and 4(a)'s rule for the bytes of a row the operator forgot: no deletion duty starts on them once the maintenance claim is held, and a deletion begun before it ends by its deadline, the step in flight at it excepted, for a server process that keeps running (revision item 36). Interfaces it adds: ```rust fn insert_backup_row(&self, row: &BackupRow) -> BoxFuture>; // sync's unowned insert-if-absent @@ -456,10 +457,28 @@ Interfaces it adds: // a root's rows whatever their `store_id`: `--forget-store` and `--move-store` (section 4), which get a root and no id; PR 10 // on Cassandra this and `read_store_ids` read `backup_index_by_status` one partition per status for the root; // migration 6's entry carries `store_kind` and `store_id`, so `read_store_ids` needs no second read, and this - // listing reads each row's `backups_by_arn` row for the rest of the row - fn rewrite_store(&self, keys: &[BackupKey], store: &StoreIdentity) -> BoxFuture>; // `--move-store`, PR 10 + // listing reads each row's `backups_by_arn` row once for the rest of the row, keeping it whatever status + // partition its entry is in (revision item 36) + fn rewrite_store(&self, from_kind: StoreKind, from_root: &str, keys: &[BackupKey], to: &StoreIdentity) -> BoxFuture>; + // `--move-store`, PR 10 (#820, revision item 36): each row of `keys` still of `from_kind` and `from_root` and not + // CREATING gets `to`'s kind, root and id; the count of rows rewritten. On Cassandra per key: the new root's + // status entry, the lightweight transaction on the row, then the old entry deleted; a transaction that does + // not apply because another move put the row under `to`'s root with another id writes that entry again + // with the row's id + fn rewrite_store_runs_key_by_key(&self) -> bool; + // `true` on Cassandra (default `false`): the move then calls `rewrite_store` one key at a time, so a + // claim it reads lost stops it before the next key; PR 10 (#820, revision item 36) + fn summarize_store_roots(&self) -> BoxFuture, StorageError>>; + // the rows per store (kind, root, id) and status, for sync's report of other stores; on Cassandra a count + // of `backup_index_by_status` entries, which the report checks against each store's listing + // (`summary_counts_status_entries`); PR 10 (#820, revision item 36) + fn remove_orphan_status_entries(&self, store_kind: StoreKind, store_root: &str) -> BoxFuture>; + // `--forget-store`'s sweep of the root's status entries under the claim: one whose row is gone or names + // another store is deleted, one whose row is of the root under another status is written again under the + // row's status (both counted); a default of zero for the backends without status entries; PR 10 (#820, + // revision item 36) ``` -`backup sync` and its options, each with its contract: Sync lists the root (every object, since the trait has no delimiter listing; the store lists in key order, so sync holds one prefix's objects at a time and the prefix of each manifest it listed, beside what it keeps of the root's rows, whose read is its larger cost (section 7), the reserved `changelog` and `exports/` components skipped; a filesystem store builds its own listing of the root first, section 7), checks each prefix against the catalog as the listing reaches it, reads the manifest and sidecar of the prefixes that have no row, and inserts an AVAILABLE row for each verified manifest whose data files the listing contains, every `data_files` key present with its `bytes`, with an insert-if-absent (`BackupCatalog::insert_backup_row`): a CreateBackup whose row landed between the check and the insert keeps its row and sync skips the prefix. Before adopting, sync parses the manifest's `backup_arn` with the one `BackupArn` type in `extenddb-core` and requires the parsed account, table and backup id to equal the listed path components and the `source_table` identity; a mismatch is logged and skipped. A manifest whose ARN is in another region is adopted under this server's region's spelling of the ARN and logged: the row carries the ARN this server issues, so `ListBackups` names this server's region and requests resolve the backup under either spelling, and the manifest keeps its own; the dead-owner probe and the restore accept a manifest whose ARN differs from the row's in the region alone (`BackupArn::names_same_backup`). One of another partition than `aws` is skipped, since every request refuses such an ARN before it reads a row (revision item 34). `--move-store ` re-points the rows of that root to the configured store. It first resolves that root's CREATING and DELETING rows itself, against the configured store: a CREATING row becomes AVAILABLE if a verified manifest is there and FAILED otherwise, and a DELETING row whose prefix there holds no manifest has its deletion finished in the configured store. For each AVAILABLE row it compares the configured store's listing of the row's prefix with the manifest's `data_files` (every key present, with its `bytes`) as well as checking the verified manifest, and stops on a row whose copy is incomplete, naming it; `--forget-backup ` deletes one row's catalog entry so the move can go ahead for the rest; it runs under the maintenance claim like the other commands, refuses a CREATING row with a live owner (the job's next conditional write would match nothing and its manifest would be adopted by the next sync) and a backup in use, one from which a restore target is CREATING, and for a legacy ARN, which has no new-format row, says so and does nothing. One rule covers the three tombstone states: the move refuses to carry a DELETING, DELETED or FAILED row whose prefix in the configured store holds a verified manifest, and names it (that prefix is a backup the operator copied before or during its deletion, not a leftover; `--forget-backup` on the row is the way out). Then it rewrites `store_kind`, `store_root` and `store_id`; the reconciler then cleans the leftovers of the moved tombstones in the configured store. `--move-store` and `--forget-store` match `store_root` exactly as stored, the `?endpoint=` suffix of an S3 root included. A root whose file names another catalog stays unhealthy until an operator adopts it with `backup sync --adopt-store`, which takes the maintenance claim, rewrites the file's owner to this catalog and then syncs, recording the file's store id on the rows it adopts (the `backup_index` row PR 9b's matrix seeds by hand is the row this adoption writes); the previous owner acts on the root until a later check that can read its catalog finds the store unhealthy, at the earliest one reconciler interval after the cycle in progress ends, unless its deletion fence (section 4(c)) refuses a deletion or a CreateBackup first, so every server of the previous catalog is stopped before the adoption. On `sqlite-memory` the catalog, and with it the `catalog_id`, is new at every start, so the server runs a plain `backup sync` itself at startup when a store is configured, which adopts a root whose file names a `sqlite-memory` catalog as `--adopt-store` does, rewriting the file's owner to the new `catalog_id` before it syncs, so the root has one owner at a time there too once the previous process has exited (a new in-memory server is started only after the old one has exited; one started beside it adopts the root while the old one still acts on it until a later check or its fence); a root owned by any other catalog is unhealthy there as anywhere, and the backup CLI commands do not exist there (no file to open), so that startup sync is the only catalog-loss recovery there. +`backup sync` and its options, each with its contract: Sync lists the root (every object, since the trait has no delimiter listing; the store lists in key order, so sync holds one prefix's objects at a time and the prefix of each manifest it listed, beside what it keeps of the root's rows, whose read is its larger cost (section 7), the reserved `changelog` and `exports/` components skipped; a filesystem store builds its own listing of the root first, section 7), checks each prefix against the catalog as the listing reaches it, reads the manifest and sidecar of the prefixes that have no row, and inserts an AVAILABLE row for each verified manifest whose data files the listing contains, every `data_files` key present with its `bytes`, with an insert-if-absent (`BackupCatalog::insert_backup_row`): a CreateBackup whose row landed between the check and the insert keeps its row and sync skips the prefix. Before adopting, sync parses the manifest's `backup_arn` with the one `BackupArn` type in `extenddb-core` and requires the parsed account, table and backup id to equal the listed path components and the `source_table` identity; a mismatch is logged and skipped. A manifest whose ARN is in another region is adopted under this server's region's spelling of the ARN and logged: the row carries the ARN this server issues, so `ListBackups` names this server's region and requests resolve the backup under either spelling, and the manifest keeps its own; the dead-owner probe and the restore accept a manifest whose ARN differs from the row's in the region alone (`BackupArn::names_same_backup`). One of another partition than `aws` is skipped, since every request refuses such an ARN before it reads a row (revision item 34). `--move-store ` re-points the rows of that root to the configured store. It needs the configured store healthy, refuses the configured root and a root with no rows, and its wait covers the jobs of every store id the catalog records for the old root as well as the configured store's (revision item 36). It first resolves that root's CREATING and DELETING rows itself, against the configured store: a CREATING row whose owner is dead becomes AVAILABLE if a verified manifest is there, and otherwise stops the move with no status written, since a copy without the complete backup says nothing about the old root's prefix (revision item 36), and a DELETING row whose prefix there holds no verified manifest has its deletion finished in the configured store (a manifest that fails verification is a leftover the deletion removes). For each AVAILABLE row it compares the configured store's listing of the row's prefix with the manifest's `data_files` (every key present, with its `bytes`) as well as checking the verified manifest and that its size and item total are the row's, and stops on a row whose copy is incomplete, naming it (every row that stops the move is named in one run, and none moves while any does; a row that would give an account two rows of one backup id stops it too); `--forget-backup ` deletes one row's catalog entry so the move can go ahead for the rest; it runs under the maintenance claim like the other commands, refuses a CREATING row with a live owner (the job's next conditional write would match nothing and its manifest would be adopted by the next sync), a FAILED row whose heartbeat is within `stale_job_after_secs` (its job may still write under the prefix) and a backup in use, one from which a restore target is CREATING, and for a legacy ARN, which has no new-format row, says so and does nothing. One rule covers the three tombstone states: the move refuses to carry a DELETING, DELETED or FAILED row whose prefix in the configured store holds a verified manifest, and names it (that prefix is a backup the operator copied before or during its deletion, not a leftover). For a FAILED row the way out is `--forget-backup` on the row, and the next sync adopts the prefix; for a DELETING or DELETED row the backup was deleted, so the way out is to remove that prefix from the configured root and run the move again (`--forget-backup` on the row would have the next sync adopt the deleted backup again). Then, after one re-read of the configured root's identity file (the deletion fence's `check_catalog`), it rewrites `store_kind`, `store_root` and `store_id` of the rows still of the old root and not CREATING, in slices of 500 keys (one key per call on Cassandra, which rewrites key by key) with the maintenance claim confirmed before each 500 keys and read, with no write, before every call, and lists the old root again, reporting a row still there; the reconciler then cleans the leftovers of the moved tombstones in the configured store. `--move-store` and `--forget-store` match `store_root` exactly as stored, the `?endpoint=` suffix of an S3 root included. A root whose file names another catalog stays unhealthy until an operator adopts it with `backup sync --adopt-store`, which takes the maintenance claim, rewrites the file's owner to this catalog and then syncs, recording the file's store id on the rows it adopts (the `backup_index` row PR 9b's matrix seeds by hand is the row this adoption writes); the previous owner acts on the root until a later check that can read its catalog finds the store unhealthy, at the earliest one reconciler interval after the cycle in progress ends, unless its deletion fence (section 4(c)) refuses a deletion or a CreateBackup first, so every server of the previous catalog is stopped before the adoption. On `sqlite-memory` the catalog, and with it the `catalog_id`, is new at every start, so the server runs a plain `backup sync` itself at startup when a store is configured, which adopts a root whose file names a `sqlite-memory` catalog as `--adopt-store` does, rewriting the file's owner to the new `catalog_id` before it syncs, so the root has one owner at a time there too once the previous process has exited (a new in-memory server is started only after the old one has exited; one started beside it adopts the root while the old one still acts on it until a later check or its fence); a root owned by any other catalog is unhealthy there as anywhere, and the backup CLI commands do not exist there (no file to open), so that startup sync is the only catalog-loss recovery there. `migrate-legacy`: PR 10 adds `extenddb backup migrate-legacy |--all`, which streams a legacy backup through the frozen reader's item stream, checking each item with the backup job's item validator (5.1 step 4), into `DataFileEncoder` and the configured store under the legacy backup id, so the ARN survives, with a manifest whose `snapshot.kind` says it is a copy of an in-catalog backup (`legacy_in_catalog`, no marker). Before it inserts the AVAILABLE row or deletes anything it re-reads the manifest it wrote through `ExtenddbManifest::from_json_bytes` and checks that the manifest's item count equals the legacy backup's; only then does it insert the row and delete the legacy rows through the reader's `delete`. It refuses a backup, keeping its legacy rows, deleting the prefix it wrote and naming the reason, when the id carries no instant (the 13-digit and bare shapes, which the manifest parser refuses, section 4), when the validator refuses an item (a value #618 let in, which main's legacy restore copies as stored), and when it cannot state the definition (a pre-0.0.4 SQLite backup recorded no vector state, so `vector_indexes` would be a guess); the way out for such a backup is to restore it through the legacy reader and take a new backup of the restored table, which has a new ARN (section 10). The maintenance claim per backend and the SQLite serve lock (4(f) states the rules): On PostgreSQL the claim is an advisory lock keyed `hashtextextended('backup_maintenance', RESTORE_LOCK_SEED)` (`MAINTENANCE_LOCK_KEY`), held exclusively on the command's dedicated connection, with the `backup_maintenance` row carrying the holder, the generation and the heartbeat; a job tests the lock with `pg_try_advisory_xact_lock_shared` in one autocommit statement, so the shared lock ends with the statement, and then reads the row's heartbeat age, so a holder whose session ended is still seen held until its heartbeat is older than `stale_job_after_secs`; a command's take tries the exclusive lock a few times over about 10 ms on its connection, so a job's one-statement shared lock does not refuse it. On MongoDB it is a document in a `backup_maintenance` collection, claimed and heartbeated like a job row (5.1 step 3) and held while its heartbeat is younger than `stale_job_after_secs`; on Cassandra a `backup_maintenance` row claimed by lightweight transaction with a coordinator-clock heartbeat, read by jobs at `LOCAL_SERIAL`. On SQLite every backup CLI command that writes the catalog takes the database's serve lock, as `destroy` and `migrate` do, so the server must be stopped first and no other claim is needed; with the serve lock held every job owner is dead, and so is the holder of any maintenance claim row, which such a command takes over whatever its heartbeat says; before the wait it runs the reconciler's dead-owner resolutions itself, as the claim's holder (the cycle takes the held branch without reading the claim back through the zero window) (a CREATING backup row to AVAILABLE or FAILED, a RUNNING restore to FAILED with its target removed, a DELETING row finished) and then waits and removes CREATING rows with a zero stale window, so a job the stop interrupted never makes it wait or refuse whatever its heartbeat says (a RUNNING `manifest` restore, which PR 9a introduces, is the exception revision item 29 states), and a server starting under the serve lock removes a maintenance claim a stopped command left; `backup list` is a read and runs beside a server. @@ -468,9 +487,9 @@ Documentation rows it corrects: upgrade manual: on SQLite the backup CLI command ### PR 11: CI and documentation -- PR 11: CI and documentation: drop the `not import_export` filters; measure the 1M-item restore on MongoDB and Cassandra and the 1M-item vector cell on PostgreSQL and SQLite (the testing summary's two open cells; the nightly schedule for the matrix shipped with PR 9); the documentation per section 10 with RPO and RTO numbers from the PR 8, PR 9 and these runs on a stated host class. Depends on PR 3, 9a and 9b. +- PR 11: CI and documentation: drop the `not import_export` filters; measure the 1M-item restore on MongoDB and Cassandra and the 1M-item vector cell on PostgreSQL and SQLite (the testing summary's two open cells; the nightly schedule for the matrix shipped with PR 9); the documentation per section 10 with RPO and RTO numbers from the PR 8, PR 9 and these runs on a stated host class. Depends on PR 3, 9a and 9b. Shipped as revision item 35 says: both filters dropped; the four cells measured at 1M, none left open (`15-backup-restore-bench-pr11.md`); `docs/manuals/14-backup-and-restore.md` with every statement of the acceptance below, the PostgreSQL dead-tuple and SQLite WAL figures measured for it (the record's "Figures measured for the manual"); the admin guide's database names; ADR-0019's Status line and a note on the Cassandra RFC's section, superseded in part; the admin and upgrade manuals' backup sections pointing at the manual. -Documentation it writes (section 10's table): New `docs/manuals/14-backup-and-restore.md`: strategy, RPO and RTO statements per store kind (RPO equals backup interval, no PITR; RTO measured in the acceptance tests and published), configuration, IAM policy example, restore drill procedure, catalog loss recovery with `backup sync`, migrating legacy backups, what a restore preserves and what it does not (the table from the readiness discussion), moving a backup to or from Amazon DynamoDB with ImportTable and export. The IAM and S3-compatible-server material points at `crates/backup-store/README.md` rather than copying it; the admin guide's database names; ADR-0019 and the Cassandra RFC's "Backup and restore" section marked as superseded; the admin and upgrade manuals' backup sections rewritten. +Documentation it writes (section 10's table): New `docs/manuals/14-backup-and-restore.md`: strategy, RPO and RTO statements per store kind (RPO equals backup interval, no PITR; RTO measured in the acceptance tests and published), configuration, IAM policy example, restore drill procedure, catalog loss recovery with `backup sync`, migrating legacy backups, what a restore preserves and what it does not (the table from the readiness discussion), moving a backup to or from Amazon DynamoDB with ImportTable and export. The IAM and S3-compatible-server material points at `crates/backup-store/README.md` rather than copying it; the admin guide's database names; ADR-0019 and the Cassandra RFC's "Backup and restore" section marked as superseded (in part, as the "Shipped as" sentence above says); the admin and upgrade manuals' backup sections rewritten. Acceptance: the statements the series defers to the deployment guide, each stated in the manual: a filesystem store keeps the same-host failure domain, said in the first sentence of the backup section, with S3 as the production recommendation; encryption at rest is the store's job; a pickup agent runs as the server's user or as root, and a root under a storage administrator's directory, an NFS mount owned by `nobody` or a group-writable parent without the sticky bit is refused; a bind mount or a filesystem without Unix permission semantics is outside what the store can observe; an `AbortIncompleteMultipartUpload` lifecycle rule, and for a versioned bucket the `NoncurrentVersionExpiration` and `ExpiredObjectDeleteMarker` rules; the least-privilege IAM policy example; a PostgreSQL snapshot retains every dead tuple written during the backup and the SQLite WAL grows by the write volume during it, with the sizing figures the manual measures; the claim connection's keepalive bound (about a minute after the loss of the owner's host) and `pg_terminate_backend` on the claim's session as the way to end it sooner; a transaction-mode pooler is unsupported for the claim; the SDK's `trace` logs carry the session token and are not enabled in production; write access to an account's prefix is backup-administrator authority for that account, with per-account IAM prefix policies; MongoDB's `minSnapshotHistoryWindowInSeconds` as the operator setting a slow backup needs; a root that reports `0700` while enforcing nothing. Parallel today: PR 1, 2, 3, 4 (PR 0 has merged); PR 5 follows PR 4 and PR 6 follows both. Critical path: PR 4, 5, 6, 8, 9a. PR 1 was not on it and merged first, as #335 with migration 5; PR 6 took id 6, #819 took 7 for `stream_pending`, and PR 9a's `restore_progress` takes id 8 (4(i)). @@ -502,7 +521,7 @@ Phase 3, later (v1.x): - Distribution matrix for the filesystem default path: unprivileged local launch, the npm launcher, the container image with and without a volume, `sqlite-memory`. - Cross-backend matrix: nightly. - Live DynamoDB: Phase 2, measured. -- Performance: backup and restore of a 1M-item table per backend, and of a 1M-item table carrying a vector index on the backends that support one, duration and peak memory recorded as the published RTO baseline; memory must stay flat with table size (streaming, never materializing the table) in the restore's own terms: on SQLite a table with secondary indexes costs SQLite's page cache for the index b-trees above the no-index figure, an unordered backup costs its page cache across the base b-tree above the key-ordered figure, and the final `COUNT(*)` costs the counting connection's page cache, each bounded by `cache_size` (section 7, the record). The backup side's 1M figures are PR 8's record; the restore's are PR 9a's (`docs/design/15-backup-restore-bench-pr9.md`): 1M items of 200 bytes in key order in 9.5 s on PostgreSQL and 26.8 s on SQLite, the peak over idle 16 and 89 MiB, the growth from 100k to 1M 5.5 MiB on PostgreSQL and 51 MiB on SQLite (the write connection's page cache and the final `COUNT(*)`, each bounded by `cache_size`, not the items; from 1M to 3M rows the SQLite peak moved 11 MiB), and the same 1M items as an unordered backup 16 MiB on PostgreSQL and 144 to 164 MiB on SQLite in 66 to 90 s (SQLite's base b-tree under scattered inserts, not the restore's buffers). The MongoDB and Cassandra 1M restore and the vector cell are still to be measured (PR 11). Index contents are not stored in the backup; every index is rebuilt from the items on restore, so the vector cell measures the dominant restore cost. +- Performance: backup and restore of a 1M-item table per backend, and of a 1M-item table carrying a vector index on the backends that support one, duration and peak memory recorded as the published RTO baseline; memory must stay flat with table size (streaming, never materializing the table) in the restore's own terms: on SQLite a table with secondary indexes costs SQLite's page cache for the index b-trees above the no-index figure, an unordered backup costs its page cache across the base b-tree above the key-ordered figure, and the final `COUNT(*)` costs the counting connection's page cache, each bounded by `cache_size` (section 7, the record). The backup side's 1M figures are PR 8's record; the restore's are PR 9a's (`docs/design/15-backup-restore-bench-pr9.md`): 1M items of 200 bytes in key order in 9.5 s on PostgreSQL and 26.8 s on SQLite, the peak over idle 16 and 89 MiB, the growth from 100k to 1M 5.5 MiB on PostgreSQL and 51 MiB on SQLite (the write connection's page cache and the final `COUNT(*)`, each bounded by `cache_size`, not the items; from 1M to 3M rows the SQLite peak moved 11 MiB), and the same 1M items as an unordered backup 16 MiB on PostgreSQL and 144 to 164 MiB on SQLite in 66 to 90 s (SQLite's base b-tree under scattered inserts, not the restore's buffers). The MongoDB and Cassandra 1M restores and the vector cell are the PR 11 record's (`docs/design/15-backup-restore-bench-pr11.md`; revision item 35), on the same host class with no cell left open: 1M items of 200 bytes restore in 33.8 s on MongoDB (its own backup, unordered) and 49.0 s on Cassandra (a key-ordered SQLite backup adopted with `backup sync`), the peak over idle 52 and 33 MiB, 16.7 and 13.2 MiB above the 100k figure; a 1M-item table whose items carry a 128-dimension vector, with a vector index, backs up in 243 s on PostgreSQL and 210 s on SQLite and restores in 246 s and 214 s, the restore's peak over idle 111 MiB on PostgreSQL (10.5 MiB above 100k) and 552 MiB on SQLite under the default arenas (198 MiB above 100k; the 1M run levelled off after about 150 s, and no other size was measured), 204 MiB under `MALLOC_ARENA_MAX=1` (8.6 MiB above 100k), where the backup took 222 s and the restore 283 s. On SQLite that memory is the restore's write batch and decode chunks, not the vector index: they grow with the item's size up to the 32 MiB batch bound whatever the attribute types (100k items of about 2.5 KB of strings held 253 MiB over idle, 130 MiB under one arena), and lists of numbers decode larger at the same text size (the same 100k items of 128 numbers with no vector index held 397 to 424 MiB, and 20,000 items of 1,536 numbers 669 MiB, 301 to 316 MiB under one arena). Against this item's criterion the vector cell is flat from 100k to 1M under `MALLOC_ARENA_MAX=1` (within about 25 MiB over the measured runs) and is not shown bounded under the default arenas. On PostgreSQL the index adds 27 to 53 MiB at 100k (five pairs of runs with and without it, each pair from one harness). On MongoDB the table becomes `ACTIVE` at the backfill worker's next 5 s cycle after the load, so the restore's duration is the load's plus up to 5 s (3.9 to 8.9 s at 100k). Index contents are not stored in the backup; every index is rebuilt from the items on restore, and the vector index is about 20% of the 100k restore's duration on SQLite (18.3 s without it against 22.7 to 23.3 s with it) and 33 to 47% on PostgreSQL (the same five pairs, 15.6 to 18.1 s without it against 24.8 to 29.7 s with it). ## Review checklist for implementation PRs @@ -593,3 +612,7 @@ Changed after a review of the draft against main `9f43d03` and PRs #384, #335, # 33. (2026-11-01, PR 9 as implemented — the design's PR 9a and PR 9b as one PR, the maintainer's decision.) The restore job runs from the store on PostgreSQL, SQLite, MongoDB and Cassandra, same-backend and cross-backend, from a filesystem or an S3 store; the matrix and the nightly are PR 9b's, shipped here. `RestoreTableFromBackup` of a `backup_index` row no longer answers the "in a backup store; this release restores in-catalog backups only" refusal `check_catalog_row_for_restore` carried; a healthy configured store's AVAILABLE row is restored, and the refusal is gone from the engine and the differences, usage and upgrade docs (5.3, section 10, PR 9a). The interface: `RestoreEngine`, a new trait in `StorageEngine`'s bound with `bulk_load(target, claim, BulkLoadFile { file_index, start_at, reapplying }, items) -> applied`, `finish_restore(target, claim, item_count, size_bytes) -> Finished { Active, Pending, NotMatched }`, `count_restore_target(target) -> u64`, `read_restore_progress` (default `[]`) and `sweep_restore_progress(running) -> u64` (default 0); `StorageEngine` has no methods of its own, so `bulk_load` is not "a new `StorageEngine` method" (section 3's PR 9a interface list). `BackupCatalog` gains `claim_restore(account_id, input: CreateTableInput, &RestoreRegistrationRow) -> (TableDescription, Box)` (the registration creates the CREATING target and returns the description the request answers; the record is born owned under the backend's generation, `RestoreRegistrationRow.owner_generation` being `UNOWNED_GENERATION` and the backend minting its own), `claim_restore_dead(key, stale_after, spool_host) -> Option>` (the restore analogue of `claim_dead_backup`, writing the claimant's `spool_host`, a fresh generation and a fresh `heartbeat_at` on the database's clock in one conditional write, where the backup takeover leaves the heartbeat: nothing waits on a restore record's old heartbeat after a takeover, and a second taker must find the record live at once, 4(e)), `read_restore_job`, `list_restore_jobs(Option)` and `remove_restore_target`/`fail_restore_job` as the reconciler uses them; `RestoreJobRow` carries `store`, `spool_host`, `file_index`, `items_applied`, `failure_reason`, `completed_at`, so the FAILED-target duty reads a row's reason and the sweeps read its store and `spool_host` (section 3, 4, 5.3, PR 9a, PR 9b). The design's `claim_restore(&RestoreJobRow) -> Box` and `record_progress(&dyn JobClaim, file_index, items_applied)` are gone: there is no `record_progress`; every backend records progress inside `bulk_load`, in the write that applied the items (PostgreSQL a `restore_progress` row in the file's data transaction, SQLite and Cassandra a `tables`-row update per batch, MongoDB a `restore_job.file_index` update per file), and the design's "conditioned on the generation" holds on SQLite, MongoDB and Cassandra but on PostgreSQL the progress row lives in the data database, which holds no job record, so the fence there is the base table's primary key (a file applied twice fails on its first key, a recorded file on the progress row's own key) and the claim's liveness, which the engine confirms before each file and whose loss, the heartbeat's, aborts a file in flight; the generation fences only the record's own writes in the catalog (DONE, FAILED, the takeover) (5.3 step 5, PR 9a). `bulk_load` carries `reapplying` so MongoDB's duplicate suppression is explicit: on a resume the first file the engine meets is `reapplying = true` and every later file false; MongoDB suppresses an all-E11000 `insert_many` only when `reapplying`, else "the backup holds a key twice"; PostgreSQL and SQLite never suppress (their base rows are plain `INSERT`s, a duplicate fails the file whatever the flag, and SQLite has no unrecorded partial write, its progress being in the batch's transaction); Cassandra's `put_restored_item` is an upsert, so a duplicate collapses and `count_restore_target` one short of the manifest's total is the only duplicate check there (5.3 step 6, PR 9a). The restore ends by counting the target against the manifest's item total on every backend, so an unordered backup holding one key in two files and a key-range check that missed are both caught; a key-ordered file runs #394's per-file `KeyRange` check, which ends the file's stream at an item outside its range. The migration for PostgreSQL's `restore_progress` is id 6 (`MigrationId::RestoreProgress`), a `StoreKind::Data` body on PostgreSQL (`data_migrations/0006_restore_progress.sql`), `NotApplicable` on SQLite, and nothing on MongoDB and Cassandra, whose bootstrappers have no `MigrationId` table; this corrects "data migration 005 for `restore_progress`" in the PR 9a section (its acceptance line and its load paragraph) and the PR 6 line on reserving numbers; item 14's "data migration 005 reserved" stands as the history it is (4(i), PR 6, PR 9a). The manifest registration locks the `backup_index` row (the lock `start_backup_delete` takes: PostgreSQL `SELECT ... FOR UPDATE`, SQLite the `BEGIN IMMEDIATE` write transaction, MongoDB `$inc lock_seq` on the document, Cassandra the `active_restores` lightweight transaction on the backup's partition), while the legacy registration keeps the `backups` row; a legacy and a store-backed restore into one name exclude each other (PostgreSQL the advisory `restore_lock_key`, which both paths take; on SQLite the store-backed path's `restore:/
` slot and, for a legacy restore, the `tables` row under the write transaction), so a `DeleteBackup` of a `backup_index` row meets only store-backed registrations and the two orderings resolve one before the other (DeleteBackup wins → the registration is `BackupNotFound` and no target; the registration wins → DeleteBackup is `BackupInUse` and the row stays AVAILABLE) (5.2, 5.3, PR 9a). The restore's ACTIVE flip sets the table and every secondary and vector index of the target in one catalog transaction on PostgreSQL and SQLite (an index of a restore target is created ACTIVE, and `activate_restore_target` sets ACTIVE any `indexes` or `vector_indexes` row that is not), on MongoDB the job writes DONE and `restore_backfill_pending` and the backfill worker flips the table ACTIVE once every index has left CREATING (`finish_restore` returns `Pending` there), and on Cassandra the one lightweight transaction flips the table alone, its index rows created ACTIVE (`Active`, never `Pending`), so a Query on each GSI and `SearchVectors` answer as soon as the target reports ACTIVE (5.3 step 6, PR 9a). The request bounds its wait for the detached registration at `RESTORE_REGISTRATION_TIMEOUT` (60 s) on every backend; past it it answers `InternalServerError` while the task goes on and the target appears through `DescribeTable`, so the differences document's MongoDB-only sentence becomes every backend's (5.3 step 4, PR 9a). The maintenance check runs after the registration on the manifest path (5.3 step 4 as written) and before the permit on the legacy path (item 29), both orders in `restore_table_from_backup` and `restore::start`; the restore's first confirmed beat stands at the instant `claim_restore` returned (the record's `heartbeat_at` was stamped inside the registration, before the return, so the instant is at or after the beat it stands for by the registration's remaining duration, which each backend bounds, 4(e)), where the backup job takes the instant before its claim's insert (5.1 step 4), so a registration that took longer than two heartbeat periods does not end the job at its first check; a held claim on the manifest path leaves a FAILED `maintenance` record whose target the job removes before it answers `LimitExceededException`. Only `StoreError::Transport` is retried on a data-file download, for up to `DOWNLOAD_RETRY_BOUND` (15 min, counted from the job's first download attempt, every file's retries against it) with `min(2^n s, 60 s)` plus up to a second of jitter, so the retry covers S3 stores (a filesystem store's `get` errors are `Io` and fail at once); `NotFound`, a denial, and a digest or size mismatch fail at once naming the file. On a filesystem store "verified in place" is two `get` passes, a hashing pass before any load and a decoding pass through `ExtenddbManifest::reader` that hashes again (so a file replaced between the passes fails in the load) with no spool; on an S3 store the files are spooled as `restore-
/.json.gz` with `.partial` → `sync_all` → rename → `.verified` (the marker holding the file's sha256, compared with the manifest's before the file is trusted; an entry left by a run of this host for a record another host then owned is removed before use) and read through `reader_prehashed`, the spool at `spool_path` (the defaulted `backup-spool/` on file-backed SQLite). A restore the shutdown stops records `RestoreJobs{cancelled}` and no worker metric, keeps its spool and its progress and writes no status (the `SHUTDOWN` reason is a backup's alone); a lost claim or a DONE write that matched no record records `RestoreJobs{failed}` and `WorkerErrorCount{RestoreJob}` with no catalog or table access; `fail_restore` writes FAILED then runs `remove_restore_target` only when that write matched. A job that learnt the record is another's (the lost claim, the DONE write's `NotMatched`, `fail_restore`'s no-table-access arm) leaves its spool entry to the sweep, since a same-host successor may be reading it (the entry is keyed by table id); every other end but the shutdown's removes it (4(e), 5.3 step 7, PR 9a). The load hands the backend one `bulk_load` per file over a stream fed in chunks of at most `LOAD_CHUNK_ITEMS` (1000) items or `LOAD_CHUNK_BYTES` (32 MiB of data-file line text) through a channel of two chunks; a restore holds the parsed manifest, one decoded line and its parse, the decoded items of four chunk positions and the backend's batch (32 MiB of item text on SQLite and Cassandra, 4 MiB per INSERT on PostgreSQL, 1,000 documents on MongoDB), which is section 7's restore sentence, with the figures measured on this PR's final tree (`docs/design/15-backup-restore-bench-pr9.md`: 848 MiB over idle on SQLite and 778 MiB on PostgreSQL at 24 worst-shape items under the default arenas, 392 and 427 MiB under one arena; 89 and 16 MiB at 1M key-ordered items of 200 bytes, in 26.8 s and 9.5 s, 144 to 164 and 16 MiB for the same items unordered), which replace the earlier sentence, "A restore holds one line's decode plus one insert batch plus the parsed manifest for the whole job; neither grows with the table": the terms stand, the figures are measured, and a restore of large items is of the backup's order. The metrics are `MetricName::{RestoreJobs, RestoreJobDuration, RestoreBytesDownloaded}` with the outcome in the operation dimension, `QuerySource::RestoreJob` for the worker metrics, and `BackupStoreErrors{get}` on every failed `get` of a restore job (the request's and a resume's two document reads and each download attempt the store failed, retried or not; a digest mismatch and the spool's own errors are not the store's); section 9's `restore_jobs_total{outcome}` and `restore_job_duration_seconds` are a converter's spellings, as item 31 said for the backup names (section 9, PR 9a). A backend's refusal of an item in `bulk_load` is not a wire exception: on MongoDB an `N` sort-key value past Decimal128's 34 significant digits is the write path's `ValidationException`, which becomes the restore's FAILED reason, and the client sees the target disappear; a 38-digit non-key number is stored as text and restored intact on every backend. The reconciler resumes a dead store-backed restore from its recorded progress (per file on PostgreSQL and MongoDB, per batch on SQLite and Cassandra) under `claim_restore_dead` once the heartbeat is stale (on SQLite at once after a restart, by the record's slot; on PostgreSQL once the owner's claim session has ended as well), re-downloading to its own spool when the dead owner's `spool_host` is not this host's; it fails a dead manifest record under a maintenance claim before it looks at the record's store (`OWNER_DEAD`, the target removed; `target_not_empty`, the target left for `remove-target`, when it recorded no progress and the target holds items), fails a record of a store other than the configured one or of no store with `OTHER_STORE` (naming `backup sync --move-store`, the target left to the FAILED-target duty), skips a resume without a permit, and FAILs a resume with no recorded progress into a non-empty target as `target_not_empty`, left for `extenddb backup remove-target --account
` (PR 10's command, shipped with PR 9, below); its two sweeps (`restore_progress` rows of tables that are not RUNNING restore targets; spool entries of restores not RUNNING with this host's `spool_host`) run only without the maintenance claim and read `list_restore_jobs(Some(RUNNING))`, a backend that cannot list them logging once and skipping both (section 4, 5.3 step 7, PR 9a). A store-backed backup of a multi-part-key table is written `unordered` and restores on every backend, the target laid out as a natively created table (`bulk_load` keys rows with the backend's own key functions), while a legacy backup of one keeps the reader's refusal; the #384 override tests run through both the legacy and the manifest paths for every case with a definition and assert the same input or the same refusal, the manifest path deriving the SSE specification from the manifest's `sse_description` where the legacy path keeps the specification as written. The cross-backend matrix (PR 9b): a cell copies the source's backup prefix into a root the target catalog owns and inserts the target's `backup_index` row as `backup sync --adopt-store` will (PR 10), so the cell proves the restore and not the adoption; it runs from the committed `devtools/backup-restore-matrix`, and `.github/workflows/matrix-nightly.yml` (a `schedule` cron and `workflow_dispatch`) runs the twelve plain cells and the duplicate-key and vector cells nightly, building the four backend binaries and starting PostgreSQL, a MongoDB replica set and Cassandra as the integration workflows do; the seed stands in for `backup sync --adopt-store` until PR 10 replaces it. The restore kill points are run by hand with the `kill-points.py` posted with PR 9 (outside the repository, as PR 8's was), one mode per backend where it applies: `restore_after_download`, `restore_after_file_commit`, `restore_sqlite_mid_batch`, `restore_cassandra_before_progress`, `restore_mongodb_after_insert_many`, `restore_postgres_before_progress_delete`, `restore_mongodb_before_worker_flip`, `restore_between_failed_and_removal`, `restore_fenced_out`, `pg_terminate_backend` and `sigterm`, each a SIGKILL or a signal at the named point with a restart and the outcome asserted (the exact count after a resume, or the target gone for a failing restore). Two points pause before the first progress record of a file or batch (`restore_mongodb_after_insert_many`, `restore_cassandra_before_progress`), so a kill at their first reach leaves written items and no progress and the resume answers `target_not_empty` (the rule of 5.3 step 7, asserted); the re-application path (MongoDB's E11000 suppression of the one unrecorded file, Cassandra's re-applied batch) is run by killing at the point's second reach, after an earlier file's or batch's progress is recorded (the script's `--second-reach`): on MongoDB a 24,000-item backup in three files resumed from file 1 with file 0 recorded and reached ACTIVE with the exact count (file 1's 8,227 items re-applied as duplicates, 7,544 applied by the resume); on Cassandra a one-file backup of 12,000 items killed before the second batch's progress write resumed from item 10,000 and reached ACTIVE with 12,000 (2,000 applied by the resume), and `restore_after_file_commit` resumed from file 1 to the exact count. A Cassandra server's CreateBackup is the in-catalog format (section 3), so a Cassandra kill point takes its backup on a SQLite source, copies the prefix under the Cassandra target's root and seeds the target's `backup_index` row as the matrix does; the single-node Cassandra of the harness needs `replication_factor = 1` in the server's configuration, as `init` writes it, since under the default of 3 every lightweight transaction and quorum read on a keyspace the server creates answers "No nodes available in query plan", which is what had blocked these points. The feature chain for the backends' pause points: `extenddb-app/backup-test-hooks` → `extenddb-server/backup-test-hooks` now pulls `extenddb-storage/test-hooks` beside `extenddb-engine/backup-test-hooks`, so a hooks build compiles `extenddb_storage::backup_catalog::restore_pause_point` and each backend's own point (`restore_sqlite_mid_batch`, `restore_cassandra_before_progress`, `restore_mongodb_after_insert_many`, `restore_postgres_before_progress_delete`, each a `pub const` beside `restore_pause_point` in `extenddb_storage::backup_catalog`, the one spelling the backends and the kill-point script use) in without naming a backend crate (the server depends on none directly); the engine's five restore points are its `KillPoint` variants (`RestoreAfterDownload`, `RestoreAfterFileCommit`, `RestoreMongodbBeforeWorkerFlip`, `RestoreBetweenFailedAndRemoval`, `RestoreFencedOut`), and that enum is the engine's list, not the whole list. Corrections made before PR 9 merged: (1) the restore takeover writes `heartbeat_at = now()` in the one conditional write with the generation and `spool_host`, on every backend, so a second taker finds the record live at once; the backup takeover keeps leaving it (4(e) states why the rules differ); (2) `LOAD_CHUNK_BYTES` is summed from the decoded line's length (the decoder's `bytes_decoded` across one item), not `item_size_bytes`; section 7's terms are restated in that unit; (3) the download retry bound is per job: `download_all` takes the job's download start once and every file's retries count against it; the module doc, 5.3 step 3 and this item say so; (4) a job that learnt the record is another's (a lost claim, a DONE or FAILED write that matched no record) leaves its spool entry to the reconciler's sweep; (5) the `.verified` marker holds the file's sha256 and `is_verified` compares it with the manifest's; `Entry::open` removes an entry whose record's `spool_host` was another host's before use; the resume reads the manifest from the spool only when the record named this host before the takeover (it read the record after the takeover, which had just written this host's name); (6) the decoder checks the heartbeat's pre-write rule (`confirms_the_claim`) at every chunk boundary beside the shutdown token; MongoDB and Cassandra re-read the record per batch; (7) a restore's `claimed_at` is the instant `claim_restore` returned, the record's `heartbeat_at` having been stamped inside the registration. Cassandra's takeover re-applies the `active_restores` entry when absent; the SQLite and Cassandra batch is bounded by `RESTORE_BATCH_BYTES` (32 MiB of item text) beside `RESTORE_BATCH_ITEMS` (10,000), both in `extenddb_storage::restore_engine`; MongoDB and Cassandra read the record before each write batch; `sweep_restore_progress` deletes from a positive list; the backends' pause-point names are `pub const`s beside `restore_pause_point` in `extenddb_storage::backup_catalog`, one spelling each (`RESTORE_SQLITE_MID_BATCH`, `RESTORE_MONGODB_AFTER_INSERT_MANY`, `RESTORE_CASSANDRA_BEFORE_PROGRESS`, `RESTORE_POSTGRES_BEFORE_PROGRESS_DELETE`), and the engine's `KillPoint` is its own list; Cassandra's `count_restore_target` reads the catalog first and answers `TableNotFound` for a key that is not a CREATING restore target with that id, as the other three do; the MongoDB registration stamps the record's heartbeat from the server's clock inside its transaction (the registering process's clock is replaced; the test registers from a process an hour behind); a duplicate primary key fails the file with `data file : holds a primary key the target already has (the backup holds a key twice): `, the engine naming the file by its key once and the backends naming no file; SQLite's startup line for an aged RUNNING record names the reconciler's duty per kind. The reconciler's store check runs first in the cycle, so the manifest arm reads this cycle's health, and the skip line names `not yet checked` for a store whose first check has not recorded; a maintenance-held dead manifest restore is counted once, in `manifest_restores_failed_and_removed`; `RestorableKeySchema::resolve` runs in the request with step 2's refusals; the request's second table-name check and second `check_supported` are gone: the parser's identity rule and version gate run first; the engine's kill points log `job reached kill point `; each backend's own point is named beside `restore_pause_point`; `finish_restore`'s `Pending`, the DONE write's `NotMatched`, the pre-write claim checks of `run`, the download's pre-attempt check and `WorkerErrorCount{RestoreJob}` on a FAILED job have engine tests; the #384 override tests run through both paths for every case with a definition. The wire test `test_create_backup_of_a_vector_table_is_refused` is `test_vector_table_backup_refused_in_catalog_and_restored_from_store`, since it asserts the round trip on a server with a store; the multi-part-key wire cell lives in `tests/test_backup_job.py` on a deployment that enables `allow_multipart_table_keys`, compares the restored item set with the source's `Scan` and makes no `GetItem` by the whole key (the preview's item paths address such a table by its first HASH attribute on every backend: a `GetItem` by the whole key finds nothing on PostgreSQL and the first HASH key's one item on SQLite, restored or not). Corrections made before PR 9 merged, second set: `extenddb backup remove-target --account
` ships with PR 9 rather than PR 10 (the PR 10 section's clause says so): the `target_not_empty` rule was written for a catalog restored from a dump that names a restore which finished and was written to since, but its common trigger on MongoDB and Cassandra, which record progress after the write, is an ordinary crash before the first progress record (the whole first data file on MongoDB, the first 10,000 items or 32 MiB on Cassandra: the kill leaves written items and no progress), and without the command this release had no way to remove such a target (DeleteTable refuses a restore target, the FAILED-target duty skips the reason, DeleteBackup and a second restore of the backup are `BackupInUseException`); the first-reach runs of `restore_mongodb_after_insert_many` and `restore_cassandra_before_progress` are that state, so the PR 9a acceptance line, the PR 9a load paragraph, the testing summary and the differences doc's restore-interruption row name it as the third outcome beside "finishes with the right count" and "gone", where they had named two; the command is the one section 4's duty list describes (the maintenance claim, a FAILED record's target whatever the reason, a RUNNING record's dead target, a legacy record's target, the three refusals, no store needed, the audit line), its tests are the engine's maintenance tests and one wire cell on SQLite seeded into the state at the catalog, and the two first-reach kill points run it and assert the target gone and DeleteBackup allowed. The spool sweep lists the spool before the records (the design's rule, which the shipped order broke); the spooled manifest and sidecar are written as the data files are and a spooled pair that fails verification yields to the store's; the resume's first confirmed beat is the takeover's return, taken beside `claim_restore_dead`; a `finish_restore` error is an unknown outcome re-read through the claim; `BackupStoreErrors{get}` counts every `get` the store failed; the request's and the resume's document reads are bounded (4 KiB, 16 MiB); the restored table's size is the source table's; `RestorePostgresBeforeProgressDelete` is gone from the engine's `KillPoint` (one spelling each holds: the four backend names are the storage crate's constants and nothing else); a registration that fails after the request stopped waiting is logged. The restore's re-applied batch on Cassandra relies on the write path rewriting an unchanged item's GSI/LSI row instead of deleting it (#742, fixed on main by #424), and the `restore_cassandra_before_progress` second-reach run reaches ACTIVE with the GSI count exact where before #742 was corrected it was one batch short. PostgreSQL's `finish_restore` deletes the progress rows after the flip's commit and does not fail the restore if that delete errors (logged and swept). A file's PostgreSQL progress row is a plain INSERT from `start_at == 0` and an upsert from `start_at > 0`. Two concurrent registrations of one backup leave one target on Cassandra (the registration LWT's `active_restores = null` is the fence). What bounds a registration's duration per backend (the advisory/`FOR UPDATE` lock, the slot, `transactionLifetimeLimitSeconds`, the LWT timeouts), so the `claimed_at`-minus-`heartbeat_at` gap can never reach the stale window, replaces the earlier "conservative" in 4(e) and in this item, which had the direction wrong: an instant later than the stamp makes the pre-write check admit a beat older than two periods by the gap, so the gap needs a bound, not a name. The SQLite restore's published memory figures are the no-index shape; a table with indexes costs SQLite's page cache (`cache_size`) plus arena fragmentation, bounded, not the restore's buffers (the record's index table: 143 to 225 MiB at 100k and 165 to 172 MiB at 300k with two GSIs on SQLite, 23 to 34 MiB on PostgreSQL), and the 60,000-item shape of the differences doc's in-catalog timings was measured store-backed on PostgreSQL, SQLite and MongoDB (0.67, 1.85 and 3.74 s to ACTIVE), the row now labelling which path each figure is. The MongoDB CI jobs' `EXTENDDB_TEST_BACKUP_STORE` is `filesystem` from this PR (`devtools/run-mongodb-tests` serves a filesystem root), where it was `none`; the Cassandra job's stays `none`; the runner's `--backup-store none` is the one way to run the legacy LSI fallback cell, the in-catalog restore path's one wire cell on MongoDB, which no CI job runs. The PR 11 line owns the two unmeasured cells (the MongoDB and Cassandra 1M restores, the vector cell) and no longer the nightly schedule, shipped here. The kill-point cells as run on the pushed head's hooks binaries (4c353191), every ACTIVE outcome asserting the base table's and the GSI's `Scan` counts: `restore_after_download`, `restore_after_file_commit`, `restore_between_failed_and_removal` and `sigterm` on all four backends; `restore_sqlite_mid_batch`; `restore_postgres_before_progress_delete`, `restore_fenced_out` and `pg_terminate_backend` on PostgreSQL; `restore_mongodb_after_insert_many` at its first reach (`target_not_empty`, then `remove-target`) and its second (the re-application, 24,000 items in three files); `restore_mongodb_before_worker_flip`; `restore_cassandra_before_progress` at its first reach (`target_not_empty`, then `remove-target`) and its second (12,000 items, the GSI exact); the Cassandra points with a SQLite source. Rerun on the build after the last corrections (the first beat before the reply, the Cassandra re-read and fence, the PostgreSQL vector drop): `pg_terminate_backend` as reworked (the lost claim at the next check 20 s after the termination, the record RUNNING with its `(0, 8228)` progress row, the takeover 45.6 s after the termination and 46.7 s after the last beat, ACTIVE 24,000 on base and GSI), `restore_after_file_commit` and `restore_mongodb_before_worker_flip` on MongoDB (resumed from data file 1 after 0 of its items, 3,771 applied by the resume, DONE then the worker's flip, 12,000 exact; DONE before the kill and the restarted worker's flip after it), `restore_cassandra_before_progress` at its second reach (resumed after 10,000, 2,000 applied, 12,000 on base and GSI) and `restore_between_failed_and_removal` on PostgreSQL with a vector backup (the target gone after the stale window, its data table and `_ddb_vec_` relation gone with it, where the pushed head's build left the relation). The matrix's four `--duplicate-key` and eight `--vector` cells ran on the final binaries. Corrections made before PR 9 merged, third set. The job's first heartbeat is sent right after `claim_restore` returns and before the reply, and the pre-write check counts its two periods from that beat's return; a `Lost` or failed first beat fails the record ("the claim was lost before the job started") and answers `InternalServerError`. `extenddb backup remove-target` decides a RUNNING owner dead through `list_dead_restore_jobs(owner_window)` on the database's clock, and the purge's wait decides a backup row's owner live through `list_backup_rows_older_than` on the heartbeat clock; neither reads a host clock. The reconciler takes nothing over and writes no FAILED on its manifest arm once the server's shutdown has begun. Under a maintenance claim a dead record with no progress over a target that holds items is failed `target_not_empty` through `fail_restore_job`, not `OWNER_DEAD`. The reconciler's `target_not_empty` lines print `extenddb backup remove-target --account
`. The restore job's stop line names the heartbeat timer's reason when the timer raised the loss, and the timer logs the reason at WARN. A DONE write whose answer was lost and whose record re-reads DONE under the job's generation ends with "complete (DONE)", not "ACTIVE". A spool document write that fails and a resume that fails before it runs remove this host's entry. The engine's tests of the 60 s bound, the slot held to the job's end and the re-reads' generation checks are cells of their own. Cassandra's progress lightweight transaction re-reads an unknown outcome (applied, one retry, or the lost claim; a second unknown outcome fails the restore), where the earlier text said the restore fails and the reconciler resumes it, which the FAILED write's removal made false. The Cassandra registration's refusal path conditions the target's removal on its own generation; a record a reconciler took over during the DDL keeps its target. PostgreSQL's `abort_restore` drops a vector index's relation with the data table. The PostgreSQL record's `heartbeat_at` is the registration transaction's start. Tests: `test_an_unknown_progress_write_outcome_is_decided_by_a_re_read`, `test_a_registration_taken_over_during_its_ddl_leaves_the_successors_target`, `test_a_mid_file_resume_records_start_at_in_every_batch_progress`, `remove_restore_target_drops_the_vector_relation_with_the_data_table`. The engine's `KillPoint` has five restore variants, named above. SQLite's takeover clock is `strftime('%Y-%m-%dT%H:%M:%fZ', 'now')`, the `NOW` constant of its catalog (4(e) names it). `extenddb backup remove-target` refuses a RUNNING record with a live owner by its own text ("stop it or wait for it"); the DONE and no-record refusals are the two "not a restore target" ones. The published 1M figures are the key-ordered shape's and the record says so beside them, with the same items measured as an unordered backup (SQLite 144 to 164 MiB over idle in 66 to 90 s against 87 to 98 MiB in 27 to 56 s key-ordered; PostgreSQL 16 MiB either way), and SQLite's final `COUNT(*)` is the counting connection's page cache, bounded by `cache_size` (32 MiB) and the same size at 1M and at 3M rows (22 to 28 MiB), where the record had called it the one term that grows with the table from a single measurement. The `pg_terminate_backend` cell of `kill-points.py` pauses the job at `restore_after_file_commit`, terminates the claim session while it is paused (psql must answer a row), waits past one heartbeat period, releases, and asserts the lost-claim stop line, the record RUNNING with its progress row, the takeover no earlier than `stale_job_after_secs` after the owner's last beat and the exact counts; the shipped cell had terminated the session after a fixed sleep and asserted the final count alone, which a 2.6 s restore satisfied without the case ever running. The script's `--vector` switch gives the source table a vector index, and the tamper cell on PostgreSQL then asserts the removed target's data table and `_ddb_vec_` relation gone. `.github/workflows/matrix-nightly.yml` installs `postgresql-client` and checks `psql --version` and the two drivers' import before the cells; it has had no run, since `workflow_dispatch` needs the file on the default branch, so its first run after the merge is the proof of the runner image. The upgrade manual counts migrations 1 to 7 as breaking and migration 8 as declared breaking on PostgreSQL for want of the additive-declaration job (on SQLite, MongoDB and Cassandra it does not apply). The body sentences each of these corrects are edited with this item (section 1's duplicate rule and Decimal128 bullets, section 3's PR 9a interface list, 4(b)'s progress members and metric name, 4(e) and 4(i), 5.2, 5.3's preamble and steps 1 and 3 to 7, section 7's restore memory sentence, section 9, section 10's table, the PR 6, PR 8, PR 9a and PR 9b sections, the PR 10 clause naming what the seed stands in for and the command shipped here, the PR 11 line, the testing summary and the review checklist); earlier items keep their wording, as item 22 requires. With the rebase onto #666's head (2026-11-03): the jobs and the reconciler are not gated by the PostgreSQL serving registration, and 4(e) says what they do in the registration's retry window, GitHub issue #755 being the gate; `backup purge`'s own cycle under the SQLite serve lock fails a dead store-backed restore and removes its target rather than resuming it (4(f)), since the command tells the cycle it holds the claim instead of the cycle reading it through a zero window; the PR 9a load paragraph gives PostgreSQL's batch as 500 rows or 4 MiB, as 5.3 step 5 does; and MongoDB's snapshot open waits, at most 2 s, for the majority-committed snapshot to show a table a `local` read shows ACTIVE, so a store-backed CreateBackup sent as soon as DescribeTable says ACTIVE is not refused as a table that does not exist. With the rebase onto `main` (2026-11-04), after #335, #666 and #681 merged: `restore_progress` was migration 7 (4(i)), declared not applicable in the MongoDB and Cassandra registries #681 added, as on SQLite; 4(e) cites the module doc of `crates/server/src/registration.rs` for what a lost serving registration does, which replaces this item's 2026-11-03 clause on it: the reconciler stops with the server's workers and spawns no resume once its own token has fired, and the running jobs go on, #755 being the gate for them; a restore job joins PR 8's job drain as a backup job does; and `remove-target` releases its serving registration when it exits and is named on MongoDB and Cassandra as on PostgreSQL. With the move onto `main` at e8ca7df3 (2026-11-04), after #424 merged: the Cassandra write path's rule for an unchanged index row (rewritten by its INSERT alone, #742) and its version stamps on every index row are #424's, so this PR no longer changes `sync_indexes` or the GSI worker; the PR 9a load paragraph, 5.3 steps 5 and 6 and this item cite #424 for what a re-applied batch relies on, and a restored item's index rows carry its base row's `version` as #424's write path gives them (`put_restored_item` is that path). After the second review round (2026-11-05): the Cassandra load no longer goes through `put_restored_item`'s lightweight transaction but writes each item's base and index rows in one logged batch `USING TIMESTAMP` a reading taken before the batch's claim check (`put_restored_item_at`), the fence against a batch a stalled writer lands after a takeover (the PR 9a load paragraph); the index rows still carry the base row's `version`, which is now that timestamp. A resume with no recorded progress fails `target_not_empty` only when the target holds more items than the backup's row counts; at or below the total it re-applies the first file with upserts (step 7), so a Cassandra crash before the first progress record resumes to ACTIVE. One instance's sixth takeover of a record fails it (`RESTORE_TAKEOVERS`, step 7). `restore_target_ready` on Cassandra checks each GSI's and LSI's table as well as the data table. The nightly workflow runs every restore kill point of every backend in four jobs (the testing summary); the kill-point script's first-reach cells assert ACTIVE with the exact count and its `--remove-target` flag is gone (the `target_not_empty` state is seeded by `tests/test_backup_job.py`'s `remove-target` cell). With the rebase onto `main` after #671 and #672 merged (2026-11-06), `restore_progress` is migration 8: #819 took 7 for PostgreSQL's `stream_pending`, so the SQL file is `0008_restore_progress.sql`, each backend's arm follows `StreamPending`, and CONTRIBUTING's worked example is migration 9. 34. (2026-11-04, PR 10 as implemented, #746.) PR 10 is split in three: #746 ships plain `backup sync`, `--prune-missing`, `--adopt-store`, the `sqlite-memory` startup sync and `BackupCatalog::insert_backup_row`; #820 carries `--move-store`, `--forget-store` and `--forget-backup` with `list_backup_rows_by_root` and `rewrite_store`, which rewrite or delete rows of other stores, and the full report of the rows of other roots; #821 carries `migrate-legacy`, whose data path (the frozen reader, `DataFileEncoder`, the item validator) shares nothing with sync but the insert; `remove-target` shipped with PR 9 (item 33). The PR 10 section's line, the PR 9b paragraph (the matrix now adopts with `backup sync`) and the doc-row table's `migrate-legacy` row say so (PR 9b, PR 10, section 10). Neither the Cassandra sweep of orphan `backup_index_by_status` entries nor the MongoDB sweep of the data collections a failed restore-target removal leaves is in any of the three: the code comments that gave them to `backup sync` now say no sweep removes them yet, and the MongoDB error line names the collections for an operator to drop. `insert_backup_row(row) -> bool` inserts the row as given, unowned (`owner_generation` 0), with `completed_at` and `heartbeat_at` on the database's clock, and answers `false` when a row with its key exists: PostgreSQL `INSERT ... ON CONFLICT DO NOTHING` with no conflict target, so both unique indexes (the key and `backup_id` alone) refuse it, on the pool and with no advisory lock; SQLite `INSERT ... ON CONFLICT DO NOTHING` under the write lock and `BEGIN IMMEDIATE`, without a job slot; MongoDB the claim's pipeline upsert filtered on `owner_generation: {$exists: false}` with the row's status, `owner_generation: 0`, and E11000 (the unique `backup_id`) meaning a row exists; Cassandra the `backups_by_arn` lightweight transaction `IF NOT EXISTS`, then the LOCAL_QUORUM batch of the two list rows and the `backup_index_by_status` entry under the row's status (the batch `claim_backup` wrote inline, now shared), an unknown outcome or a `NotMatched` answer re-read at `LOCAL_SERIAL` and settled as inserted when the row read is this one (its entries then written, which completes an insert that stopped between its two writes), as existing when it is another; no row after a lost answer is sent once more, and no row after `NotMatched` answers `false` (a legacy in-catalog backup holds the ARN); a failed batch of the entries is sent once more before the error. The row is AVAILABLE from the transaction on, so a DeleteBackup between the transaction and the batch moves a status entry the batch has not written yet, and the batch then writes an AVAILABLE entry no later write removes; every reader of the status table re-reads the row and skips it, one wasted read each, until a sweep of orphan entries exists. Cassandra has no `backup_id` uniqueness, so its insert-if-absent is per ARN spelling: the same id under another region is another row there, as the trait doc says, and sync closes the gap itself by reading the row under both spellings of the manifest's ARN (this server's region and the manifest's) before the insert and skipping a backup the manifest's spelling, when it is another region's, has a row for, while a row under this server's spelling, the one the insert writes, is left to the insert (which answers `false` for it or, on Cassandra, completes its own torn insert), so through sync no account holds two rows of one backup id under the manifest's account, table and id on any backend (section 3, 4); on Cassandra a row of that id under another table that the rows read did not return (a row of another root, or one whose list entries are missing) is not seen by that read, and within one run the row check keeps a second prefix of an id the run adopted out. Sync reads the configured root's rows first, every status and every store id the catalog records for the root (`read_store_ids`, then `list_backup_rows` per id), keeping of each only the status, store id and table by account and backup id and the key of each AVAILABLE row of the configured store id (the backends collect the rows whole first, so the read peaks at about 22.5 KiB per row of a five-GSI table, section 7), and checks each listed prefix against that set before it reads anything under the prefix; the sync contract's "checks each prefix against the catalog as the listing reaches it" holds with the rows read once up front, and the insert-if-absent then decides against a CreateBackup that landed after the read and against rows of other roots. The listing is consumed as it streams: the store lists in key order, so a prefix is complete once the listing passes it, and sync holds that prefix's objects and the prefix of each manifest it listed; the filesystem store's own listing is the larger cost, measured in section 7. A listing that fails part-way keeps the rows inserted before it, does not read the prefix it was in, and makes no missing-manifest report. An insert that answers `false` is re-read to say which row it met: one that landed after the read, one of another store, account or table holding the backup id, or on Cassandra a row whose list entries are missing (a CreateBackup's claim that stopped between its two writes; sync's own insert that stopped there is completed by the next sync of the prefix). The prefix check is the dead-owner duty's: `reconciler::verify_prefix` reads the manifest and sidecar within their bounds, checks the digest and the reader's acceptance and every listed data file at its size from the listing sync already holds; the dead-owner probe compares the manifest's ARN with the row's apart from the region (a backup adopted from another region's manifest carries this server's region in its row), and sync compares the manifest's account, table and backup id with the path, adopts a manifest of another region under this server's region's spelling of the ARN, logging it (requests resolve it under either spelling), and skips one of another partition than `aws`, since every request refuses such an ARN before it reads a row. `layout.rs` gains `EXPORTS_COMPONENT`, `CHANGELOG_COMPONENT`, `classify_store_key` and `parse_store_prefix`: keys outside every backup prefix are passed over, the reserved components and a third component of another id shape (only the 14-digit shape carries a manifest) are reported as skipped and never read. The adopted row carries the manifest's name, ARN, size and item total and `source_table`, the id's instant as `created_at`, and the configured store's identity (section 4, PR 10). Rows of other roots are reported in part: sync prints the prefixes whose backup id a row of another root holds (the read under the ARN's spellings, or the insert's re-read). The rows of the configured root under another store id are not a case sync meets: a store whose rows carry another id than its file, or more than one id, is unhealthy, and sync refuses an unhealthy store. A listing of every root's rows needs `list_backup_rows_by_root`, which is #820's, so until then 4(g)'s "rows for another store are reported" holds for that one case (4(g)). Every sync takes the maintenance claim where it runs beside servers (PostgreSQL, MongoDB, Cassandra), plain sync with no wait for running jobs: its insert-if-absent orders it against a CreateBackup, and the claim orders it against `purge`, whose `purge_row` removes a dead CREATING row before it deletes the prefix, which can hold a complete manifest (a job that died after its manifest); a sync between the two steps would adopt bytes purge then deletes. The commands' claim (`HeldClaim`) is confirmed by the job's rule: each beat is bounded by one period (`MaintenanceClaim::heartbeat_within`, whose bound on SQLite starts once the beat holds the engine write lock), the claim reads lost once its last confirmed beat is two periods old, and a beat in flight counts only where it parks before its bound (`heartbeat_parks_before_its_bound`, SQLite); before each step that cannot be undone the command sends one beat of its own (`HeldClaim::recheck`) and stops if it finds the claim lost: sync's insert, `--prune-missing`'s removal, `--adopt-store`'s rewrite of the identity file, and purge's removal of a dead CREATING row and the deletion of that row's prefix. On MongoDB and Cassandra, which take a claim over on the heartbeat's age alone, a command stopped (SIGSTOP, a paused VM) past `stale_job_after_secs` can lose its claim to another command; it then stops before its next such step, so sync and purge never act at once while each reads its claim held. What remains is a stop after the re-check confirmed the claim and before the step's own call reached the catalog or the store, or with that call already sent: the call lands when the process resumes. While the claim is held the jobs refuse, as during purge (4(f)), and a sync killed while it holds the claim keeps the jobs of every instance refused until its heartbeat is `stale_job_after_secs` old. The restore's line for a row whose prefix lacks its sidecar says to put the prefix back; only a missing manifest names `--prune-missing`, the one case it removes. On SQLite it opens the catalog as `serve` does, under the serve lock (exit 6 beside a server, the refusal purge gives), and plain sync runs no dead-owner resolution: the lock keeps every server and every other command off the file, which is all the PR 10 rule that every writing command refuses to run beside a server asks, so nothing changes between the row read and the insert. The rows a stopped server left (a CREATING or DELETING backup, a RUNNING store-backed restore) are rows sync skips, and the next server start resolves the backups and resumes the restore; the dead-owner resolutions purge runs would fail that restore (`OWNER_DEAD`, under the claim no resume is spawned) and remove its target. `--prune-missing` and `--adopt-store` take the claim on SQLite too and run those resolutions before their wait, as purge does, so they fail such a restore and remove its target; every command that runs them prints what they did, the failed restores included. `--prune-missing` and `--adopt-store` also wait for running jobs as purge does; every sync installs purge's signal handling, checks it before each prefix and exits 1 "stopped" on a signal, releasing the claim it holds and its serving registration. `--prune-missing` removes each AVAILABLE row whose manifest is gone outright, through `BackupCatalog::forget_available_backup_row` (added here on the four backends: under the row's lock as `start_backup_delete` takes it, the status check and the in-use rule with a delete of the row, and on Cassandra its index entries after it), and deletes nothing in the store. The manifest probe sees only the newest AVAILABLE row's manifest, so a host-local root that holds the newest backup passes it while it lacks older ones; with no tombstone left, no deletion duty on the host that holds those bytes acts on them, but the row is gone for every instance: the backup is BackupNotFound on every instance until a sync on the host that holds the bytes adopts it again, which is the host-local acceptance line as amended here. The objects a removed row's prefix still holds stay until an operator deletes the prefix by hand: `purge` deletes only the prefixes of rows, and every later sync reports the prefix as not a complete backup. `--prune-missing` exits 1 when it leaves a row a later run may remove (a store or catalog call failed, the manifest's `head` included, which keeps the row; or a restore is reading the backup, "run the command again once the restore ends") and 0 when the rows it leaves have their manifest back or were moved by another writer. An unhealthy or `checking` store refuses plain sync and `--prune-missing`, exit 1, naming the problem and changing no row. Sync exits 1 when a prefix was skipped for a store read or catalog call that failed (the run is incomplete; the next run adopts it), and a startup sync that met one stays pending, its skips logged at WARN on every run and its other skips at INFO until a run has listed the whole root and logged its line of counts (a first run cut at the bound or stopped by an error does not count) and at DEBUG after, so a retried sync of a large root does not log a line per prefix each cycle; the command's summary line reads `incomplete` for such a run. So when the newest AVAILABLE row's own manifest is gone the store is unhealthy and neither command runs; the way out in this release is to put the prefix back (#820's `--forget-backup` is the other). The serving registration is named `extenddb backup sync`, and the catalog refusals exit 6 and 3 as the other commands' do (4(f), PR 10). `--adopt-store` is `store_identity::adopt`: with the claim held it rewrites `extenddb-store.json` with a plain `put`, the file's `catalog_id` and `backend` changed and its `store_id` and `created_at_ms` kept, reads it back, and syncs with the adopted identity; it logs an `extenddb::audit` intent line before the put and the audit line after the re-read with its outcome (adopted, or race lost), so a put whose caller stopped waiting is on record too. The re-read detects another adopter's put that landed between this put and the re-read (the store trait has no conditional replace); the command reads the file once more after the sync and fails naming the owner when it changed, which detects one that landed during the sync; a put after both reads is found by this catalog's next check. It refuses when this catalog's rows for the root carry another store id, and on a root this catalog owns it says so and syncs. A later check of the previous owner that can read its catalog finds the file naming another catalog, at the earliest one reconciler interval after its cycle in progress ends; until then it acts on the root, except where its deletion fence stops it. `StoreFence` re-reads the file (one `get` through the store's bounded calls, under its check bound and, on a filesystem root, in a call slot of its own, so a deletion holding the root's call slot does not hold it) right before the deletion of a backup's objects (`finish_backup_delete`, which DeleteBackup and the reconciler's DELETING duty run, one read before its two `delete_prefix` calls, the backup job's failure path, the FAILED-prefix and DELETED-prefix duties, purge, whose one read comes before its row change and its two deletions), before DeleteBackup's DELETING write and before purge's removal of a dead CREATING row or its DELETING write, before the reconciler removes a tombstone, before a CreateBackup's row (at the request) and again in the job right before its first put; when the file names another catalog it refuses, records `ForeignCatalog` so the store is unhealthy at once (and a file naming this catalog under another store id than the store's check settled, a root replaced by another root of the catalog, is refused and recorded as `IdMismatch`; DeleteBackup and CreateBackup answer the unhealthy store's `ValidationException`, and the cycle re-tests the store's health after each store duty and runs none after the refusal, the dead-owner duty included), a root with no file, a file the store refuses to serve and one that does not parse are refused and recorded as the next check would record them (`FileMissingWithRows`, `Denied`, `FileUnreadable`, `Validate`), so CreateBackup answers the unhealthy store's `ValidationException` for them too and a job refused before its first put is FAILED with no object; and a read that fails in transport refuses the call and leaves the health to the next check (CreateBackup answers `InternalServerError`), except at DeleteBackup's read before the DELETING write, where it is a store failure: the request writes DELETING and answers DELETED, as for a deletion the store does not answer (revision item 30), and the read before the deletion refuses it. A refusal before the row is changed leaves it as read, so DescribeBackup does not answer a refused DeleteBackup with DELETED; one at the read before the deletion (an adoption between the two reads) leaves the row DELETING with nothing deleted and DeleteBackup answering DELETED (revision item 30), which this catalog's reconciler finishes only if this catalog owns the root again. The restore deletes nothing in the store, so it has no fence, and neither has `--prune-missing`'s removal of a row, which deletes no object. A write already past the fence's read lands, so every server of the catalog the file names is stopped before an adoption, and a new in-memory server is started only after the old one has exited; the admin guide, the refusal and the command's output say so (4(c)). For `backup sync` a defaulted SQLite root whose file names another catalog while this catalog has no rows for it is configured and unhealthy (`Caller::SyncCommand`) instead of the warning that leaves it unconfigured (item 29), so `--adopt-store` can adopt the root a catalog re-created by `destroy` and `init` finds; the server and purge keep the warning (4(c), section 7, PR 7). The identity file gains `ephemeral_catalog` (`#[serde(default)]`, so the format stays version 1 both ways), written `true` by a server whose SQLite database is in memory (`database_file()` names no file: the state, not the `sqlite-memory` cargo feature) and `false` by every other writer, an adoption included. The adoption of a root an in-memory catalog owned is part of `store_identity::establish` when this catalog is itself in memory and the file carries the marker, and never of `recheck`, so two in-memory servers on one root do not take it from each other each cycle; a root whose file names a durable catalog stays unhealthy for an in-memory server, as 4(c) requires. The startup sync runs on every start of an in-memory catalog with a store configured, not only after an adoption (on a fresh root it is one listing): for a filesystem store in `serve` before the routes serve; for an S3 store, and for a startup sync not finished, in the reconciler's cycle after a healthy check. Every run is bounded at 60 s (`sync::STARTUP_SYNC_TIMEOUT`), in `serve` and in the cycle, where the duties after it wait at most that bound; a run that stops before it listed the whole root, reaches the bound (the rows it inserted stay, but every run lists the root from its first key, so the sync completes only while listing the whole root fits in the bound; past it the sync stays pending and is retried every cycle), or skipped a prefix for a store or catalog error stays pending. On both store kinds the run's listing is collected whole before its first key reaches the run (`BoundedCalls::list` collects the stream into one `Vec` of `ObjectMeta`, one per object of the root, held in memory until the run ends), so a listing that outlasts the bound reaches no prefix; on S3 the abandoned listing's requests end with the dropped future. On a filesystem store a run abandoned at the bound does not stop its listing (the walk has no cancel flag): it runs to its end on the blocking pool, holding the listing it builds and the root's call slot, marked in flight, so no second walk starts beside it: until the walk ends, each cycle's check is refused at once on the mark (the refusal names the caller whose call has not returned, here the startup sync's) and records the store `unreachable`, so CreateBackup, DeleteBackup and restores are refused, the deletions wait, and the sync, which runs only on a healthy store, does not run; the first cycle after the walk ends checks the store healthy and runs the sync again, from the first key. A root too large to list within the bound is never synced by an in-memory server, which an operator resolves with a smaller root or a durable catalog. `serve` answers nothing, `/health` included, until the startup sync before the routes ends, while a signal in that time ends the process at once. Until a run finishes, requests for the earlier backups answer `BackupNotFoundException`, `/health` possibly already healthy; the log line that reports the adopted count ends that window, and for a prefix that line reports as having met a store or catalog error, a later cycle's line does. It takes no claim: the process is the catalog's only user (4(c), PR 7, PR 10). Every `extenddb backup` command refuses an in-memory SQLite database before it opens a catalog, exit 1, with a sentence saying why (a location that does not parse is reported with its parse error instead, as `serve` reports it): it would open a new, empty catalog that ends with the command, and a writing command would hand an adopted root to a catalog nothing can open again; this is the sync contract's "the backup CLI commands do not exist there" (PR 10). The matrix (`devtools/backup-restore-matrix`) copies the source's prefix under the target's root and runs `extenddb backup sync` on the target (on SQLite with the target's server stopped) in place of the hand-written row, so a cell proves the adoption and the restore; `--adopt-store` points the target's store at the source's root, adopts it and checks that the source's server then reports the store owned by another catalog, and the nightly runs that cell from SQLite to PostgreSQL and back (PR 9b). 4(c), 4(e), 4(f), 4(g), decision 6, section 7's memory paragraph, the PR 10 contract, 5.3's failure step (`OTHER_STORE`'s reason, whose text this PR changed) and the testing summary's kill-point line (a Cassandra kill point adopts its copied prefix with `backup sync`) are edited with this item. + +35. (2026-11-05, PR 11 as implemented, #747.) The CI filters: `--filter "not import_export"` came with the PostgreSQL integration job's first version (687e46ff) and was copied into the SQLite job (556c501e); neither commit gives a reason. A foreground server wrote a PID file in every tree up to e0d6a0af, which made foreground mode write none by default and added `--write-pid-file` to the job's start in the same commit; 556c501e came after e0d6a0af, and its SQLite job passes the flag. So the restart `devtools/run-tests` makes for the import and export tests could stop the job's server in every tree, and the filter was not working around it. The MongoDB and Cassandra jobs already run `tests/test_import_export.py` unfiltered (`--pytest --comprehensive --terraform --parallel`). The expression selects out the ten tests of that file and nothing else (none of the comprehensive suite, the CLI lifecycle list or the backup-store step); run the job's way on release builds of this change they passed 10 of 10 on SQLite and on PostgreSQL, before the rebase onto main 887b6e26 and again on eb9e48de after it, so both filters are dropped and no narrower one is kept. The open cells (testing summary) are measured in `15-backup-restore-bench-pr11.md`, in `-pr9.md`'s method, none left open; the Cassandra cells ran on a single 4.1 node of their own with the Cassandra integration workflow's service settings (`PasswordAuthenticator`, a 1 GiB heap), since a node running `AllowAllAuthenticator` cannot hold the role `extenddb init` creates. The cells and the later runs on the same host with other harnesses used release builds of earlier versions of this change on PR 10 at 358ff8bd (main 94b5f689): 358ff8bd's Rust sources (30553b38), and 04c29155 and f78e9bc1, which add the help text of two `init` options. The rebase onto main 887b6e26 brought main's later changes to the backup and restore code (#700 and #740 as merged, and #474), so none of those builds has this change's Rust sources. The later runs reproduce the cells in kind and are in the record, with a control of items of 2.5 KB strings run on f78e9bc1; the 100k cells, run again on release builds of eb9e48de (the change after the rebase), reproduce in kind, the peaks within 8.4% and the durations up to 32% longer at a host load of 5.3 to 11.0 against 1.3 to 8.5 for the earlier cells (the record's "Reproduced on the rebased tree"; no 1M cell was run again). Findings beyond the figures: SQLite's vector restore under glibc's default arenas is higher at 1M than at 100k (552 against 354 MiB over idle; the 1M run levelled off after about 150 s, and no other size was measured) and is 195 and 204 MiB under `MALLOC_ARENA_MAX=1` (flat within about 25 MiB over the measured runs), the allocator's term section 7 describes for large items; that memory is the restore's write batch (up to 10,000 items or 32 MiB of item text) and decode chunks, not the vector index, and grows with the item's size up to the batch bound whatever the attribute types (100k items of about 2.5 KB of strings held 253 MiB, 130 MiB under one arena), lists of numbers decoding larger at the same text size (345 to 424 MiB and 184 to 195 MiB at 100k items of 128 numbers, with or without the index; 20,000 items of 1,536 numbers 669 MiB and 301 to 316 MiB under one arena), while on PostgreSQL the index adds 27 to 53 MiB (five pairs of runs with and without it, each from one harness; a pair whose two runs came from different harnesses is left out); one arena cost the 1M restore 32% in duration and two 100k restores running at once 131% (57.5 against 24.9 s); the vector index is about 20% of the 100k restore's duration on SQLite and 33 to 47% on PostgreSQL (the same pairs); MongoDB's restore becomes `ACTIVE` at the backfill worker's next cycle after the load (`GSI_BACKFILL_INTERVAL`, 5 s, which the restore's end does not wake), so its duration is the load's plus up to 5 s, 3.9 to 8.9 s at 100k over five runs; the peaks of one cell varied by up to about 20% between runs; and a whole-table `SearchVectors` on SQLite is the exact scan of `crates/storage-sqlite/src/vector_search.rs` (5.5 to 15.8 s over 1M 128-dimension vectors in seven runs, 0.24 s on PostgreSQL). `docs/manuals/14-backup-and-restore.md` is the operator's account derived from this document, which remains the record (section 10's first row): the strategy, RPO and RTO per store kind with the three records' figures and their host class, the configuration (pointing at the admin guide's rows and the store crate), securing the store, what a running backup costs each database, the restore drill, catalog-loss recovery with `backup sync`, `--adopt-store` and the in-memory startup sync, legacy backups (with `migrate-legacy`, #821, not shipped, the restore-and-back-up-again path), what a restore preserves, and ImportTable and export interchange with Amazon DynamoDB (a store-prefix import is #748). Each PR 11 acceptance statement is in it. Figures measured for it on the same host class: a 25 s PostgreSQL backup beside 12,156 overwrites of 200-byte items in another table grew that table from 0.4 to 4.4 MiB, its dead row versions "not yet removable" by a `VACUUM` during the backup (0.5 MiB for the same writer with no backup); on SQLite the WAL grows by the pages each write changes, which depends on the indexes: a 22.8 s backup beside 15,472 overwrites of a table with no secondary index grew the `-wal` file from 4.0 to 61.1 MiB (about 3.9 KB per write), and 13,996 overwrites of a table with two GSIs during a 25.6 s backup from 4.1 to 1,294 MiB (82 to 97 KB per write over four runs, about 20 to 24 pages: an asynchronous GSI write goes through `gsi_pending` and is applied in later transactions); the same writers with no backup left it at 4.0 and 4.1 MiB, and the file keeps its grown size after the backup and after a stop. The record's "Figures measured for the manual" section carries these runs' method (the writer, the relation and file measured, the `VACUUM (VERBOSE)` 10 s in), build and load, with their reproductions on 04c29155 and on eb9e48de. Its process-level statements were checked against release builds of 358ff8bd's Rust sources on SQLite and PostgreSQL: the root placement refusals and their texts and an absent root's, the drill, a re-created catalog's `owned by another catalog` health with `[backup.store]` configured and, on file-backed SQLite with the default root, the logged warning, no `backup_store` component and in-catalog `CreateBackup` until the adoption, the accounts re-created with `manage create-account --account-id` and the old key pairs kept with `manage import-access-key`, `--adopt-store` exiting 6 beside a SQLite server and adopting once it stopped, the in-memory startup sync line and, on the `sqlite-memory,dev-mode` build, the adopted backups answering `AccessDeniedException` until their account was re-created with its old id, the in-catalog create's warning and the restore of its backup through a store-configured server, an S3 store at moto with the default spool, `backup sync` exiting 6 beside a SQLite server, ImportTable refusing a gzip file and a `FileSource.Path` through a symbolic link and importing a decompressed backup data file, the export's `FilePath` and `S3Prefix`, and on PostgreSQL the claim session found by its lock key (`hashtextextended('backup:', RESTORE_LOCK_SEED)` in the catalog database, distinct from the `extenddb serve` registration session) whose termination stopped the job at its next heartbeat and left the row to the reconciler, which failed it once stale. Checked on release builds of 04c29155 as well: an off-host copy with loose modes refused by `backup sync` (exit 1, naming the directory) and its restore failing, the store unhealthy only when the copy is the newest backup; the registration session's termination failing every store-backed backup the server ran (reason `shutdown`); `--prune-missing` waiting on a backup that failed within `stale_job_after_secs`; a catalog database restored alone beside a newer data database (tables created since absent and empty when created again, tables deleted since listed and answering `ResourceNotFoundException`); both symbolic-link refusals of `FileSource.Path`; an on-demand table's GSIs reported with 0 and 0 throughput in `SourceTableFeatureDetails`; and the PostgreSQL pair dumped with the server stopped under `umask 077` (files 0600) and restored into new databases created with `createdb -O` by the admin role, after which `CreateTable` and `PutItem` of a new table work, while a restore over the existing databases restores nothing and one into databases the admin role owns breaks `CreateTable` ("permission denied for schema public"). On release builds of f78e9bc1, on PostgreSQL 16 containers: the entry's procedure as it now stands (an older `pg_dump` refused, "aborting because of server version mismatch", exit 1, leaving an empty file, which `psql -v ON_ERROR_STOP=1` restores with exit 0, after which the server refuses the catalog as not initialized; both dumps exit 0 and non-empty; the old databases renamed aside, the restored server reading the table and its GSI and creating a table, then the old databases dropped), and on a new, empty server `createdb -O` refused until the application role was created with the dump's password, the restore, and the server, whose catalog still named the old server's address, connecting its data pool there: it put a new table's items into the old server's data database while that server ran, failed to start once it was gone, and served the restored table, with `CreateTable` and `PutItem` landing on the new server, once `data_database_connection_string` was set to the new address; and an access key imported through the management API from a request body read from a file. `init` refuses an existing config file, and `init --overwrite` regenerates it keeping only the storage connection settings, so the manual's catalog-loss steps and the admin guide's identity paragraph copy the file aside first and put `[backup]` and `[backup.store]` back before the server starts; on release builds of 925509bd (on PR 10 at be84a7eb, main b8ad9cd5), a server started without them had no store on PostgreSQL (`--adopt-store` exits 1) and a new, empty default root on SQLite (`--adopt-store` adopts nothing), and with them put back each adopted the root and restored the backup. Also on those builds: an off-host copy under account and table directories made with `mkdir -p` under a `022` umask stops `backup sync` at the account directory (exit 1) until `chmod -R u=rwX,go=` on that directory, and one under directories made with `install -d -m 0700`, or of the account directory whole, adopts and restores; and the upgrade manual's rollback on PostgreSQL 16 (dumps checked with `test -s`, which an older `pg_dump`'s empty file fails; the pair renamed aside, restored with `ON_ERROR_STOP` and the renamed copies dropped once the server had read every table). After the rebase onto main 887b6e26, which brought #840, a re-created catalog has no `allow_credential_import` value, so the import of the old key pairs answers 403 "Credential import is disabled (allow_credential_import = false)" until the setting is `true`: the manual's catalog-loss step sets it with `extenddb settings set` beside the running server and turns it off after the imports, and its in-memory section, since `extenddb settings` cannot reach an in-memory catalog, sets it with the management API's `PUT /management/settings/allow_credential_import` after each start; both checked on release builds of eb9e48de with these edits, on PostgreSQL and on in-memory SQLite (the refusal, the setting, both import routes, the setting turned off, the imported pairs signing requests, and on the in-memory server the adopted backup restored with them). The manual's other process-level statements were checked again on release builds of eb9e48de: the root placement refusals, what a restore preserves, the drill on SQLite, PostgreSQL and an S3 store at moto, the off-host copy shapes, a catalog restored from dumps and the claim and registration sessions on PostgreSQL, a catalog lost and re-created on SQLite with `--adopt-store`, the in-memory startup sync, the legacy path, ImportTable and export, the split-root remedy by hand on SQLite, the admin guide's Data Recovery entry end to end on PostgreSQL 16 and `init`'s database names. A catalog restored from a dump whose newest `AVAILABLE` backup was deleted after the dump keeps its store unhealthy, and this release has no command that clears that without a copy of the prefix (`--prune-missing` refuses while the store is unhealthy); the manual says to take the dump after deletions. The admin guide's Data Recovery entry named the PostgreSQL data database `extenddb_catalog_data`; `init` names it after the catalog without its `_catalog` suffix (`extenddb` by default) and records it as `data_database_name` (the connection string as `data_database_connection_string`), and the entry, the architecture, design, quickstart and security-model guides, `AGENTS.md`, the troubleshooting guide and the `extenddb` skill's runtime-symptoms reference (the `pg_stat_activity` query), the technical-debt register's A-1 entry, the storage-extension guide's database list and `init --help` now say so, with each backend's databases (`init --help` also says that only the PostgreSQL backend reads `--data-db` and `--catalog-db`); the quickstart's `init` table gives `--pg-user` (the admin role, `$USER` then `postgres`), `--pg-pass`, `--extenddb-user` and `--extenddb-pass` with their defaults, and its sample lines show migration 8, as #740 left them; a catalog name without the suffix needs `--data-db`, since `init` otherwise derives the catalog's own name, creates it once and fails on the second create, leaving an empty database to drop before `init` runs again (the defect is not changed here). On PostgreSQL the entry, the security model, the deployment guide's checklist and the manual treat the catalog and data databases as one copy (both dumps with the server stopped, or a base backup), with `umask 077` before the dumps, since the catalog dump holds the encryption key and the application role's password, and the restore goes into new, empty databases owned by the application role, with the old databases renamed aside until the restored server has read every table, the `pg_dump` of the server's major version and both dump files checked, and on a new PostgreSQL server the application role created first and, at another address, the recorded data-database address set; the entry's SQLite row gives the restore (the current `-wal` and `-shm` removed first), and the upgrade manual's dumps take `umask 077` and are checked, and its rollback renames the pair aside, restores both with `ON_ERROR_STOP` and drops the renamed copies only once the previous binary has read every table; the manual's catalog-loss section says what a catalog older than the tables gives and that tables backed up since restore once `backup sync` lists their backups. ADR-0019 and the RFC section are superseded in part, not whole: CreateBackup on Cassandra still writes their in-catalog format, before it returns (section 3). ADR-0019 changes only its Status line, as `docs/adr/README.md` allows, to the "Superseded in part by [document]" form the README's process now names for a decision a later design replaces in part, and section 10's closing paragraph states what that Status covers, including that the reconciler now removes an interrupted in-catalog create; the RFC's section carries a note, and its section 4 and Known Gaps lines on synchronous operations and on restored indexes carry notes. Edited with this item: section 7's memory paragraph (the restore-memory and vector-index sentences), section 10's table row and closing paragraph, the PR 11 line and its "Documentation it writes" line, and the testing summary's performance paragraph. + +36. (2026-11-06, PR 10b as implemented, #820.) #820 ships `backup sync --move-store `, `--forget-store ` and `--forget-backup ` and plain sync's report of the rows of other stores, with three `BackupCatalog` methods on the four backends: `list_backup_rows_by_root(kind, root, status)` as the interfaces block had it; `rewrite_store(from_kind, from_root, keys, to)`, which rewrites only a row still of the old kind and root and not CREATING and counts the rows it rewrote, so a row another writer moved, or the CREATING row of a server still on the old root, is left rather than moved without its bytes (PostgreSQL in batches of 5,000 keys, each one `UPDATE` over `unnest` of its keys in a transaction the client commits, under `SET LOCAL lock_timeout` 3 s and `statement_timeout` 30 s with up to three attempts on a lock timeout, so the statement of a command killed while it waits on a row lock commits nothing once the claim is gone and the batches before it stay committed; one statement over every key does not fit a fixed bound, since over a catalog of 400,000 rows with 0006's indexes and 1,449-byte descriptions one over 200,000 keys took 16.4 and 16.9 s in plain SQL on PostgreSQL 16, where a batch of 5,000 took 0.23 to 0.29 s; SQLite one `BEGIN IMMEDIATE` transaction, every key or none; MongoDB `update_many` in chunks of 1,000 keys that also `$inc` `lock_seq`, a convention: the update and a transaction that writes the document are serialized (when the update commits after the transaction's snapshot and before its write, the transaction re-runs and reads the row as the update left it; when the transaction wrote first, the update waits for it and applies its filter to the committed document), and a registration keeps the store its server is configured with; Cassandra per key; the trait doc says what a client's death in the call leaves on each); and `summarize_store_roots()`, the rows per store (kind, root, id) and status, which the design did not have: `list_backup_rows_by_root` takes a root, and nothing listed the roots (PostgreSQL and SQLite one `GROUP BY`, MongoDB a `$group`, Cassandra the status table's partitions with their entries counted and not re-read, which the report of other stores then checks against the root's listing kept to each store's id, printing what no row of its status stands behind as status entries). On Cassandra `list_backup_rows_by_root` reads the entries of all five status partitions of the root whatever status it is asked for, dedupes them by (account, ARN), reads each row once and keeps it when its kind and root match (and its status, when one is asked for), so a row whose only entry is under an earlier status (a status change whose entry batch failed, which is logged and not sent again, or whose writer died between its row and its batch) is listed under its own status by the move, the forget options and the report of other stores. A partial rewrite (between PostgreSQL batches, between MongoDB chunks, between Cassandra keys, or between the move's slices) leaves rows under both roots; every step is idempotent and the command exits 1 naming what is left. On Cassandra `rewrite_store` writes the new root's status entry before the row's lightweight transaction (`IF store_kind = ? AND store_root = ? AND backup_status = `) and deletes the old entry after it, the one exception to the module doc's row-first rule: a stop after the new entry leaves only an entry every reader skips, where the row first would leave a moved row that no listing of the configured store returns; an unknown outcome is settled by a `LOCAL_SERIAL` read, which counts the row as this call's write only when it carries `to` whole (kind, root and store id; `to`'s root under another store id is another store, left and not counted), and `NotMatched` reads the row at `LOCAL_SERIAL` first, keeps the new entry when the row already names `to` (this call's write), writes the entry again under the row's own store when the row is now of `to`'s root under another store id with the status read (another move to that root), and otherwise removes the new entry; either way it tries again, up to three attempts. `start_backup_delete`'s and `set_backup_deleted`'s lightweight transactions carry the store kind and root they read, so a DeleteBackup of a server on the old root that read the row before the rewrite does not apply, reads the row again and moves its status entry under the root the row now names: the moved row is never left without its own entry. The report of other stores reads `summarize_store_roots` and names each CREATING and DELETING row through `list_backup_rows` of that store's exact identity (on Cassandra through `list_backup_rows_by_root` of its root, kept to the store's id); a store of the configured root under another id counts as another store, and its line names `--forget-backup ` as the way to remove such a row, since `--move-store` and `--forget-store` refuse the configured root (the line naming those two options is printed only when another root is listed); it is printed after every sync that listed the root to its end, skipped when the run stopped or the listing failed, and a failed read of it exits 1. So 4(g)'s "rows for another store are reported" now holds for every such row, where item 34 had it for the prefixes whose backup id such a row holds (section 4, PR 10). `--move-store` needs the configured store healthy, refuses the configured root and a root with no rows (printing the stores the catalog holds rows for; a root matches as stored, byte for byte), takes the maintenance claim on every backend, SQLite included under its serve lock, and waits over the configured store and every store id the catalog records for the old root (`maintenance::wait_for_quiet` takes a slice of identities), since a CreateBackup of a server still on the old root has its row there. Each row of the old root is checked against the configured store as the PR 10 contract says, read thus: a CREATING row goes through the reconciler's per-row resolution (`resolve_dead_creating_row`, split out of `resolve_dead_creating_rows` so both callers share one body) with the configured store's calls and `OnIncomplete::Leave`: it is written AVAILABLE when the copy holds its complete backup, and any other probe result stops the move with no status written (the takeover's generation is the one change), since a copy without the complete backup proves nothing about the old root's prefix (a copy's order is arbitrary) and a FAILED row would have the configured store's reconciler delete the copy and a server still on the old root the original; the way out is a reconciler on the old root (a server, or on SQLite plain sync, configured with it), which resolves the row there, or `--forget-backup`; the PR 10 contract's sentence is edited with this item; a DELETING row whose prefix there holds no verified manifest is finished with `finish_backup_delete` behind the fence, and stays DELETED when the move then stops; an AVAILABLE row is probed by `probe_manifest` (now `pub(super)`) and its manifest's size and item total must be the row's, so another backup under the same ARN is reported, not moved; the old root's rows and the configured root's are grouped by (account, backup id) before any row is checked, and every old-root row of a group of more than one (with a row of the configured root, or among the moved rows, which only Cassandra can hold) stops the move, so no such row gets a resolution or a finished deletion (on Cassandra two spellings of an ARN share one prefix), and the configured root's rows are read before anything changes, so a failed read stops the move with nothing changed. Every stopping row is named in one pass and none moves while any does; with none, `StoreFence::check_catalog` re-reads the identity file once before `rewrite_store` (a refusal records the store unhealthy, as the fence does everywhere), which the move calls in slices of 500 keys (`REWRITE_SLICE`), or one key per call on a backend that rewrites key by key (Cassandra, `BackupCatalog::rewrite_store_runs_key_by_key`), with a beat of the claim before each 500 keys and the signal and the claim's local state (`HeldClaim::lost`, no write) read before every call, so a move whose claim another holder took during the rewrite (on MongoDB and Cassandra, a command stopped past `stale_job_after_secs`, which reads its claim lost as it resumes since no beat confirmed it within two periods) writes nothing after the one call it had sent or was about to send (one key on Cassandra, one statement of up to 500 keys on MongoDB), and a signal stops it the same way, with the earlier calls' rows moved and the rest left under the old root; the old root is listed again and a row still there is reported with exit 1 (a key the old root no longer lists counts as moved only when its row, read by its ARN, 16 at a time, names the configured store, and a row that names no such store, removed by another writer during the move or moved to a third store, is reported as gone; a re-list that fails after a rewrite that counted every key takes every key as moved, and a failed read of a key's row takes that key and the rest as moved), and a plain sync of the configured root follows, in whose report of other stores a row of the old root (an insert that landed during that sync) is named and exits 1; a move that stops runs no sync. One `extenddb::audit` line is logged per row moved, resolved or finished (section 4, 5.2, PR 10). The removals of `--forget-store` and `--forget-backup` delete no object and read no store, so they run on a store in any state and with none configured (`kind = "disabled"`; a configured root the command cannot open, such as a filesystem directory that does not exist, stops it before the claim, and `kind = "disabled"` runs it), under the claim on every backend, with the wait over the configured store when its identity is known and otherwise every store id the catalog records for the configured root (and the root's ids for `--forget-store`; with no store the wait covers the RUNNING restores). They open the store writing nothing (described as a read leaves it, so no default root is created and no `validate` probe file written; the identity file read, never created; when it agrees with the rows, `check_configured_store_read_only` runs once and is recorded), and on SQLite the command's own pass over what the stopped server left runs without the store's check (`ReconcilerState::for_maintenance_holder_without_store_check`). Each status is removed with a method that exists, no fourth one: AVAILABLE by `forget_available_backup_row` with its in-use rule, DELETED and FAILED by `remove_backup_row`, CREATING by `remove_dead_creating_backup_row` on the catalog's clock, DELETING by `set_backup_deleted` then `remove_backup_row` (no duty meets the DELETED state between them, by the order of the listings: one that listed rows before the claim listed this one DELETING, and one that lists it DELETED did so after the claim was taken and reads the claim held); a removal that meets another status reads the row once more and applies that status's removal. `remove_backup_row` takes the store kind and root the caller read and removes only a row still of them (Cassandra's lightweight transaction `IF backup_status = ? AND store_kind = ? AND store_root = ?`, `NotMatched` answering `false`), so a tombstone duty of a server on the old root does not remove a row the move rewrote after the duty read it. A DELETING row of the configured store is the exception to "delete no object": after the wait for the duties (so a stop during the wait leaves the row DELETING and its objects untouched), the forget finishes its deletion, behind the fence (`finish_backup_delete`, the deletion its DeleteBackup asked for), which needs the store's check to have passed, and otherwise refuses the row; when the check failed on an identity mismatch that the row's own store id takes part in (`IdMismatch`, `MultipleRowIds`: the backup manual's copied identity file), the refusal says so and names the way out the manual gives, the same command with `[backup.store] kind = "disabled"`, whose generic DELETING path removes the row and keeps its bytes, then the move's sync, which adopts the backup again, and a DeleteBackup of it; on SQLite the command's own pass may finish it first, and the removal of its DELETED row says so; a DELETING row of another store keeps its bytes unless a server configured with that store finishes the deletion (its reconciler, under the claim, as the DeleteBackup asked), and a sync of that root after its return with the bytes adopts the backup again. While the configured root has no identity file and the catalog holds rows for it (`FileMissingWithRows`: an unmounted volume), every row of that root is refused, and so on SQLite is every row of the defaulted root found missing while the catalog holds rows for it (moved away, or a symbolic link to a volume that is gone: `configure_described` reads `read_store_ids` for the path and for each path met while every symbolic link along it is replaced by its target, component by component up to 40 links (a chain of links, a relative target, a directory link), each also with its longest existing ancestor canonicalized, and answers `NotConfigured::DefaultRootMissingWithRows`, which the command's header prints in place of "no server has run with this database file"), since a server's check rewrites a missing file at a root with no rows and reports the store healthy. Both wait for running jobs as `purge` does, refusing when one still runs at the wait's end; a CREATING or FAILED row whose heartbeat, written after the wait, is within the owner window, or that is of a store the wait does not cover, is left, as is an in-use row (whose text names `extenddb backup remove-target --account
` for a failed restore's target), and the command exits 1. `--forget-backup` repeats (in order, stopping at the first ARN not removed; an ARN that no row, under either spelling, and no in-catalog backup carries is printed as having nothing to remove and counts as done, so a command line can be run again after a stop; every ARN's form is checked before the claim is taken), takes the ARN as the row carries it and names a row under the other spelling without removing it, reports an earlier release's in-catalog backup as one (exit 1, nothing changed), and then, when a store is configured, runs the store's check once and prints it, naming the next ARN whose manifest is missing; that check writes nothing (`check_configured_store_read_only`: the identity read without the create and without `validate`, the id compared with the one the command opened the store with, the manifest probe), and is not run when the root had no identity file as the command opened it. That is the remedy for a restored catalog whose newest AVAILABLE backup was deleted after the catalog's backup, which item 34 left with "put the prefix back"; `--prune-missing` keeps refusing on `ManifestMissing`, the host-local detector, and plain sync's refusal there names `--forget-backup` with the ARN. After its removals `--forget-store` sweeps, under the claim, the root's status entries, each checked against its row at `LOCAL_SERIAL` (`remove_orphan_status_entries`, answering `StatusEntrySweep { removed, repaired }`): an entry whose row is gone or names another store is deleted (on Cassandra a deleter whose lightweight transaction lands before a rewrite of its row and whose entry batch lands after it leaves one under the old root), and one whose row is of the root under another status is written again under the row's status and the stale one deleted; it prints both counts, then the store ids the catalog still records for the root, without changing the exit (section 4, PR 10). The reconciler's `Duty::fenced` re-reads the row after the fence and skips one that is gone or whose status or store changed, then reads the maintenance claim again and starts nothing while it is held, two catalog reads per deletion; each FAILED-prefix and DELETED-prefix deletion and each tombstone removal that read the claim free must end by its deadline, `DUTY_DELETION_BOUND` (30 s, the S3 store's per-call bound, under the SDK's 60 s attempt timeout) after that read was sent: a read answered after the deadline starts nothing, the prefix deletion runs through the store's new `delete_prefix_before(prefix, deadline) -> PrefixDeletion { deleted, finished }` (the filesystem walk checks the deadline beside its cancel flag before each removal, the S3 loop before each `ListObjectsV2`, `DeleteObjects`, `ListMultipartUploads` and `AbortMultipartUpload`; the trait's default checks once, for a one-step store), and the call is also dropped at the deadline; past it the duty stops for the cycle, the row left for the next. Beside servers the forget options wait `FORGET_WAIT`, the bound plus one 30 s store call, from the claim before their first FAILED or DELETED removal and print the wait; on SQLite, under the serve lock, they do not wait. So what a deletion duty can still do to the bytes of a row a forget removes (4(a)) is the step in flight at its deadline, one removal on a filesystem root or one S3 request (up to 1,000 keys); the bound, like every lease in the design, assumes the server's process keeps running, and one paused past the wait between its last check and a call already sent (SIGSTOP, a frozen cgroup, a VM pause) runs that call when it resumes. A marker written per deletion, which the forget's removal would be conditioned on, is the full remedy, and is not taken: it adds a catalog write to every deletion for a case outside every other lease. The store-backed CreateBackup reads the claim again immediately before `claim_backup`, and the store-backed restore immediately before its registration as well as after it, so a claim taken before that read refuses the job with no row or record (`LimitExceededException`); what remains is the window from that read to the insert, however long the insert takes: a CreateBackup of a server still on the old root whose insert lands during the move's wait is listed by it, one that lands before the rewrite is left CREATING under the old root and reported by the re-list (the rewrite skips CREATING rows), one that lands during the move's sync of the configured root is named by that sync's closing report of other stores if it lands before it (exit 1), and one that lands after only by a later plain sync; the backup manual says to check plain sync's report of other stores before deleting the old root. `OTHER_STORE` now names the move ("or copy its tree under the configured root and move the backup's rows to the configured store with `extenddb backup sync --move-store `, and restore again"), as 5.3 step 7 says, and the reconciler's module doc with it. Each of the three confirms the maintenance claim with a beat of its own (`HeldClaim::recheck`, item 34) right before each step that cannot be undone, and stops when it finds the claim lost: the move before a dead CREATING row's resolution, a DELETING row's finished deletion and each 500 keys of the rewrite, reading the claim's local state before every other call of it (`MoveTarget::claim`); the forget options before each row's removal, a DELETING row's DELETED write or finished deletion and the removal of the DELETED row it leaves, and `--forget-store` before its sweep of the status entries (`ForgetRun::claim`). A stop there leaves the row as the steps before it left it, and the command's stop line names the step. As for sync and purge, what remains is a stop after the re-check confirmed the claim and before the step's own call reached the catalog or the store, or with that call already sent: the call lands when the process resumes, though on MongoDB and Cassandra another command may have taken the claim over during the stop. The three options conflict with each other, with `--adopt-store` and with `--prune-missing`. Memory: every backend reads a listing's rows whole, and the move makes two such reads, the old root's rows, which it keeps without their parsed description, and the configured root's rows, of which it keeps the (account, backup id) pairs for the two-rows check (its sync reads them again); after the rewrite it reads each moved row by its ARN rather than listing the configured store. Each read is about 27 KiB per row on SQLite and 23 on PostgreSQL above about 50 MiB, at 1,452 bytes of table description per row and growing with it: a move of 20,000 rows of a five-GSI table into an empty store peaked at 569 MiB on a SQLite debug build and 500 MiB on a PostgreSQL one, and a move of 50 rows on the PostgreSQL debug build at 49 MiB into an empty store and 231 MiB into a store of 8,000 rows (section 7). The PR 10 section's split line, interfaces block and sync contract, and section 7, are edited with this item; 4(c), 4(f), 4(g), 5.3 step 7, PR 8's claim-held paragraph and the PR 10 acceptance and transition lines state what was built and stay as written. The backup manual's procedure copies the old tree without its `extenddb-store.json` (the restarted servers gave the new root its own; a copied file turns every running server's store unhealthy until a restart, and, when a backup was taken under the new root meanwhile, needs that backup's row forgotten before the move, whose sync adopts it again). Not in #820: `migrate-legacy` (#821), a sweep of Cassandra's orphan status entries beyond `--forget-store`'s root, the same all-partition read for Cassandra's store-scoped `list_backup_rows`, which the reconciler reads (a DELETING row of the configured store whose entry batch failed is not finished by its reconciler until the entry is repaired; `--forget-backup` of its ARN finishes and removes it), and any detection of a server left configured with the old root, which the backup manual's procedure reconfigures first. diff --git a/docs/design/README.md b/docs/design/README.md index f0ee934ed..870933166 100755 --- a/docs/design/README.md +++ b/docs/design/README.md @@ -22,6 +22,7 @@ This directory contains the authoritative design documents for ExtendDB (extendd | 15 | [Backup and Restore](15-backup-restore.md) | Overview of the off-host backup series: the invariants, the per-operation contracts and a short section per later PR; the format, store and snapshot documentation lives with their crates (`crates/core/src/backup_format`, `crates/backup-store`, `crates/storage/src/snapshot.rs`) | | 15b | [Backup memory and duration record, PR 8](15-backup-restore-bench-pr8.md) | The measurements section 7 and revision item 31 of the design cite: peak RSS and duration of the 100k, 1M, worst-expansion and incompressible backups per backend, the allocator series, the PostgreSQL page's cost | | 15c | [Restore memory and duration record, PR 9](15-backup-restore-bench-pr9.md) | The measurements section 7 and revision item 33 of the design cite for the restore: peak RSS and duration of the 100k, 1M, worst-expansion and large-item restores on SQLite and PostgreSQL, with and without one allocator arena, the terms the code bounds and the sizing sentence | +| 15d | [Backup and restore record, PR 11](15-backup-restore-bench-pr11.md) | The measurements the testing summary cites for the MongoDB, Cassandra and vector cells: peak RSS and duration of the 100k and 1M restores on MongoDB and Cassandra and of the backup and restore of 100k- and 1M-item tables with a 128-dimension vector index on PostgreSQL and SQLite, with and without one allocator arena; the 100k and 1M MongoDB backups; later runs that attribute SQLite's restore memory to the write batch, which grows with the item's size whatever the attribute types (the same items with no vector index, items of a 2,290-character string, 20,000 items of 1,536 numbers, two restores at once, the peak's spread over nine runs), give the PostgreSQL vector index's share from five same-harness pairs and MongoDB's restore duration with the backfill worker's cycle; SQLite's WAL growth beside a backup with and without two GSIs; and the figures measured for the backup manual (PostgreSQL dead row versions and SQLite WAL growth during a backup) with their reproductions | ## How to Use These Docs diff --git a/docs/differences-from-dynamodb.md b/docs/differences-from-dynamodb.md index b768e0a76..c6e1b7ac5 100755 --- a/docs/differences-from-dynamodb.md +++ b/docs/differences-from-dynamodb.md @@ -108,7 +108,7 @@ Behaviour stated as measured, and every number below, was observed with release | Area | DynamoDB | ExtendDB | |------|----------|------| | What a restore carries across | Key schema, attribute definitions, items, secondary indexes, billing mode and provisioned throughput, on-demand throughput (table and GSI level), SSE settings. Table class is not carried: a restore is `STANDARD` whatever the source's class (measured). Stream settings, TTL, tags, auto scaling, IAM policies, and CloudWatch settings must be reapplied by hand. `DescribeTable` on a freshly restored table reports `ItemCount` 0 and `TableSizeBytes` 0 (measured on 15 restored tables, one of them ten minutes after `ACTIVE`) | The same set on all four backends, with one gap: the table's on-demand throughput is carried, a GSI's is not (ExtendDB records no per-index on-demand maximum, and `OnDemandThroughput` is not a member of its GSI definition). Table class is not restored: a `STANDARD_INFREQUENT_ACCESS` source restores as `STANDARD`, as on the service. Streams, TTL, tags, and deletion protection are not restored. `DescribeTable` on a restored table reports `ItemCount` 0 and `TableSizeBytes` 0 when the table goes `ACTIVE`, as the service does (measured: 0 and 0 on 15 restored tables, one of them ten minutes after it went `ACTIVE`), until the statistics pass next rewrites both from the restored rows *(from the code)*. A backup written before catalog 0.0.4 (PostgreSQL, SQLite, Cassandra) has no recorded table definition and restores with its keys, items, and billing mode but no secondary indexes. A provisioned one restores with 5/5 throughput on PostgreSQL and SQLite and with its recorded throughput on Cassandra | -| How `RestoreTableFromBackup` runs | Returns at once with the target `CREATING`; the service copies in the background | The same on all four backends: the response carries the target `CREATING`, and the items are copied by a background task in the server, so a client that times out or disconnects does not stop it. Once the request has been authenticated and authorized, the rest of it, from reading the backup to registering the restore, runs in a task of its own, so a client that disconnects after that does not cancel it either: the restore runs and the client finds the target with `DescribeTable`. A client that disconnects while the request is still being authenticated or authorized cancels it, and nothing is created *(from the code)*. Measured with a client whose 1 ms read timeout closed its connection before any response arrived: on all four backends the restore was registered and finished. Measured *(in-catalog restore, an earlier build)* with a 60,000-item backup of a table with no secondary indexes: response in 0.032 s (PostgreSQL), 0.012 s (SQLite), 0.053 s (MongoDB), 0.151 s (Cassandra); `ACTIVE` after 1.5 s, 2.1 s, 3.6 s, and 11.2 s. Measured *(store-backed restore, this release)* with the same shape (60,000 items of 200 bytes, composite key, no secondary index) from a filesystem store on loopback: response in 0.051 s (PostgreSQL), 0.016 s (SQLite), 0.055 s (MongoDB); `ACTIVE` after 0.67 s, 1.85 s and 3.74 s (on MongoDB the job's DONE at 1.8 s and the index backfill worker's flip after); 1M items of 200 bytes in key order (the server's own backups) `ACTIVE` after 9.5 s on PostgreSQL and 26.8 s on SQLite, the same items as an unordered backup (a MongoDB source, a multi-part-key table) after 17.5 s and 66 to 90 s (`docs/design/15-backup-restore-bench-pr9.md`, which has the host and its load); the store-backed response includes the sidecar and manifest reads and its `ACTIVE` time the download and the per-file verification, and Cassandra as a store-backed target was measured at 12,000 items only (the kill points: a resumed restore of 12,000 items of 4,000 bytes in two data files reached `ACTIVE` 8.3 s after its takeover on a single-node container). On Cassandra the response waits for the target table's schema to be created: 0.75 to 1.3 s with five restores running at once *(earlier build)*, which is longer than a client read timeout of 1 s. On every backend the request waits at most 60 s (`RESTORE_REGISTRATION_TIMEOUT`) for its background task to register the `CREATING` target of a store-backed restore (a bound against a connection pool that has stopped answering; the manifest and sidecar reads that precede the registration are not under it); past that it answers `InternalServerError` while the task goes on, so the target can appear after the error, holding the backup in use (`BackupInUseException` to `DeleteBackup`) and the name until the restore ends; `DescribeTable` is where a client finds it *(from the code)*. Because that 60 s bound equals botocore's default read timeout, a client whose first attempt times out while the registration is still in flight (a backup's `backup_index` row held under a slow lock) retries: a retry that names the same backup and target waits for the in-flight registration and is answered its `CREATING` description rather than registering a second target, and a restore of a *different* backup into the same name while the first is in flight is `TableInUseException` *(from the code)*. The wait is per server process: each server keeps its own list of in-flight registrations, so a retry waits for the first attempt's registration only when it reaches the server that holds it; a retry that reaches another server behind a load balancer finds the name free until the registration commits and then `TableInUseException` (the registration took the name) or, once the target exists, the same `CREATING` description from the catalog *(from the code)*. The job permit is the last of the request's checks: a restore that finds every `[backup] max_concurrent_jobs` slot held (default 4; backups and restores together, per server process) waits up to 30 s for one and is then `LimitExceededException` ("Too many backup and restore jobs are running on this server (at most N at once, the [backup] max_concurrent_jobs bound); retry later"), creating nothing, where the service accepted five restores sent together (measured on DynamoDB); on the store-backed path the permit is taken before the manifest is read, after the catalog row's state, the target name and the backup's in-use rule are checked, in that order (a restore of a backup into its own CREATING target is `TableInUseException`, measured on DynamoDB). A restore runs inside the server process; if the process stops, see the abandoned-restore row | +| How `RestoreTableFromBackup` runs | Returns at once with the target `CREATING`; the service copies in the background | The same on all four backends: the response carries the target `CREATING`, and the items are copied by a background task in the server, so a client that times out or disconnects does not stop it. Once the request has been authenticated and authorized, the rest of it, from reading the backup to registering the restore, runs in a task of its own, so a client that disconnects after that does not cancel it either: the restore runs and the client finds the target with `DescribeTable`. A client that disconnects while the request is still being authenticated or authorized cancels it, and nothing is created *(from the code)*. Measured with a client whose 1 ms read timeout closed its connection before any response arrived: on all four backends the restore was registered and finished. Measured *(in-catalog restore, an earlier build)* with a 60,000-item backup of a table with no secondary indexes: response in 0.032 s (PostgreSQL), 0.012 s (SQLite), 0.053 s (MongoDB), 0.151 s (Cassandra); `ACTIVE` after 1.5 s, 2.1 s, 3.6 s, and 11.2 s. Measured *(store-backed restore, this release)* with the same shape (60,000 items of 200 bytes, composite key, no secondary index) from a filesystem store on loopback: response in 0.051 s (PostgreSQL), 0.016 s (SQLite), 0.055 s (MongoDB); `ACTIVE` after 0.67 s, 1.85 s and 3.74 s (on MongoDB the job's DONE at 1.8 s and the index backfill worker's flip after); 1M items of 200 bytes in key order (the server's own backups) `ACTIVE` after 9.5 s on PostgreSQL and 26.8 s on SQLite, the same items as an unordered backup (a MongoDB source, a multi-part-key table) after 17.5 s and 66 to 90 s (`docs/design/15-backup-restore-bench-pr9.md`, which has the host and its load); the store-backed response includes the sidecar and manifest reads and its `ACTIVE` time the download and the per-file verification, and Cassandra as a store-backed target was measured at 12,000 items (the kill points: a resumed restore of 12,000 items of 4,000 bytes in two data files reached `ACTIVE` 8.3 s after its takeover on a single-node container) and at 1M items of 200 bytes in key order, `ACTIVE` after 49.0 s, and MongoDB's restore of its own unordered backup of 1M such items `ACTIVE` after 33.8 s, which another run may exceed by up to 5 s, since MongoDB's backfill worker flips the table at its next 5 s cycle after the load (`docs/design/15-backup-restore-bench-pr11.md`, which has the host and its load). On Cassandra the response waits for the target table's schema to be created: 0.75 to 1.3 s with five restores running at once *(earlier build)*, which is longer than a client read timeout of 1 s. On every backend the request waits at most 60 s (`RESTORE_REGISTRATION_TIMEOUT`) for its background task to register the `CREATING` target of a store-backed restore (a bound against a connection pool that has stopped answering; the manifest and sidecar reads that precede the registration are not under it); past that it answers `InternalServerError` while the task goes on, so the target can appear after the error, holding the backup in use (`BackupInUseException` to `DeleteBackup`) and the name until the restore ends; `DescribeTable` is where a client finds it *(from the code)*. Because that 60 s bound equals botocore's default read timeout, a client whose first attempt times out while the registration is still in flight (a backup's `backup_index` row held under a slow lock) retries: a retry that names the same backup and target waits for the in-flight registration and is answered its `CREATING` description rather than registering a second target, and a restore of a *different* backup into the same name while the first is in flight is `TableInUseException` *(from the code)*. The wait is per server process: each server keeps its own list of in-flight registrations, so a retry waits for the first attempt's registration only when it reaches the server that holds it; a retry that reaches another server behind a load balancer finds the name free until the registration commits and then `TableInUseException` (the registration took the name) or, once the target exists, the same `CREATING` description from the catalog *(from the code)*. The job permit is the last of the request's checks: a restore that finds every `[backup] max_concurrent_jobs` slot held (default 4; backups and restores together, per server process) waits up to 30 s for one and is then `LimitExceededException` ("Too many backup and restore jobs are running on this server (at most N at once, the [backup] max_concurrent_jobs bound); retry later"), creating nothing, where the service accepted five restores sent together (measured on DynamoDB); on the store-backed path the permit is taken before the manifest is read, after the catalog row's state, the target name and the backup's in-use rule are checked, in that order (a restore of a backup into its own CREATING target is `TableInUseException`, measured on DynamoDB). A restore runs inside the server process; if the process stops, see the abandoned-restore row | | `RestoreSummary` | The restore response and `DescribeTable` while `CREATING` report `SourceBackupArn`, `SourceTableArn`, `RestoreDateTime` (the backup's `BackupCreationDateTime`, not the time the restore ran), and `RestoreInProgress=true`. Once the table is `ACTIVE`, `DescribeTable` omits `RestoreSummary` | The same on all four backends. No response that describes a table (`CreateTable`, `UpdateTable`, `DeleteTable`, `DescribeTable`, `RestoreTableFromBackup`) carries `RestoreSummary` unless the table is `CREATING`; what the service's `UpdateTable` returns for a restored table was not measured | | Reads on a restore target | A `GetItem` or `Scan` on the target while `CREATING` is `ResourceNotFoundException`; the first read after `DescribeTable` reports `ACTIVE` succeeds (measured) | The same on all four backends. The server's key-info cache stores a negative entry (5 s TTL by default) for a name the catalog does not have, and does not store one for a name the catalog has in a state other than `ACTIVE`: on a not-found answer it asks the catalog once more with `DescribeTable`, and authorization records the name as missing in the key-info snapshot it hands the engine, so the engine does not look it up again. A `GetItem`, `PutItem`, `UpdateItem`, `DeleteItem`, `Query`, or `Scan` on the target while it is `CREATING` therefore costs exactly one `table_key_info` and one `describe_table` catalog load, uncached, per request (on SQLite five SQL statements: one for the key info, four for `DescribeTable`; on PostgreSQL `describe_table` is two queries), plus one `describe_table` on the first such request for the throttle registry; the first read after `ACTIVE` loads once more and is then cached. Measured on SQLite with a `GetItem` loop on a `CREATING` restore target of 100,000 items: 40 requests refused, 40 key-info and 41 `DescribeTable` loads in the server's statement log; with the hand-off to the engine removed, 80 and 81. In an earlier run of the fidelity test the first `GetItem` after `ACTIVE` answered in 3 ms after 72 refused reads. | | `RestoreTableFromBackup` overrides | Current request members are `BillingModeOverride`, `ProvisionedThroughputOverride`, `GlobalSecondaryIndexOverride`, `LocalSecondaryIndexOverride`, `SSESpecificationOverride`, `OnDemandThroughputOverride`, and `VectorIndexOverride`; DynamoDB applies them to the restored table | See the overrides table below. A refused override is refused before the target table is created | @@ -124,7 +124,7 @@ Behaviour stated as measured, and every number below, was observed with release | Concurrent restores of one backup | A second `RestoreTableFromBackup` of a backup while a restore from it is still `CREATING` is `BackupInUseException` (measured) | The same on all four backends: the second restore is refused with `BackupInUseException` and creates nothing; the check runs inside the transaction that registers the restore (PostgreSQL, with the backup row locked `FOR UPDATE`; SQLite), inside the claim on the backup document followed by a check for a `CREATING` target of the backup (MongoDB), or inside the lightweight transaction on the backup row (Cassandra), so two simultaneous requests cannot both pass. For a store-backed restore the registration takes the same lock on the `backup_index` row instead of the legacy `backups` row (PostgreSQL `SELECT ... FOR UPDATE`, SQLite the `BEGIN IMMEDIATE` write transaction, MongoDB a `$inc lock_seq` on the `backup_index` document, Cassandra the `active_restores` lightweight transaction on the backup's partition), the same ordering guarantee, so a legacy restore and a store-backed restore of two different backups never contend and a `DeleteBackup` of a store-backed backup meets only store-backed registrations. Two concurrent registrations of one backup leave one restore target on Cassandra as on MongoDB: the registration's lightweight transaction (`IF backup_status = 'AVAILABLE' AND active_restores = null`) is linearized on the backup's partition, so the second finds `active_restores` non-null and is `BackupInUseException`; the in-use pre-read is advisory *(storage-level tests on both)*. Once the first target is `ACTIVE` the backup restores again; on MongoDB the check reads majority-committed data (at the default transaction read concern) while `DescribeTable` reads at `local`, so a `RestoreTableFromBackup` or `DeleteBackup` issued within the majority-commit lag after `DescribeTable` first shows the target `ACTIVE` can still be `BackupInUseException` and succeeds when retried (the lag is the journal flush plus any concurrent batch write still holding an earlier oplog slot; measured at 7 to 12 ms, once in about 2,000 observations, with a 3,000-item insert running beside the write). A restore of a backup whose `DeleteBackup` is in progress answers `BackupNotFoundException` on every backend (the service's answer was not measured), by two mechanisms: on MongoDB and Cassandra the delete marks the backup `DELETING` before it drops the data, and the restore finds no `AVAILABLE` backup; on PostgreSQL and SQLite the restore waits for the delete's transaction and then finds no backup. Earlier builds accepted concurrent restores of one backup on PostgreSQL, SQLite, and Cassandra, and on MongoDB after a 2 s retry | | Restore interrupted by the server stopping | Managed by the service | The copy runs in the server process, so a stopped process leaves its target `CREATING` with a RUNNING restore job record whose heartbeat stops. The engine's reconciler, once at startup and then every `backup.reconcile_interval_secs` (default 60 s), marks the record FAILED and removes the target once the heartbeat is older than `backup.stale_job_after_secs` (default 300 s) on the database's clock, on every backend alike, logs the removal with the table name, and then lets `DeleteBackup` and a new restore into the name succeed *(from the code)*. On SQLite a restart ages every record of the stopped process at once (one process serves the file), so the first cycle removes it without the stale window. Until then `DeleteTable` and `DeleteBackup` are refused as during a running restore. A store-backed restore (one run from a `[backup.store]`) is instead resumed, not discarded: its owner's death leaves a RUNNING restore job record with a stale heartbeat and the target `CREATING`, and the reconciler of any instance with the configured store takes the record over under a fresh owner generation once the heartbeat is stale (on SQLite at once after a restart, by the record's slot; on PostgreSQL once the owner's claim session has ended as well), stamping a fresh heartbeat in the same write so no second instance takes it too, and resumes the load from its recorded progress (per data file on PostgreSQL, per batch on SQLite, MongoDB and Cassandra), re-downloading to its own spool when the dead owner's `spool_host` is not this host's; the target reaches `ACTIVE` with the backup's full item count, its secondary indexes complete: a resumed Cassandra restore re-applies its last batch as idempotent upserts that rewrite the same base and index rows (the write path rewrites an unchanged item's GSI and LSI rows rather than deleting them, #424). A resume that finds no recorded progress and a target holding more items than the backup's total is `FAILED` with reason `target_not_empty` and the target is left in place: the rule protects a catalog restored from a dump that names a restore which finished and was written to since. A target holding no more than the total with no recorded progress is a first data file applied in part: on Cassandra, which records a batch's progress after its writes, an ordinary crash before the first progress record (the first 10,000 items or 32 MiB) leaves written items and no progress, and the resume re-applies the first file with idempotent upserts and reaches `ACTIVE` with the exact count *(from the code; the storage and engine cells)*. MongoDB records each batch's documents and its progress in one transaction, so a mid-file crash leaves no items the record does not account for and resumes to `ACTIVE`; only a crash before the first batch commits leaves an empty target, which resumes from the start. A record that one instance's reconciler has taken over five times without the restore ending (a resume whose final count or progress read fails every time, or that stops with the record RUNNING) is failed on the next takeover with reason `owner dead: taken over 5 times by one instance without the restore ending` and its target removed, so a record that can never finish does not hold its name and its backup for as long as the server runs; the count is per instance and a restart starts it over *(from the code)*. The target stays `CREATING` with a `FAILED` record: `DeleteTable` on it is `ResourceInUseException`, `DeleteBackup` of its backup and a second restore of that backup are `BackupInUseException`, and the reconciler does not remove it; `extenddb backup remove-target --account
` (the admin guide's `backup` section) removes the target and its record, after which `DeleteBackup` and a new restore into the name succeed. A legacy (in-catalog) restore is still failed and its target removed. Measured *(binary-level, this release)*: SIGTERM during a store-backed restore left the record RUNNING and the restarted server resumed it to `ACTIVE` with every item (on PostgreSQL after the 45 s stale window, on SQLite at once); a SIGKILL at each restore kill point left the target to finish with the exact count (the base table's and the GSI's) or, for a failing restore, gone, except at the first reach of the Cassandra point that pauses before a batch's first progress record: on Cassandra (killed after the first batch's 10,000 upserts, before their record) the resume of that build answered `target_not_empty` and left the target `CREATING`, and `extenddb backup remove-target` then removed it, after which `DescribeTable` was `ResourceNotFoundException` and `DeleteBackup` succeeded; this release resumes that case with upserts to `ACTIVE` *(from the code; the nightly kill-point job runs it)*. On MongoDB a SIGKILL after a file's `insert_many` leaves the batch's items and its progress committed together, so the resume continues to `ACTIVE` rather than `target_not_empty`. Not covered *(from the code)*: on MongoDB, a target whose base copy had finished has its record DONE and is left alone by the reconciler; the index backfill worker finishes it after a restart, and if that backfill fails on every attempt the worker retries it every 5 s with a warning and never gives up, so the target stays `CREATING` (and `DeleteTable` and `DeleteBackup` stay refused) until the cause is fixed or the target's catalog document is removed by hand (no failing case was found to reproduce this). A target left `CREATING` by a build before this release is backfilled by the catalog migration as a RUNNING record with a heartbeat at the epoch, so the first cycle removes it. Heartbeats and the stale window are measured on the database server's clock, so the application hosts' clocks play no part. The same reconciler resolves a store-backed `CreateBackup` the server stopped (one reconciler with one duty list, in place of per-backend sweeps): a `CREATING` backup row whose owner is dead (on PostgreSQL and MongoDB a heartbeat older than `stale_job_after_secs`, on PostgreSQL once the owner's claim session has ended as well; on SQLite, where one process serves the file, every `CREATING` row after a restart and, at runtime, a row whose in-process slot no task holds, so a job task that died is resolved by the next cycle without a restart) is taken over under a fresh owner generation and becomes `AVAILABLE` when a complete, verified manifest is under its prefix in the configured store (the process died between the manifest write and the status write) and `FAILED` otherwise, never continuing the upload; the objects of a `FAILED` backup are deleted once its heartbeat is older than `stale_job_after_secs`; a `DELETING` backup's deletion is finished; a `DELETED` backup's prefix that still lists objects is swept; and `FAILED` and `DELETED` rows are removed once their prefix is empty and their `completed_at` is older than `failed_row_retention_secs` (7 days). Measured *(binary-level, this release)*: SIGKILL between the data file and the manifest left the row `CREATING` and the data file under the prefix, and after the restart the row was `FAILED` in the first cycle on SQLite and 46 s after the kill on PostgreSQL (`stale_job_after_secs = 45`), with the prefix deleted in the same cycle on PostgreSQL (the stale heartbeat is what made the owner dead) and once the dead owner's heartbeat was stale on SQLite, 45 s after the kill (the slot made the owner dead at once; the prefix waits for the heartbeat clock on every backend); the same at a kill between the three documents and the manifest, which leaves the documents under the prefix and no manifest; SIGKILL between the manifest write and the status write left the five objects, and the row was `AVAILABLE` with the manifest's size and count after the restart (SQLite at once, PostgreSQL after 48 s); SIGTERM during an upload left the row `FAILED` with reason `shutdown` and an empty prefix. These duties run only while the configured store is healthy, and under a maintenance claim only the dead-owner and `DELETING` duties run. A `FAILED` backup is `BackupNotFound` to clients; `extenddb backup list --failed` prints it with its reason | | Point-in-time recovery | 35-day window: `UpdateContinuousBackups` enables continuous backups, and `RestoreTableToPointInTime` restores to any second in the window | Not supported; use on-demand backups (`CreateBackup` and `RestoreTableFromBackup`) instead. `DescribeContinuousBackups` reports `ContinuousBackupsStatus: ENABLED` (the service reports this unconditionally) with `PointInTimeRecoveryStatus: DISABLED` and no restorable-time window. `UpdateContinuousBackups` with `PointInTimeRecoveryEnabled: true` returns `ContinuousBackupsUnavailableException`; with `false` it returns the same description. `RestoreTableToPointInTime` checks the source table, then returns `PointInTimeRecoveryUnavailableException` ("Point in time recovery is not enabled for table ''"), the exception the service models on that operation. A table that does not exist is `TableNotFoundException` ("Table not found: ") on all three operations, as on the service. Same on all four backends | -| Where backups live | Managed storage, independent of the table | With a configured `[backup.store]` (a filesystem root, which is the default beside a SQLite database file, or an S3 bucket and prefix) on PostgreSQL, SQLite and MongoDB: under the store's root, at `/
//`, as gzip data files of DynamoDB JSON lines (`data/000001.json.gz`, ...) and the manifest documents (`extenddb-manifest.json` last, the commit record), with a `backup_index` row in the catalog as the index; a lost database loses the rows, not the bytes (`extenddb backup sync` gives a new catalog a row for each backup under the root, and `--adopt-store` first makes it the owner of a root another catalog owned). A prefix copied from a server of another region, whose manifest spells its ARN in that region, is adopted under this server's region: its row carries the ARN this server issues, so `ListBackups` names this server's region (every `ListBackups` answer measured on the service names the endpoint's region) and `DescribeBackup`, `RestoreTableFromBackup` and `DeleteBackup` find it under either region's spelling, as they find the backups this server made and as the service finds a backup under another region's spelling of its ARN (measured; `DescribeBackup` answers with this server's spelling, as it does for this server's own backups, where the service echoes the spelling sent); the manifest keeps the spelling it was written with, and so does the source table's ARN, which `DescribeBackup`'s `SourceTableDetails.TableArn`, `ListBackups`' `BackupSummaries[].TableArn` and the restore's `RestoreSummary.SourceTableArn` answer in the manifest's region (a backup this server made names this server's region in every ARN it answers); the restore and the reconciler accept a manifest whose ARN differs from the row's in the region alone. Without a store, and on Cassandra always, inside the database the server already uses, so a lost database loses its backups too: on PostgreSQL in the catalog database (`backups`, `backup_items`, `backup_definitions`, `table_restores`), on SQLite in the one database file, on MongoDB as a `backups` metadata collection in the catalog database and one data collection per backup in the data database, on Cassandra as backup metadata in the catalog keyspace and item payloads in the account's keyspace; take database-level backups for off-host copies there | +| Where backups live | Managed storage, independent of the table | With a configured `[backup.store]` (a filesystem root, which is the default beside a SQLite database file, or an S3 bucket and prefix) on PostgreSQL, SQLite and MongoDB: under the store's root, at `/
//`, as gzip data files of DynamoDB JSON lines (`data/000001.json.gz`, ...) and the manifest documents (`extenddb-manifest.json` last, the commit record), with a `backup_index` row in the catalog as the index; a lost database loses the rows, not the bytes (`extenddb backup sync` gives a new catalog a row for each backup under the root, and `--adopt-store` first makes it the owner of a root another catalog owned; a root copied to a new place keeps its backups' ARNs once `--move-store` re-points their rows). A prefix copied from a server of another region, whose manifest spells its ARN in that region, is adopted under this server's region: its row carries the ARN this server issues, so `ListBackups` names this server's region (every `ListBackups` answer measured on the service names the endpoint's region) and `DescribeBackup`, `RestoreTableFromBackup` and `DeleteBackup` find it under either region's spelling, as they find the backups this server made and as the service finds a backup under another region's spelling of its ARN (measured; `DescribeBackup` answers with this server's spelling, as it does for this server's own backups, where the service echoes the spelling sent); the manifest keeps the spelling it was written with, and so does the source table's ARN, which `DescribeBackup`'s `SourceTableDetails.TableArn`, `ListBackups`' `BackupSummaries[].TableArn` and the restore's `RestoreSummary.SourceTableArn` answer in the manifest's region (a backup this server made names this server's region in every ARN it answers); the restore and the reconciler accept a manifest whose ARN differs from the row's in the region alone. Without a store, and on Cassandra always, inside the database the server already uses, so a lost database loses its backups too: on PostgreSQL in the catalog database (`backups`, `backup_items`, `backup_definitions`, `table_restores`), on SQLite in the one database file, on MongoDB as a `backups` metadata collection in the catalog database and one data collection per backup in the data database, on Cassandra as backup metadata in the catalog keyspace and item payloads in the account's keyspace; take database-level backups for off-host copies there | | Backup creation | `CreateBackup` returns while the backup is `CREATING`; `ListBackups` and `DescribeBackup` expose that state until it becomes `AVAILABLE`. `BackupName` and `TableName` are checked as the model says: length 3 to 255 and `[a-zA-Z0-9_.-]+`, with "1 validation error detected: Value '' at 'backupName' failed to satisfy constraint: " texts, joined under "2 validation errors detected:" when both fail (measured). `TableName` may be the table's ARN (measured). A table that does not exist or is still `CREATING` is `TableNotFoundException` "Table not found: " (measured) | With a configured `[backup.store]` on PostgreSQL, SQLite and MongoDB, `CreateBackup` takes the table's snapshot and writes the catalog row in the request and returns the backup `CREATING`; a job then uploads it and the backup becomes `AVAILABLE`, which `DescribeBackup` and `ListBackups` show as on the service (`BackupSizeBytes` is 0 while `CREATING` and the compressed size once `AVAILABLE`; `ItemCount` and `TableSizeBytes` in `SourceTableDetails` are the table's figures as last counted by the stats pass while `CREATING`, which is 0 and 0 for a table the stats pass has not reached yet, such as one loaded seconds before CreateBackup, and the backup's exact figures once `AVAILABLE`: the items written and the sum of their DynamoDB sizes; the service reported 0 and 0 in every fresh-table backup measured). `CREATING` lasts the whole upload (the durations below), and while it lasts `DeleteBackup` and `RestoreTableFromBackup` of the backup answer `BackupInUseException` "Backup is being created: " on all three backends; the 16 service backups measured were `AVAILABLE` 12 to 90 ms after `CreateBackup` answered, so a client that deletes or restores a backup right after creating it polls `DescribeBackup` first here. The backup is the table as it stood when its snapshot opened, which is inside the request but after the request's checks (the table's state, the store's health, the maintenance claim) and after any wait for a job permit, up to 30 s (below): every write acknowledged before the open is in the backup and none acknowledged after it, so a `PutItem` acknowledged while the request waited for a permit is in the backup (measured: a write acknowledged 1.01 s after a `CreateBackup` that waited for a permit was in it, 2 of 2 runs), and `BackupCreationDateTime` is the open's millisecond. Measured on the service: a restored backup contained every write acknowledged before `CreateBackup` was sent and the 18 writes acknowledged in the 0.1 s up to `BackupCreationDateTime`, and none of the 11,754 acknowledged in the 75 s that followed, so the service takes the table at an instant inside the request as well. A backup the job cannot finish (a store failure, an item no restore can load, a shutdown) is `FAILED`, which is `BackupNotFound` to clients, and its objects are deleted (the restore-interruption row). A `CreateBackup` still waiting in its snapshot open or row insert when the server stops writes no row; it is answered `ServiceUnavailable` "The server is shutting down. Retry the request." only after a registration loss, where the jobs' token is cancelled while the listener still serves, since on SIGTERM the listener's 5 s connection drain closes the connection before that token is cancelled (measured: the connection closed 8.2 s after the request was sent). Measured *(this release, 16-core host, filesystem store, the record in `docs/design/15-backup-restore-bench-pr8.md`)*: 100,000 items of a composite-key table with two GSIs `AVAILABLE` in 1.85 s on PostgreSQL and 1.44 s on SQLite, 1,000,000 in 36.1 s and 12.45 s. Without a store, and on Cassandra always, `CreateBackup` copies the items before it returns and returns the backup `AVAILABLE`. On PostgreSQL, MongoDB, and Cassandra a concurrent `ListBackups` never saw an in-catalog backup `CREATING` *(earlier build)*; on SQLite the backup row is visible as `CREATING` to other requests while the copy runs. Measured *(earlier build, in-catalog)*: 1.3 s for 60,000 items on PostgreSQL, 1.9 s for 60,000 on SQLite, 0.2 s for 20,000 on MongoDB, 5.1 s for 20,000 on Cassandra. MongoDB's in-catalog backups report `BackupSizeBytes` 0. The name checks are the service's, with its texts (the length clause before the pattern clause when both fail; the service's order was not captured), counting characters, with `TableName` taking the deployment's table-name limits; before them a 200 KB name was stored and returned in every `ListBackups` page (a 500 on Cassandra) and a NUL byte was a 500 on PostgreSQL. `TableName` as an ARN (`arn:aws:dynamodb:::table/`) is accepted on all four backends: an ARN whose region is not the server's (`server.region`, compared as written, so `US-EAST-1` differs from `us-east-1`) is `ValidationException` "1 validation error detected: Invalid AWS region in ''" (the service's answer, measured on DynamoDB twice; an earlier build resolved it to the server region's table), another account's is refused with the service's answers (see "A resource ARN in another account"), the account checked before the region, so another account's ARN in another region is `AccessDeniedException` (measured on DynamoDB for CreateBackup and ListBackups, twice each); an SDK built from the model refuses the ARN form client-side, so only a hand-built request sends it. `TableNotFoundException` "Table not found: " for a missing or `CREATING` table on all four backends, built from the request's name (the backends' own payloads differ). Without a store, and on Cassandra, a table the deployment can create is backed up whatever its name, as the service does (measured on DynamoDB: a table named `...` answered `ContinuousBackupsUnavailableException` "Backups are being enabled for the table: .... Please retry later" 13.2 s after `CreateTable` and was backed up at 33.7 s): the in-catalog format writes no store prefix. With a store, a name over 255 bytes (reachable only with `[limits] max_table_name_length` raised) or made only of dots fits no store prefix, so the store-backed create refuses it with `ValidationException` before it opens a snapshot or writes a row ("TableName cannot be backed up: a backup ARN cannot be formed for it: invalid backup ARN: table name of N bytes is longer than 255" / "... invalid backup ARN: table name \"...\" is only dots"). Every table under a `server.region` outside `[a-z0-9-]` is refused (the `BackupArn` row); the checks run after the table's ACTIVE probe and before the job permit, so the refusal takes no slot. The permit is the last check: a request that finds every `[backup] max_concurrent_jobs` slot held (default 4) waits up to 30 s for one and is then `LimitExceededException` ("Too many backup and restore jobs are running on this server (at most N at once, the [backup] max_concurrent_jobs bound); retry later"), where the service accepted six `CreateBackup` calls sent together (measured on DynamoDB, two runs) | | `DeleteBackup` response | The full `DescribeBackup` description with `BackupStatus` `DELETED`, including `SourceTableDetails.ProvisionedThroughput` (a required member) and `SourceTableFeatureDetails` (measured) | The same on all four backends: the engine reads the backup's recorded definition before the delete and fills the description as `DescribeBackup` does, with `BackupStatus` `DELETED`. A backup whose definition this build cannot read or that does not parse is still deleted and described without feature details (logged at warn), which means a damaged `PROVISIONED` backup is described with on-demand `ProvisionedThroughput` `{0, 0}`: ExtendDB's choice, deletability over the throughput member of a row that is being removed, where `DescribeBackup` on the same row is an `InternalServerError`; a failure to reach the database fails the request and deletes nothing. | | A backup maintenance command holding the catalog's claim | Not applicable: the service has no maintenance command | While `extenddb backup purge` holds the catalog-level maintenance claim, `CreateBackup` and `RestoreTableFromBackup` on every instance of the catalog answer HTTP 400 `LimitExceededException` "A backup maintenance command holds this catalog's maintenance claim; backup and restore jobs are refused until it finishes", after every other check of the request (a request refused on its own terms, a bad table name or override, gets its own error) and before the job permit. The refusal ends when the command releases the claim, or when a command that died goes stale after `[backup] stale_job_after_secs`. | diff --git a/docs/manuals/01-architecture-guide.md b/docs/manuals/01-architecture-guide.md index 7ecd8df1a..401472f38 100755 --- a/docs/manuals/01-architecture-guide.md +++ b/docs/manuals/01-architecture-guide.md @@ -203,7 +203,7 @@ commands at the cost of needing a writable `run_dir`. extenddb uses a dual-database architecture: - **Catalog database** (e.g., `extenddb_catalog`): Stores table metadata, account/user/group/role/policy definitions, access keys, settings, stream metadata, and metrics. Shared across all accounts. -- **Data database** (e.g., `extenddb_catalog_data`): Stores user items, GSI/LSI data, and stream records. Each table gets its own PostgreSQL table. +- **Data database** (`extenddb` for the default catalog `extenddb_catalog`: the catalog's name without its `_catalog` suffix): Stores user items, GSI/LSI data, and stream records. Each table gets its own PostgreSQL table. On PostgreSQL, SQLite, Cassandra and MongoDB the catalog is versioned by its migration ledger, the `extenddb_migrations` table (on MongoDB a collection, in the catalog database and in the data database): the catalog version is the highest migration recorded in every store, and `serve` reads the catalog's status at startup and refuses a catalog that is behind it or carries a breaking migration it does not know. On Cassandra every account keyspace has a ledger beside the catalog keyspace's. A server that refuses its catalog prints the status and the command that resolves it; for pending migrations that is `extenddb migrate --yes`. diff --git a/docs/manuals/02-design-guide.md b/docs/manuals/02-design-guide.md index 571e7014b..601d93541 100755 --- a/docs/manuals/02-design-guide.md +++ b/docs/manuals/02-design-guide.md @@ -11,9 +11,9 @@ examples below name the PostgreSQL databases; other backends map the same two roles onto their own structures (see backend-specific documentation). - **Catalog** (e.g., the `extenddb_catalog` PostgreSQL database): All metadata — table definitions, indexes, accounts, IAM entities, settings, stream metadata, and schema history. -- **Data** (e.g., the `extenddb_catalog_data` PostgreSQL database): User item data. Each DynamoDB table maps to a PostgreSQL table. GSI and LSI data are stored in separate PostgreSQL tables. +- **Data** (the `extenddb` PostgreSQL database for the default catalog `extenddb_catalog`): User item data. Each DynamoDB table maps to a PostgreSQL table. GSI and LSI data are stored in separate PostgreSQL tables. -On PostgreSQL, the data database connection string is stored in the catalog's `settings` table under the key `data_database_url`. This allows the catalog and data databases to live on different PostgreSQL instances. +On PostgreSQL, the data database connection string is stored in the catalog's `settings` table under the key `data_database_connection_string`, and its name under `data_database_name`. This allows the catalog and data databases to live on different PostgreSQL instances. ### Catalog Tables diff --git a/docs/manuals/04-quickstart-setup-guide.md b/docs/manuals/04-quickstart-setup-guide.md index d62fabfb2..d3d197414 100755 --- a/docs/manuals/04-quickstart-setup-guide.md +++ b/docs/manuals/04-quickstart-setup-guide.md @@ -133,6 +133,7 @@ Check the version: # catalog migration 9 (postgres) # commit abc1234 # built 2026-04-17T12:00:00Z +# test-hooks disabled ``` ### Initialization @@ -148,13 +149,15 @@ Options: | Flag | Default | Description | |------|---------|-------------| | `--catalog-db` | `extenddb_catalog` | Catalog database name | -| `--data-db` | `_data` | Data database name | +| `--data-db` | the catalog's name without `_catalog` (`extenddb` for the default catalog) | Data database name | | `--pg-host` | `localhost` | PostgreSQL host | | `--pg-port` | `5432` | PostgreSQL port | -| `--pg-user` | `extenddb` | PostgreSQL user | -| `--pg-password` | `extenddb-local-dev` | PostgreSQL password | +| `--pg-user` | `$USER`, then `postgres` | PostgreSQL admin role, used for `CREATE DATABASE` and `CREATE USER` | +| `--pg-pass` | none (or `EXTENDDB_PG_PASSWORD`) | The admin role's password | +| `--extenddb-user` | `extenddb` | The application role the server connects as | +| `--extenddb-pass` | `extenddb-local-dev` (or `EXTENDDB_APP_PASSWORD`) | The application role's password | -The command generates `extenddb.toml` with the connection details. If `extenddb.toml` already exists, `init` loads defaults from it. +The command generates `extenddb.toml` with the connection details. If `extenddb.toml` already exists, `init` refuses (`Error: Config file "extenddb.toml" already exists. Use --overwrite to delete and regenerate it.`, exit 255); with `--overwrite` it loads the PostgreSQL connection defaults from the existing file and writes a new one. **Save the admin credentials** printed during init. They are shown once and cannot be retrieved later. diff --git a/docs/manuals/05-admin-guide.md b/docs/manuals/05-admin-guide.md index 4da1c0d80..c2b958f6a 100755 --- a/docs/manuals/05-admin-guide.md +++ b/docs/manuals/05-admin-guide.md @@ -145,7 +145,7 @@ Backup and restore jobs are bounded per server process by `[backup] max_concurre #### [backup] -The knobs backup and restore jobs and the reconciler read; every key has a default, and the section may be left out. +The knobs backup and restore jobs and the reconciler read; every key has a default, and the section may be left out. This section and `[backup.store]` below are the reference for the keys and the `extenddb backup` commands; the [Backup and Restore manual](14-backup-and-restore.md) covers the strategy, the recovery point and time, securing the store, the restore drill and catalog-loss recovery. | Key | Default | Description | |-----|---------|-------------| @@ -154,7 +154,7 @@ The knobs backup and restore jobs and the reconciler read; every key has a defau | `stale_job_after_secs` | `300` | Heartbeat age after which a job's owner is dead (on PostgreSQL, once the owner's claim session has ended as well) and the reconciler resolves the job: a dead backup job with a complete manifest under its prefix is AVAILABLE, one without is FAILED and its prefix deleted; a dead in-catalog restore job is FAILED and its target removed; a dead store-backed restore is resumed from its recorded progress (the differences document's restore-interruption row). Must be at least three heartbeat periods, where the period is the larger of `heartbeat_interval_secs` and the 15 s the in-catalog jobs heartbeat at (so 45 at the least, whatever `heartbeat_interval_secs` says); on SQLite a long write transaction delays a heartbeat, so keep a margin above that floor | | `failed_row_retention_secs` | `604800` | How long FAILED and DELETED backup rows of the configured store are kept once their objects are gone (7 days); `extenddb backup list --failed` prints the FAILED ones with their reason. At least `stale_job_after_secs`: the reconciler deletes a stopped job's late object only under its row, so the row outlives the window in which that job can still write; a smaller value is refused at startup | | `reconcile_interval_secs` | `60` | The reconciler's cycle; it also runs once at startup. Each cycle checks the store, resolves backup and restore jobs whose owner is dead, finishes deletions, deletes the objects of FAILED backups once their heartbeat is older than `stale_job_after_secs`, and removes tombstones past `failed_row_retention_secs` (the differences document's restore-interruption row lists the duties) | -| `max_concurrent_jobs` | `4` | Backup and restore jobs this instance runs at once (at most 1024); a request over it waits up to 30 s for a slot, then `LimitExceededException`. On PostgreSQL the pool sizes are raised to `2 * max_concurrent_jobs + 10` when smaller; on SQLite `[storage.sqlite] pool_size` is raised to `2 * (max_concurrent_jobs + 1)`, at most 34, when its snapshot slots (`pool_size / 2 - 1`, between 1 and 16) would refuse a job this knob admits. Memory: a running backup holds about 490 MiB (PostgreSQL) or 650 MiB (SQLite) at most in the pipeline's own terms for a table of items at the 32 MiB line cap, a few tens of MiB for 200-byte items (the figures are for the default `max_item_size_bytes`, 400 KB, where no stored item's line exceeds the cap; on an instance that raises it the terms are at the largest line a stored item expands to, about 68 times the item's size for the worst shape); with glibc's default allocator the process's high-water mark grows with the number of large items a job reads (measured 760 MiB over idle at 48 items of 27.7 MB on a quiet host and 1,281 MiB on a loaded one, 1.39 and 2.35 times the terms at that item size) because the items' strings are allocated on one thread and freed on another and the per-thread arenas keep that memory. Set `MALLOC_ARENA_MAX=1` in the server's environment (the deployment guide's service unit carries the line, commented out) on a host that backs up large items: the peak is then the terms' and does not grow with the table (measured 452 to 481 MiB over idle at 12, 24 and 48 items of 27.7 MB on SQLite, in two series on two hosts' loads, and 340 MiB `VmHWM` (about 200 MiB over idle) at 12 items on PostgreSQL); one arena serializes the worker threads' allocations on one lock and costs the request path a few percent (measured 3.5 to 9.2% on GetItem, Query and Scan on SQLite and about 2% on PostgreSQL (the scans, 2.1%), where the request path waits on the database on PostgreSQL). Size the host for 1.2 times the terms (about 590 MiB on PostgreSQL, 780 MiB on SQLite) times this knob under `MALLOC_ARENA_MAX=1`; without it the peak grows with the large rows a job reads and no bound was measured (1,281 MiB over idle at 48 items of 27.7 MB is the largest seen). A running restore holds the parsed manifest, one decoded data-file line and its parse, the decoded items of 32 MiB of data-file text plus one line per chunk (the chunk closes with the item that reaches the bound) at four chunk positions between the decoder and the load (two in the channel, the one being filled, the one being drained; a chunk also closes at 1,000 items, and the chunks never hold more than one data file's items), and the backend's write batch: at most 32 MiB of item text on SQLite and Cassandra (10,000 items when they are smaller), held beside the decoded items of that text; 500 items (or 4 MiB) per INSERT statement inside one transaction per data file on PostgreSQL; 1,000 documents per `insert_many` on MongoDB (the driver splits the call); times this knob. For a table of large items that is the same order as the backup, not less: measured `VmHWM` over idle from a fresh start (`docs/design/15-backup-restore-bench-pr9.md`), 24 items of 27.7 MB restore with a peak of 848 MiB on SQLite and 778 MiB on PostgreSQL under the default allocator (848 and 1,013 MiB at 12 and 48 such items on SQLite: the peak grows with the large items a restore reads, as the backup's does, and no bound was measured), and 392 MiB on SQLite and 427 MiB on PostgreSQL under `MALLOC_ARENA_MAX=1`, the same setting as above; 1,200 items of 350 KB in one data file 270 MiB on SQLite and 156 MiB on PostgreSQL; 1M items of 200 bytes under a composite key with no secondary index, the backup's lines in key order as the server's own backups are, 89 MiB on SQLite (the write connection's page cache and the final `COUNT(*)`, each bounded by `cache_size`; the count's share was 22 to 28 MiB at 1M and at 3M rows) and 16 MiB on PostgreSQL, and the same 1M items as an unordered backup (every MongoDB-sourced backup and every multi-part-key table) 144 to 164 MiB on SQLite in 66 to 90 s against 27 to 56 s key-ordered (scattered inserts dirty the whole base b-tree; the restore's buffers are the same) and 16 MiB on PostgreSQL; the same items under the same key with two full-projection GSIs 143 to 225 MiB on SQLite at 100,000 items in three runs and 165 to 172 MiB at 300,000, and 23 to 34 MiB on PostgreSQL at both: the excess on SQLite is the backend's own per-connection page cache (`cache_size`, 32 MiB) holding the index tables' b-tree pages plus the allocator's arenas (about a third less under `MALLOC_ARENA_MAX=1`), not the restore's buffers, which the 32 MiB batch bound holds whatever the indexes; it is bounded by `cache_size` and grows sub-linearly with the table. Size the host for about 450 MiB per running restore of large items under `MALLOC_ARENA_MAX=1`, times this knob, plus about 32 MiB of page cache per running restore of an indexed table on SQLite, and read the 1M figure for 200-byte items as the key-ordered shape's: an unordered backup of them restores in about 1.5 to 1.8 times that on SQLite | +| `max_concurrent_jobs` | `4` | Backup and restore jobs this instance runs at once (at most 1024); a request over it waits up to 30 s for a slot, then `LimitExceededException`. On PostgreSQL the pool sizes are raised to `2 * max_concurrent_jobs + 10` when smaller; on SQLite `[storage.sqlite] pool_size` is raised to `2 * (max_concurrent_jobs + 1)`, at most 34, when its snapshot slots (`pool_size / 2 - 1`, between 1 and 16) would refuse a job this knob admits. Memory: a running backup holds about 490 MiB (PostgreSQL) or 650 MiB (SQLite) at most in the pipeline's own terms for a table of items at the 32 MiB line cap, a few tens of MiB for 200-byte items (the figures are for the default `max_item_size_bytes`, 400 KB, where no stored item's line exceeds the cap; on an instance that raises it the terms are at the largest line a stored item expands to, about 68 times the item's size for the worst shape); with glibc's default allocator the process's high-water mark grows with the number of large items a job reads (measured 760 MiB over idle at 48 items of 27.7 MB on a quiet host and 1,281 MiB on a loaded one, 1.39 and 2.35 times the terms at that item size) because the items' strings are allocated on one thread and freed on another and the per-thread arenas keep that memory. Set `MALLOC_ARENA_MAX=1` in the server's environment (the deployment guide's service unit carries the line, commented out) on a host that backs up large items: the peak is then the terms' and does not grow with the table (measured 452 to 481 MiB over idle at 12, 24 and 48 items of 27.7 MB on SQLite, in two series on two hosts' loads, and 340 MiB `VmHWM` (about 200 MiB over idle) at 12 items on PostgreSQL); one arena serializes the worker threads' allocations on one lock and costs the request path a few percent (measured 3.5 to 9.2% on GetItem, Query and Scan on SQLite and about 2% on PostgreSQL (the scans, 2.1%), where the request path waits on the database on PostgreSQL). Size the host for 1.2 times the terms (about 590 MiB on PostgreSQL, 780 MiB on SQLite) times this knob under `MALLOC_ARENA_MAX=1`; without it the peak grows with the large rows a job reads and no bound was measured (1,281 MiB over idle at 48 items of 27.7 MB is the largest seen). A running restore holds the parsed manifest, one decoded data-file line and its parse, the decoded items of 32 MiB of data-file text plus one line per chunk (the chunk closes with the item that reaches the bound) at four chunk positions between the decoder and the load (two in the channel, the one being filled, the one being drained; a chunk also closes at 1,000 items, and the chunks never hold more than one data file's items), and the backend's write batch: at most 32 MiB of item text on SQLite and Cassandra (10,000 items when they are smaller), held beside the decoded items of that text; 500 items (or 4 MiB) per INSERT statement inside one transaction per data file on PostgreSQL; 1,000 documents per `insert_many` on MongoDB (the driver splits the call); times this knob. For a table of large items that is the same order as the backup, not less: measured `VmHWM` over idle from a fresh start (`docs/design/15-backup-restore-bench-pr9.md`), 24 items of 27.7 MB restore with a peak of 848 MiB on SQLite and 778 MiB on PostgreSQL under the default allocator (848 and 1,013 MiB at 12 and 48 such items on SQLite: the peak grows with the large items a restore reads, as the backup's does, and no bound was measured), and 392 MiB on SQLite and 427 MiB on PostgreSQL under `MALLOC_ARENA_MAX=1`, the same setting as above; 1,200 items of 350 KB in one data file 270 MiB on SQLite and 156 MiB on PostgreSQL; 1M items of 200 bytes under a composite key with no secondary index, the backup's lines in key order as the server's own backups are, 89 MiB on SQLite (the write connection's page cache and the final `COUNT(*)`, each bounded by `cache_size`; the count's share was 22 to 28 MiB at 1M and at 3M rows) and 16 MiB on PostgreSQL, and the same 1M items as an unordered backup (every MongoDB-sourced backup and every multi-part-key table) 144 to 164 MiB on SQLite in 66 to 90 s against 27 to 56 s key-ordered (scattered inserts dirty the whole base b-tree; the restore's buffers are the same) and 16 MiB on PostgreSQL; the same items under the same key with two full-projection GSIs 143 to 225 MiB on SQLite at 100,000 items in three runs and 165 to 172 MiB at 300,000, and 23 to 34 MiB on PostgreSQL at both: the excess on SQLite is the backend's own per-connection page cache (`cache_size`, 32 MiB) holding the index tables' b-tree pages plus the allocator's arenas (about a third less under `MALLOC_ARENA_MAX=1`), not the restore's buffers, which the 32 MiB batch bound holds whatever the indexes; it is bounded by `cache_size` and grows sub-linearly with the table. Size the host for about 450 MiB per running restore of large items under `MALLOC_ARENA_MAX=1`, times this knob, plus about 32 MiB of page cache per running restore of an indexed table on SQLite, and read the 1M figure for 200-byte items as the key-ordered shape's: an unordered backup of them restores in about 1.5 to 1.8 times that on SQLite. The page-cache term is for B-tree indexes. On SQLite a restore of items of more than a few hundred bytes, of any attribute types, holds more: its write batch (up to 10,000 items or 32 MiB of item text) and the decode chunks in front of it grow with the item's size up to that bound, and lists of numbers decode larger than strings of the same size; the [Backup and Restore manual](14-backup-and-restore.md#memory-while-a-job-runs) has the measured points | | `spool_path` | (unset) | Restore download area for an S3 store, created `0700` with files `0600`, holding a `.lock` another instance on the host refuses. A store-backed restore from an S3 store downloads each data file here under `restore-
/` (`.json.gz`, `` zero-padded to six digits, with a `.verified` marker holding the file's sha256, written only after the file's digests matched and trusted only while it names the digest the manifest records, plus the manifest and sidecar), reads it from there, and removes the entry when the restore ends, except when it ended because the record had become another instance's (its claim lost, a status write that matched no record), where the sweep removes it; an entry left by an earlier run of this host for a record another host then owned is removed before a new run of the same restore uses it; the reconciler sweeps the `restore-` entry of a restore no longer RUNNING with this host's `spool_host`. Its disk cost is the compressed backup of the restore in flight. A filesystem store needs no spool (files are hashed in place and read again). Required with an S3 store, except on file-backed SQLite where it defaults to `backup-spool/` beside the database file. Refused inside or containing a filesystem store root, and when it is or contains the database file's directory | #### [backup.store] @@ -171,11 +171,11 @@ When the section is absent, the backend decides: on file-backed SQLite it is a f Startup opens the configured store and, for a filesystem store, validates it and checks its identity before the listener is up (bounded by 5 s: a mount that stops answering after the root is opened leaves the server starting with the store `checking`; a root that does not answer at startup holds the server before the listener, as the database directory would, with the port bound and silent); an S3 store's validation and identity check are requests to the endpoint, and run after the listener is up, in the reconciler's first cycle, so an endpoint that stalls never holds the data plane down (the component is `checking` until the first result, and the operations that touch the store refuse meanwhile as they do while it is unhealthy). A configuration error, every one decided without a request to the store, stops startup with the key, path or bucket in the message: a root that does not exist or fails the owner-only rule, a filesystem root another user can reach, a filesystem root that is or contains the database directory, a bucket name the SDK routes to S3 Express, a bad part size or prefix, an insecure endpoint, an S3 store without `spool_path` on a backend with no default, a `spool_path` that breaks its placement rule, that another live instance holds or that exists with a mode other than `0700`. A transport failure (no region, no credentials, an endpoint that refuses connections or does not answer within the check's bound), a missing or denied bucket, an I/O failure of a filesystem root's probe write or resolution (a full or read-only disk, a stale network handle), or a failed identity check leaves the server running with the store unhealthy: the problem is logged once at error, `/health` reports the `backup_store` component unhealthy (status `degraded`), CreateBackup (where it writes to the store) and the restore and delete of a store-backed backup are refused with a `ValidationException` that names the store's kind and the problem's category and not its root, the reconciler acts on no row of the store, and the check runs again each `reconcile_interval_secs` until it passes, at which point the server logs `is healthy again` and resumes without a restart. Each check's store calls are bounded (5 s for a filesystem root, 30 s for a bucket); the catalog's own reads (the catalog id, the rows' store ids, the newest AVAILABLE row) run beside the calls and not under the bound, so a slow catalog costs the check time and never marks the store unhealthy. A store that has stopped answering costs a reconciler cycle that bound and nothing more, the cycle's other duties run as before, and the abandoned check is logged; on a filesystem root the abandoned check's call stays parked on one thread until the mount answers, no later cycle starts a check while it is (logged each cycle, the store staying `unreachable`), a root's calls run one at a time so that calls started before the first reached its bound (two `DeleteBackup` requests together, a `DeleteBackup` beside a check) wait for it under the bound, then have the bound for their own call (a slow call behind another finishes, where one bound over both would abandon it), and park no thread of their own, and `stop` or SIGTERM still ends the server, the runtime giving that thread 5 s before the process exits without it (after the job wait of up to 30 s when a backup job is running; see Stopping). The deletions `DeleteBackup` and the reconciler run against the store go through the same bound and rule: `DeleteBackup` answers the description with `BackupStatus` `DELETED` once its DELETING write has committed, and a deletion the store does not answer within the bound, or that leaves an object under the prefix, leaves the row DELETING for the reconciler and is logged, never an error to the client. A default SQLite root whose startup checks did not finish starts `checking` with its rule kept: a first check that fails while the catalog has no rows for it is the same warning and leaves the store unconfigured. CreateBackup on Cassandra is not refused by an unhealthy store: its in-catalog create touches no store. -The store identity. The root carries `extenddb-store.json`: a random store id and the id of the catalog that owns the root. Startup creates it when the root has no file and the catalog has no rows for the root, and otherwise reads it and compares: a file naming another catalog, a store id that differs from the one the catalog's rows for the root carry, a root with no file while rows exist, or rows carrying more than one id each mark the store unhealthy. Instances that share a catalog must share one store root (one S3 bucket and prefix, or one shared mount): host-local roots configured at the same path give each instance different bytes under one root label, so each check also `head`s the manifest of the newest AVAILABLE backup and marks the store unhealthy when it is absent, which is what a second instance with a local copy of the directory sees. The file is left by `backup purge` and by every deletion; remove it only when retiring the root, or after `extenddb destroy`: a catalog re-created with `init` has a new id, and the old root's file names the old one, so the re-initialised server reports the store owned by another catalog until `extenddb backup sync --adopt-store` makes the new catalog its owner (below), the file is removed or a new root configured (on the default SQLite root, that is the logged warning above for the server, which keeps CreateBackup on the in-catalog path, while `backup sync` handles that root as configured, so `--adopt-store` adopts it). The default root, `backups` beside the database file, is shared by every database file in that directory: `--adopt-store` on it takes the root from whichever catalog wrote its file, so give each database its own `[backup.store]` when several share a directory. A server on an in-memory SQLite database writes the file with `ephemeral_catalog: true`; at its next start it adopts a root whose file names another catalog and carries that marker, and no other, so it never takes a durable catalog's root, and then syncs the root's backups into its new catalog (the `path` row above: until that sync's log line, the earlier backups answer `BackupNotFoundException`). A catalog whose root another catalog adopted (by `--adopt-store`, or a second in-memory server's start) keeps acting on the root until a later check that can read the catalog finds it owned by another catalog, at the earliest one reconciler interval after the cycle in progress ends (a check whose catalog read fails leaves the health as it was), with one exception, its deletion fence: right before it deletes a backup's objects (DeleteBackup, the reconciler's deletions, a failed backup job's cleanup, `purge`), removes a FAILED or DELETED row, before a CreateBackup's row (at the request) and again in the backup job right before its first object, and also before DeleteBackup and `purge` change the row (`purge` reads it once before the row change and its two deletions), a server or `purge` re-reads `extenddb-store.json`, and when the file names another catalog, names this catalog under another store id than the one the store's check settled (a root replaced by another root of this catalog), or the root has none, it deletes nothing and refuses the request. The read records on the store what the next check would record, so the store is unhealthy at once: `owned by another catalog`, `identity mismatch`, `identity file missing` for a root with no file, and, for a file the store refuses to serve or that does not parse, what the check would report (`failed validation` for a file a filesystem root may not read, such as one at mode 000; `permission denied` for an S3 denial; `identity file unreadable` for one that does not parse; `failed validation` too for an S3 server error the SDK's retries did not clear, such as a 503 `SlowDown`, which leaves the store unhealthy until a check passes (the next runs one reconciler interval later)); a CreateBackup refused there answers the unhealthy-store `ValidationException`, and one whose job meets the refusal before its first object is `FAILED` with no object written. The read has a call slot of its own on a filesystem root, so a DeleteBackup's deletion of a prefix of many objects does not hold a CreateBackup's read behind it. Only a read that fails in transport (the store unreachable, the read past the store's 5 s bound) records nothing and leaves the decision to the next check: CreateBackup then answers `InternalServerError`, and DeleteBackup takes it at the read before the row is changed as it takes a store that does not answer: it writes DELETING and answers `DELETED`, the read before the deletion refuses it, and the row stays DELETING for the reconciler, whose deletion runs the fence again. `--prune-missing`'s removal of a row is not fenced: it deletes no object. A refusal at the read before the row is changed (the file names another catalog or another store id) leaves the row as it was, so DescribeBackup still answers AVAILABLE. One at the read before the deletion (an adoption landed between the two) leaves an AVAILABLE row DELETING with nothing deleted (DeleteBackup and DescribeBackup there answer `DELETED`), which this catalog's reconciler finishes only if this catalog owns the root again, and a dead CREATING row that `purge` already removed has its prefix kept. The read narrows the window to the gap between it and the deletion; on S3 that gap is a listing and a `DeleteObjects`, each under the SDK's retries, so seconds when the store throttles. A check that read the file before the adoption and ends after a fence's refusal records the store healthy again until the next cycle's check: each deletion stays fenced in that time, but a restore is accepted. A backup job of the previous owner that fails after the adoption leaves its partial prefix (no manifest, and no row of the new owner) under the root, which no duty or command of the new owner removes. A write that has already passed that read still lands, so stop every server of the catalog a file names before adopting its root. While the catalog has no AVAILABLE backup for a root, two host-local directories at one path cannot be told apart (each instance creates its own file and id, and a file copied from one directory to the other gives both one id), and the manifest probe has nothing to probe: a backup one instance completed under its own directory looks, to the other instance's reconciler, like a dead owner's row with no manifest, which it marks FAILED, and the first instance's restart then deletes the only copy. Point every instance at one shared root before the first backup; copying `extenddb-store.json` between directories defeats the check rather than satisfying it. The identity file is read up to 64 KiB; a larger object under its key is reported unreadable. +The store identity. The root carries `extenddb-store.json`: a random store id and the id of the catalog that owns the root. Startup creates it when the root has no file and the catalog has no rows for the root, and otherwise reads it and compares: a file naming another catalog, a store id that differs from the one the catalog's rows for the root carry, a root with no file while rows exist, or rows carrying more than one id each mark the store unhealthy. Instances that share a catalog must share one store root (one S3 bucket and prefix, or one shared mount): host-local roots configured at the same path give each instance different bytes under one root label, so each check also `head`s the manifest of the newest AVAILABLE backup and marks the store unhealthy when it is absent, which is what a second instance with a local copy of the directory sees. The file is left by `backup purge` and by every deletion; remove it only when retiring the root, or after `extenddb destroy`: a catalog re-created with `init` has a new id, and the old root's file names the old one, so the re-initialised server reports the store owned by another catalog until `extenddb backup sync --adopt-store` makes the new catalog its owner (below), the file is removed or a new root configured (on the default SQLite root, that is the logged warning above for the server, which keeps CreateBackup on the in-catalog path, while `backup sync` handles that root as configured, so `--adopt-store` adopts it). `destroy` leaves the config file as well, which `init` refuses; `init --overwrite` regenerates the whole file from defaults, keeping only the storage connection settings (a `port`, TLS path or limit edited in the default sections reverts too) and dropping `[backup]`, `[backup.store]` and every other added section, so copy the file aside before `init` and put the copy back whole, apart from the storage settings, before the server starts: without `[backup.store]` a PostgreSQL, MongoDB or Cassandra server has no store and `--adopt-store` exits 1, and a SQLite server takes the default root, which for a store configured elsewhere is new and empty, so `--adopt-store` adopts nothing. A root that has to change (a new volume, a new bucket) keeps its backups' rows when its tree is copied under the new root and `extenddb backup sync --move-store ` re-points them (below). The default root, `backups` beside the database file, is shared by every database file in that directory: `--adopt-store` on it takes the root from whichever catalog wrote its file, so give each database its own `[backup.store]` when several share a directory. A server on an in-memory SQLite database writes the file with `ephemeral_catalog: true`; at its next start it adopts a root whose file names another catalog and carries that marker, and no other, so it never takes a durable catalog's root, and then syncs the root's backups into its new catalog (the `path` row above: until that sync's log line, the earlier backups answer `BackupNotFoundException`). A catalog whose root another catalog adopted (by `--adopt-store`, or a second in-memory server's start) keeps acting on the root until a later check that can read the catalog finds it owned by another catalog, at the earliest one reconciler interval after the cycle in progress ends (a check whose catalog read fails leaves the health as it was), with one exception, its deletion fence: right before it deletes a backup's objects (DeleteBackup, the reconciler's deletions, a failed backup job's cleanup, `purge`), removes a FAILED or DELETED row, before a CreateBackup's row (at the request) and again in the backup job right before its first object, and also before DeleteBackup and `purge` change the row (`purge` reads it once before the row change and its two deletions), a server or `purge` re-reads `extenddb-store.json`, and when the file names another catalog, names this catalog under another store id than the one the store's check settled (a root replaced by another root of this catalog), or the root has none, it deletes nothing and refuses the request. The read records on the store what the next check would record, so the store is unhealthy at once: `owned by another catalog`, `identity mismatch`, `identity file missing` for a root with no file, and, for a file the store refuses to serve or that does not parse, what the check would report (`failed validation` for a file a filesystem root may not read, such as one at mode 000; `permission denied` for an S3 denial; `identity file unreadable` for one that does not parse; `failed validation` too for an S3 server error the SDK's retries did not clear, such as a 503 `SlowDown`, which leaves the store unhealthy until a check passes (the next runs one reconciler interval later)); a CreateBackup refused there answers the unhealthy-store `ValidationException`, and one whose job meets the refusal before its first object is `FAILED` with no object written. The read has a call slot of its own on a filesystem root, so a DeleteBackup's deletion of a prefix of many objects does not hold a CreateBackup's read behind it. Only a read that fails in transport (the store unreachable, the read past the store's 5 s bound) records nothing and leaves the decision to the next check: CreateBackup then answers `InternalServerError`, and DeleteBackup takes it at the read before the row is changed as it takes a store that does not answer: it writes DELETING and answers `DELETED`, the read before the deletion refuses it, and the row stays DELETING for the reconciler, whose deletion runs the fence again. `--prune-missing`'s removal of a row is not fenced: it deletes no object. A refusal at the read before the row is changed (the file names another catalog or another store id) leaves the row as it was, so DescribeBackup still answers AVAILABLE. One at the read before the deletion (an adoption landed between the two) leaves an AVAILABLE row DELETING with nothing deleted (DeleteBackup and DescribeBackup there answer `DELETED`), which this catalog's reconciler finishes only if this catalog owns the root again, and a dead CREATING row that `purge` already removed has its prefix kept. The read narrows the window to the gap between it and the deletion; on S3 that gap is a listing and a `DeleteObjects`, each under the SDK's retries, so seconds when the store throttles. A check that read the file before the adoption and ends after a fence's refusal records the store healthy again until the next cycle's check: each deletion stays fenced in that time, but a restore is accepted. A backup job of the previous owner that fails after the adoption leaves its partial prefix (no manifest, and no row of the new owner) under the root, which no duty or command of the new owner removes. A write that has already passed that read still lands, so stop every server of the catalog a file names before adopting its root. While the catalog has no AVAILABLE backup for a root, two host-local directories at one path cannot be told apart (each instance creates its own file and id, and a file copied from one directory to the other gives both one id), and the manifest probe has nothing to probe: a backup one instance completed under its own directory looks, to the other instance's reconciler, like a dead owner's row with no manifest, which it marks FAILED, and the first instance's restart then deletes the only copy. Point every instance at one shared root before the first backup; copying `extenddb-store.json` between directories defeats the check rather than satisfying it. The identity file is read up to 64 KiB; a larger object under its key is reported unreadable. -`extenddb destroy` removes the databases and leaves the store untouched: it is the copy that survives destroy. `extenddb backup list [--failed]` prints the catalog's rows for the configured store's root, every store id they carry, with each row's `store_root` and `store_id`; it is a read that runs beside a server (on SQLite it takes no serve lock), reads the identity file without creating one, and runs on an unhealthy store, naming the problem in its header. It lists backups only, never restore jobs: a restore's record is the job's log line, not a `backup list` row. `extenddb backup purge --confirm` deletes every backup of the configured store, each backup's objects (its own prefix, never a whole account prefix, so a prefix the catalog has no row for survives) and its catalog row outright, leaving no tombstone, and leaves `extenddb-store.json` and the `exports/` component; it needs the store healthy. It takes the catalog-level maintenance claim first and heartbeats it for the command's whole run, waits up to `2 * stale_job_after_secs` until no store-backed CreateBackup is running on any instance (a CREATING or FAILED row with a heartbeat within `stale_job_after_secs` on the catalog database's clock) and no restore is RUNNING, refuses and releases the claim if they have not finished, deletes a CREATING row the wait saw no live owner for as it deletes any other row, finishes any DeleteBackup left in progress as part of the purge (a backup a DeleteBackup finishes while the purge runs is counted as deleted, not left), and logs each deletion under the `extenddb::audit` target with the account and the ARN. While the claim is held, CreateBackup and RestoreTableFromBackup on every instance answer `LimitExceededException` saying that a backup maintenance command holds the catalog's maintenance claim (the message does not say which command; the second command's refusal and the claim's row do), and a second maintenance command fails at once naming the holder. Without `--confirm` it prints the plan, changes nothing and exits 0. `--config` is accepted before or after the subcommand. None of the backup commands runs the startup recovery a server runs (on PostgreSQL the vector index rebuild, the control-plane recovery and the GSI queue workers; on MongoDB the key-repair pass; on Cassandra the control-plane transitions; on SQLite the interrupted-restore and index passes): that work belongs to the servers, and a command that ran it could exit with it half done. RUNNING restores are read by status, with no age, so a restore whose record was stamped this instant holds the wait too. On SQLite `purge --confirm` opens the database as `serve` does and so takes the serve lock: stop the server first (while one runs on the file the command refuses with exit 6, as a second `serve` does); under that lock it first resolves what the stopped server left (a RUNNING restore is failed and its target removed, the CREATING target of a FAILED restore removed, an in-catalog CREATING backup removed, a DELETING row finished) and deletes a CREATING or FAILED row whatever its heartbeat says, a heartbeat stamped ahead of the clock by a backward clock step included (the server that wrote it is stopped, so the zero window the command passes selects by status alone), so such a job never makes it wait or refuse. That includes a RUNNING restore from a backup store, which a server would resume: under the serve lock the command fails it and removes its target, as for an in-catalog restore (a resume is a job, which the maintenance claim excludes), except one that recorded no progress while its target holds items, which is failed `target_not_empty` and its target left for `extenddb backup remove-target`. A server starting on the file removes a claim a killed purge left, so no job is refused for the stale window, as a purge run again after the killed one takes its claim over. On the other backends it runs beside live servers, and a claim whose holder died expires after `stale_job_after_secs`. On PostgreSQL, MongoDB and Cassandra the backup commands (`list`, `purge`, and `sync` and `remove-target` below) open the catalog as a server does and hold a serving registration for their run, named after the command (`extenddb backup list`, `extenddb backup purge`, `extenddb backup sync`, `extenddb backup remove-target`: on PostgreSQL the session's `application_name`, on MongoDB the `command` field of its `extenddb_servers` document, on Cassandra the `release` column of its `extenddb_servers` row, after the release and ` as `), and release it when they exit: `extenddb migrate --yes` waits for them as for a server and names them while it waits, and while a migration runs they refuse at once (exit 6), as they do on a catalog at another migration than the binary's (exit 3, with the `extenddb migrate --yes` advice `serve` prints). On every backend Ctrl-C or `SIGTERM` makes the command release what it holds and exit: the serving registration after the open, if the signal came during it; the claim at once while it waits, and before its next backup while it deletes (the release itself is bounded by 10 s, past which the claim is left to the stale rule), so an interrupted purge leaves no half-deleted backup and no claim to wait out; run it again for the rest. The dry run on an unhealthy store lists the rows and says that `--confirm` refuses until the store's check passes. +`extenddb destroy` removes the databases and leaves the store untouched: it is the copy that survives destroy. `extenddb backup list [--failed]` prints the catalog's rows for the configured store's root, every store id they carry, with each row's `store_root` and `store_id`; it is a read that runs beside a server (on SQLite it takes no serve lock), reads the identity file without creating one, and runs on an unhealthy store, naming the problem in its header. It lists backups only, never restore jobs: a restore's record is the job's log line, not a `backup list` row. `extenddb backup purge --confirm` deletes every backup of the configured store, each backup's objects (its own prefix, never a whole account prefix, so a prefix the catalog has no row for survives) and its catalog row outright, leaving no tombstone, and leaves `extenddb-store.json` and the `exports/` component; it needs the store healthy. It takes the catalog-level maintenance claim first and heartbeats it for the command's whole run, waits up to `2 * stale_job_after_secs` until no store-backed CreateBackup is running on any instance (a CREATING or FAILED row with a heartbeat within `stale_job_after_secs` on the catalog database's clock) and no restore is RUNNING, refuses and releases the claim if they have not finished, deletes a CREATING row the wait saw no live owner for as it deletes any other row, finishes any DeleteBackup left in progress as part of the purge (a backup a DeleteBackup finishes while the purge runs is counted as deleted, not left), and logs each deletion under the `extenddb::audit` target with the account and the ARN. While the claim is held, CreateBackup and RestoreTableFromBackup on every instance answer `LimitExceededException` saying that a backup maintenance command holds the catalog's maintenance claim (the message does not say which command; the second command's refusal and the claim's row do), and a second maintenance command fails at once naming the holder. Without `--confirm` it prints the plan, changes nothing and exits 0. `--config` is accepted before or after the subcommand. None of the backup commands runs the startup recovery a server runs (on PostgreSQL the vector index rebuild, the control-plane recovery and the GSI queue workers; on MongoDB the key-repair pass; on Cassandra the control-plane transitions; on SQLite the interrupted-restore and index passes): that work belongs to the servers, and a command that ran it could exit with it half done. RUNNING restores are read by status, with no age, so a restore whose record was stamped this instant holds the wait too. On SQLite `purge --confirm` opens the database as `serve` does and so takes the serve lock: stop the server first (while one runs on the file the command refuses with exit 6, as a second `serve` does); under that lock it first resolves what the stopped server left (a RUNNING restore is failed and its target removed, the CREATING target of a FAILED restore removed, an in-catalog CREATING backup removed, a DELETING row finished) and deletes a CREATING or FAILED row whatever its heartbeat says, a heartbeat stamped ahead of the clock by a backward clock step included (the server that wrote it is stopped, so the zero window the command passes selects by status alone), so such a job never makes it wait or refuse. That includes a RUNNING restore from a backup store, which a server would resume: under the serve lock the command fails it and removes its target, as for an in-catalog restore (a resume is a job, which the maintenance claim excludes), except one that recorded no progress while its target holds items, which is failed `target_not_empty` and its target left for `extenddb backup remove-target`. A server starting on the file removes a claim a killed purge left, so no job is refused for the stale window, as a purge run again after the killed one takes its claim over. On the other backends it runs beside live servers, and a claim whose holder died expires after `stale_job_after_secs`. On PostgreSQL, MongoDB and Cassandra the backup commands (`list`, `purge`, and `sync` and `remove-target` below) open the catalog as a server does and hold a serving registration for their run, named after the command (`extenddb backup list`, `extenddb backup purge`, `extenddb backup sync`, `extenddb backup remove-target`: on PostgreSQL the session's `application_name`, on MongoDB the `command` field of its `extenddb_servers` document, on Cassandra the `release` column of its `extenddb_servers` row, after the release and ` as `), and release it when they exit: `extenddb migrate --yes` waits for them as for a server and names them while it waits, and while a migration runs they refuse at once (exit 6). On a catalog `serve` would refuse they exit with `serve`'s code and message: 3 with migrations pending (with the `extenddb migrate --yes` advice), 4 on a catalog a later release migrated with a breaking migration, and 5 on one that needs attention (the upgrade manual's [catalog status](07-upgrade-manual.md#the-catalog-status) table). On every backend Ctrl-C or `SIGTERM` makes the command release what it holds and exit: the serving registration after the open, if the signal came during it; the claim at once while it waits, and before its next backup while it deletes (the release itself is bounded by 10 s, past which the claim is left to the stale rule), so an interrupted purge leaves no half-deleted backup and no claim to wait out; run it again for the rest. The dry run on an unhealthy store lists the rows and says that `--confirm` refuses until the store's check passes. -`extenddb backup sync [--adopt-store] [--prune-missing]` rebuilds the catalog's rows from the configured store: after a lost or re-created catalog, or after a backup's prefix was copied under the root from another deployment or another backend. It reads the catalog's rows for the root (of every status and every store id they carry), then lists the whole root and handles each `/
//` prefix as the listing passes it, checking it against those rows before it reads anything under it. Sync holds one prefix's object list at a time and the prefix of each manifest it saw; a filesystem store builds its listing of the whole root in memory first (about 72 bytes plus the key per object), an S3 store lists a page at a time. The report keeps an entry per prefix for the whole run, and it lists every prefix the catalog already has a row for among the skipped ones, so on a root of small backups the per-prefix term dominates: about 350 bytes per prefix at one object per prefix (62 MiB of peak RSS at 50,000 such prefixes and 112 MiB at 200,000, against 45 MiB for an empty root; about 165 bytes per object at 100 objects per prefix), measured on a SQLite debug build with a filesystem root and no rows. Of the catalog's rows it keeps each row's status, store id and table, and the key of each AVAILABLE row of the configured store, but every backend reads the rows whole before sync sees the first, so the read is the larger cost and grows with the rows of the root, tombstones within their retention included: about 22.5 KiB per row of a table with five GSIs (1,452 bytes of source table description per row; 610 MiB of peak RSS at 25,000 rows and 1,144 MiB at 50,000, against 47 MiB with none; 148 MiB at 50,000 rows whose description is `{}`), measured on a SQLite debug build on a 16-core x86_64 Linux host with glibc's default arenas. `backup list` peaks the same over those rows. A prefix with no row gets an AVAILABLE row under the configured store's identity when its manifest and sidecar verify, every data file the manifest lists is under the prefix at its recorded size, and the manifest's ARN names the prefix's account, table and backup id; an ARN of another region is adopted under this server's region and logged (requests find it under either spelling), and one of another partition than `aws` is skipped, since every request refuses such an ARN. The row carries the manifest's backup name, size, item count and source table, and the backup id's instant as its creation time. It never adopts a prefix with a row of any status (CREATING, DELETING, DELETED, FAILED or AVAILABLE), one whose manifest, sidecar or data files are missing or do not match, one whose manifest names another backup, or anything under `exports/` or a table's `changelog/`, and prints each with its reason. Before the insert it reads the row under both spellings of the ARN (this server's region and the manifest's), and the insert happens only if no row exists: a CreateBackup whose row landed after sync read the rows keeps its row, as does a row of another root, account or table holding the backup id, and sync changes no existing row. It also prints the AVAILABLE rows whose manifest the listing does not hold. It exits 0 once it ran, skipped prefixes included, and 1 when a prefix was skipped because a store read or a catalog call failed (the run is incomplete, and its summary line says `incomplete`; a second run adopts it), when the listing failed part-way (the backups adopted before the failure keep their rows), when a signal stopped it, or when `--prune-missing` left a row it may remove on a later run (a store or catalog call failed, its manifest's `head` included, or a restore is reading the backup: run the command again once the restore ends). A row `--prune-missing` leaves because its manifest is back or another writer moved it is reported, and the exit stays 0: +`extenddb backup sync [--adopt-store] [--prune-missing]` (or with one of `--move-store `, `--forget-store ` and `--forget-backup `, below) rebuilds the catalog's rows from the configured store: after a lost or re-created catalog, or after a backup's prefix was copied under the root from another deployment or another backend. It reads the catalog's rows for the root (of every status and every store id they carry), then lists the whole root and handles each `/
//` prefix as the listing passes it, checking it against those rows before it reads anything under it. Sync holds one prefix's object list at a time and the prefix of each manifest it saw; a filesystem store builds its listing of the whole root in memory first (about 72 bytes plus the key per object), an S3 store lists a page at a time. The report keeps an entry per prefix for the whole run, and it lists every prefix the catalog already has a row for among the skipped ones, so on a root of small backups the per-prefix term dominates: about 350 bytes per prefix at one object per prefix (62 MiB of peak RSS at 50,000 such prefixes and 112 MiB at 200,000, against 45 MiB for an empty root; about 165 bytes per object at 100 objects per prefix), measured on a SQLite debug build with a filesystem root and no rows. Of the catalog's rows it keeps each row's status, store id and table, and the key of each AVAILABLE row of the configured store, but every backend reads the rows whole before sync sees the first, so the read is the larger cost and grows with the rows of the root, tombstones within their retention included: about 22.5 KiB per row of a table with five GSIs (1,452 bytes of source table description per row; 610 MiB of peak RSS at 25,000 rows and 1,144 MiB at 50,000, against 47 MiB with none; 148 MiB at 50,000 rows whose description is `{}`), measured on a SQLite debug build on a 16-core x86_64 Linux host with glibc's default arenas. `backup list` peaks the same over those rows. A prefix with no row gets an AVAILABLE row under the configured store's identity when its manifest and sidecar verify, every data file the manifest lists is under the prefix at its recorded size, and the manifest's ARN names the prefix's account, table and backup id; an ARN of another region is adopted under this server's region and logged (requests find it under either spelling), and one of another partition than `aws` is skipped, since every request refuses such an ARN. The row carries the manifest's backup name, size, item count and source table, and the backup id's instant as its creation time. It never adopts a prefix with a row of any status (CREATING, DELETING, DELETED, FAILED or AVAILABLE), one whose manifest, sidecar or data files are missing or do not match, one whose manifest names another backup, or anything under `exports/` or a table's `changelog/`, and prints each with its reason. Before the insert it reads the row under both spellings of the ARN (this server's region and the manifest's), and the insert happens only if no row exists: a CreateBackup whose row landed after sync read the rows keeps its row, as does a row of another root, account or table holding the backup id, and sync changes no existing row. It also prints the AVAILABLE rows whose manifest the listing does not hold, and, once the listing has reached its end, the rows of the other stores (below). It exits 0 once it ran, skipped prefixes included, and 1 when a prefix was skipped because a store read or a catalog call failed (the run is incomplete, and its summary line says `incomplete`; a second run adopts it), when the listing failed part-way (the backups adopted before the failure keep their rows), when a signal stopped it, when the rows of the other stores could not be read, or when `--prune-missing` left a row it may remove on a later run (a store or catalog call failed, its manifest's `head` included, or a restore is reading the backup: run the command again once the restore ends). A row `--prune-missing` leaves because its manifest is back or another writer moved it is reported, and the exit stays 0: ``` === extenddb backup sync === @@ -190,10 +190,32 @@ Store: filesystem (store id ) --- AVAILABLE rows whose manifest is not in the store (the rows were AVAILABLE when the catalog was read): (none) +--- Rows of other stores (left as they are; sync reads and changes none of them): + filesystem /srv/backups/old (store id ): 41 AVAILABLE, 1 DELETING, 2 DELETED + DELETING (account ) + (to keep a root's backups, copy its tree under the configured root and run extenddb backup sync --move-store ; to drop its rows, extenddb backup sync --forget-store ) + === extenddb backup sync complete: 1 backup(s) adopted, 1 prefix(es) skipped, 0 AVAILABLE row(s) whose manifest is gone === ``` -Sync needs the store healthy: on an unhealthy store it refuses, exit 1, naming the problem and changing no row. That covers an empty root under existing rows, a root whose file carries another store id and a root without the newest AVAILABLE row's manifest, which is what an unmounted volume shows. When the newest AVAILABLE row's own manifest is gone, the store is unhealthy for the same reason, so plain sync and `--prune-missing` both refuse; put the prefix (or its manifest) back and run the command again. On PostgreSQL, MongoDB and Cassandra every sync takes the maintenance claim and heartbeats it, so no `purge` runs between its read of the rows and its insert while the claim is held. Each command that holds the claim reads it lost once no heartbeat confirmed it for two heartbeat periods, and sends one heartbeat of its own right before each step it cannot undo (sync's insert, `--prune-missing`'s removal, `--adopt-store`'s rewrite of the identity file, `purge`'s removal of a dead CREATING row and of that row's prefix), stopping when the claim is lost: on MongoDB and Cassandra a command stopped for `stale_job_after_secs` (a suspended process, a paused VM) can have its claim taken over by another, and then stops before that step. A call the command had already sent when it was stopped still lands when it resumes. Plain sync waits for no job, and while it holds the claim CreateBackup and RestoreTableFromBackup are refused, as during `purge`. A sync killed while it holds the claim (`SIGKILL`, a lost host) leaves it held until its heartbeat is `stale_job_after_secs` old (300 s by default): until then every instance refuses CreateBackup and RestoreTableFromBackup. On PostgreSQL the claim is a session's advisory lock, so the next maintenance command (the sync or `purge` run again) takes a killed command's claim over at once and ends that window; MongoDB and Cassandra take it over only once it is stale. `--prune-missing` and `--adopt-store` also wait for running jobs as `purge` does. A first Ctrl-C or `SIGTERM` stops any sync before its next prefix (or the wait at once), releases the claim it holds and exits 1; the backups adopted before the stop keep their rows. `--prune-missing` removes each AVAILABLE row whose manifest is gone outright, one line per row, and deletes nothing in the store: no DELETING or DELETED row is left for a deletion duty to act on, so on host-local roots (each instance's root at one path holding other bytes) a prune on the instance that lacks a backup's bytes removes its row for every instance of the catalog and leaves the bytes on the other host: the backup answers `BackupNotFoundException` on every instance, the one holding the bytes included, until a sync on that instance adopts them again. A row a restore is reading, or one another writer moved meanwhile, is reported and left (the first exits 1). The objects a removed row's prefix still holds stay until an operator deletes the prefix by hand: `purge` deletes only the prefixes of rows, and every later sync reports the prefix as not a complete backup. `--adopt-store` is the way a durable catalog takes a root whose `extenddb-store.json` names another catalog (plain sync refuses such a root, naming the option; a server on an in-memory database adopts, at startup, a root another in-memory catalog wrote, as the identity paragraph says): under the claim it first runs the check the server would run on the root once this catalog owns it (this catalog's rows for the root against the file's store id, and the manifest of this catalog's newest AVAILABLE backup of the root) and refuses, with nothing written and the file still naming its owner, a root that check fails (a backup of this catalog whose manifest is gone from the root, say); then it logs its intent under `extenddb::audit`, rewrites the file's catalog id and backend, keeping the store id and creation time, reads the file back and logs the outcome (adopted, or the race lost when another adopter's write landed between the two), and syncs. Each store call runs under the store's 5 s bound (the catalog's reads and the claim's heartbeat do not count against it), and a store that stops answering ends the command with exit 1 naming the step that did not finish (the first read, the rewrite, whose outcome the audit log's intent line then leaves open, or the re-read). Every sync, plain or adopting, reads the file once more at the end and fails, exit 1, naming the owner, when another catalog adopted the root during the run: the rows it inserted are then rows of a root this catalog does not own, inert until this catalog adopts the root again. A write that lands after that read is found by this catalog's next check. It refuses when this catalog's rows for the root carry another store id, and on a root this catalog already owns it says so and syncs. Stop every server of the catalog the file names before running it: the previous owner finds the store owned by another catalog only at a later check that can read its catalog, and until then its deletion fence (the identity paragraph) refuses each deletion and CreateBackup once it re-reads the file, but a write already past that read lands. A second maintenance command while one holds the claim fails at once. On SQLite every form takes the serve lock: stop the server first (beside a running server it exits 6). Plain sync there resolves nothing the stopped server left: the lock already keeps every server and command off the file, so a CREATING or DELETING backup and a RUNNING restore the server left are rows sync skips, and the next server start resolves them and resumes the restore. `--prune-missing` and `--adopt-store` take the claim and first resolve what the stopped server left, as `purge` does, which fails a store-backed restore the server left RUNNING and removes its target; each prints what it resolved. The rows of other roots are reported in part: a prefix whose backup id a row of another root holds is skipped naming that root; rows of other roots as such are not listed. +Sync needs the store healthy: on an unhealthy store it refuses, exit 1, naming the problem and changing no row. That covers an empty root under existing rows, which is what an unmounted volume shows ("the root has no extenddb-store.json but the catalog has rows for it"; `--forget-backup` leaves that root's rows and `--forget-store` refuses the configured root, below), a root whose file carries another store id and a root without the newest AVAILABLE row's manifest, which a root not shared by every server (another host's directory at the same path) shows too. When the newest AVAILABLE row's own manifest is gone, the store is unhealthy for the same reason, so plain sync and `--prune-missing` both refuse; put the prefix (or its manifest) back and run the command again, or, when that backup is gone for good, remove its row with `--forget-backup ` (below), which runs on the unhealthy store. On PostgreSQL, MongoDB and Cassandra every sync takes the maintenance claim and heartbeats it, so no `purge` runs between its read of the rows and its insert while the claim is held. Each command that holds the claim reads it lost once no heartbeat confirmed it for two heartbeat periods, and sends one heartbeat of its own right before each step it cannot undo (sync's insert, `--prune-missing`'s removal, `--adopt-store`'s rewrite of the identity file, `purge`'s removal of a dead CREATING row and of that row's prefix, `--move-store`'s resolution of a dead CREATING row, its finished deletion of a DELETING row and its rewrite, each removal, DELETED write and finished deletion of `--forget-store` and `--forget-backup`, and `--forget-store`'s sweep of the status entries), stopping when the claim is lost: on MongoDB and Cassandra a command stopped for `stale_job_after_secs` (a suspended process, a paused VM) can have its claim taken over by another, and then stops before that step. A call the command had already sent when it was stopped still lands when it resumes. Plain sync waits for no job, and while it holds the claim CreateBackup and RestoreTableFromBackup are refused, as during `purge`. A sync killed while it holds the claim (`SIGKILL`, a lost host) leaves it held until its heartbeat is `stale_job_after_secs` old (300 s by default): until then every instance refuses CreateBackup and RestoreTableFromBackup. On PostgreSQL the claim is a session's advisory lock, so the next maintenance command (the sync or `purge` run again) takes a killed command's claim over at once and ends that window; MongoDB and Cassandra take it over only once it is stale. `--prune-missing` and `--adopt-store` also wait for running jobs as `purge` does. A first Ctrl-C or `SIGTERM` stops any sync before its next prefix (or the wait at once), releases the claim it holds and exits 1; the backups adopted before the stop keep their rows. `--prune-missing` removes each AVAILABLE row whose manifest is gone outright, one line per row, and deletes nothing in the store: no DELETING or DELETED row is left for a deletion duty to act on, so on host-local roots (each instance's root at one path holding other bytes) a prune on the instance that lacks a backup's bytes removes its row for every instance of the catalog and leaves the bytes on the other host: the backup answers `BackupNotFoundException` on every instance, the one holding the bytes included, until a sync on that instance adopts them again. A row a restore is reading, or one another writer moved meanwhile, is reported and left (the first exits 1). The objects a removed row's prefix still holds stay until an operator deletes the prefix by hand: `purge` deletes only the prefixes of rows, and every later sync reports the prefix as not a complete backup. `--adopt-store` is the way a durable catalog takes a root whose `extenddb-store.json` names another catalog (plain sync refuses such a root, naming the option; a server on an in-memory database adopts, at startup, a root another in-memory catalog wrote, as the identity paragraph says): under the claim it first runs the check the server would run on the root once this catalog owns it (this catalog's rows for the root against the file's store id, and the manifest of this catalog's newest AVAILABLE backup of the root) and refuses, with nothing written and the file still naming its owner, a root that check fails (a backup of this catalog whose manifest is gone from the root, say); then it logs its intent under `extenddb::audit`, rewrites the file's catalog id and backend, keeping the store id and creation time, reads the file back and logs the outcome (adopted, or the race lost when another adopter's write landed between the two), and syncs. Each store call runs under the store's 5 s bound (the catalog's reads and the claim's heartbeat do not count against it), and a store that stops answering ends the command with exit 1 naming the step that did not finish (the first read, the rewrite, whose outcome the audit log's intent line then leaves open, or the re-read). Every sync, plain or adopting, reads the file once more at the end and fails, exit 1, naming the owner, when another catalog adopted the root during the run: the rows it inserted are then rows of a root this catalog does not own, inert until this catalog adopts the root again. A write that lands after that read is found by this catalog's next check. It refuses when this catalog's rows for the root carry another store id, and on a root this catalog already owns it says so and syncs. Stop every server of the catalog the file names before running it: the previous owner finds the store owned by another catalog only at a later check that can read its catalog, and until then its deletion fence (the identity paragraph) refuses each deletion and CreateBackup once it re-reads the file, but a write already past that read lands. A second maintenance command while one holds the claim fails at once. On SQLite every form takes the serve lock: stop the server first (beside a running server it exits 6). Plain sync there resolves nothing the stopped server left: the lock already keeps every server and command off the file, so a CREATING or DELETING backup and a RUNNING restore the server left are rows sync skips, and the next server start resolves them and resumes the restore. `--prune-missing` and `--adopt-store` take the claim and first resolve what the stopped server left, as `purge` does, which fails a store-backed restore the server left RUNNING and removes its target; each prints what it resolved. Sync never changes a row of another store (another kind or root, or the configured root under another store id): a prefix whose backup id such a row holds is skipped naming that row's status and store, and the report's section of other stores, printed once the listing has reached its end and skipped when the run stopped, gives each other store with its rows per status and each of its CREATING and DELETING rows by ARN (the rows the reconciler of the configured store leaves alone), then the two options that act on them (a store of the configured root under another store id, which both options refuse, names `--forget-backup ` on its line instead); on Cassandra the summary counts status entries, which a write that stopped part-way, a status change whose entry batch failed, or a deleter whose entry batch landed after a move of its row, can leave with no row of their status behind them, so each count is checked against the root's listing (which finds a row whatever status its entry is under, and counts it under its own) and what no row of its status stands behind is printed as status entries (with no row of the root left, `--forget-store ` removes them). + +`extenddb backup sync --move-store ` re-points the rows of a root whose tree was copied under the configured root, so the backups keep their ARNs (the backup manual's [Moving a store to a new root](14-backup-and-restore.md#moving-a-store-to-a-new-root) gives the procedure: reconfigure every server first, copy the tree without the old root's `extenddb-store.json`, which would turn every running server's store unhealthy, and finish the copy before the move). The old root is given as the catalog stores it and as the report of other stores prints it, and matches byte for byte (a trailing `/`, or an S3 root's `?endpoint=` suffix, makes another root); a root with no rows is refused, printing the stores the catalog holds rows for, and so is the configured root. It needs the configured store healthy, takes the maintenance claim on every backend (on SQLite under the serve lock as well) and waits, as `purge` does, for the running jobs of the configured store and of every store id the catalog records for the old root. It then checks each row of the old root against the configured store: a CREATING row whose owner is dead is resolved AVAILABLE when the copy holds its complete backup, and otherwise stops the move with no status written (the copy proves nothing about the old root's prefix, and a FAILED row would have the reconciler delete the copy: a reconciler on the old root resolves it there, or `--forget-backup` removes it), and one with a live owner stops the move; a DELETING row whose copied prefix holds no verified manifest has its deletion finished in the configured store, behind the deletion fence; a DELETING, DELETED or FAILED row whose copied prefix holds a verified manifest stops the move, naming `--forget-backup`, since moving the tombstone would have the reconciler delete a complete backup; an AVAILABLE row needs its prefix complete under the configured root (the manifest and its sidecar, the row's ARN, every data file at its recorded size, the row's size and item count); every row of the old root that would give an account two rows of one backup id (with a row of the configured root, or, on Cassandra, with another spelling of its ARN among the moved rows) stops it, all of them set aside before any row is checked, so none of them is resolved or has its deletion finished. Every row that stops the move is printed with its reason in one run, and while any does no row moves and the command exits 1 (the rows resolved AVAILABLE and the finished deletions keep their new status; a second run sees them). Otherwise the identity file is re-read once, so no row moves into a root another catalog adopted since the check, the rows' store is rewritten (only rows still of the old root and not CREATING) in slices of 500 keys (on Cassandra, which rewrites key by key, one key per call), the claim confirmed with a beat before each 500 keys and its state read, with no write, before every call, so a move whose claim another command took over (on MongoDB and Cassandra, a move stopped past `stale_job_after_secs`) writes nothing after the one call it had sent or was about to send when it stopped (one key on Cassandra, one statement of up to 500 keys on MongoDB) and exits 1 with the earlier calls' rows moved and the rest listed under the old root, for a second run; a first Ctrl-C or `SIGTERM` during the rewrite stops it the same way before its next call; each moved row is logged under `extenddb::audit`; on PostgreSQL the rewrite runs in batches of at most 5,000 keys (a slice is one), each one statement in a transaction of its own under a 3 s lock timeout (up to three attempts) and a 30 s statement timeout (on the host above a batch of 5,000 keys took 0.23 to 0.29 s over a catalog of 400,000 rows, where one statement over 200,000 keys took 16.4 and 16.9 s); then the old root is listed again (a row still there, as a CreateBackup of a server left on the old root leaves it, exits 1: its prefix was written after the copy, so reconfigure that server, copy the prefix under the new root or remove the row with `--forget-backup`, and run the move again; a row the old root no longer lists is counted as moved only when its row, read by its ARN, names the configured store, and one that names no such store is reported as gone), and a plain sync of the configured root follows with its report; a row of the old root in that report's section of other stores is named and the command exits 1 ("a row landed under the old root during the move"). A CreateBackup reads the maintenance claim once more right before its row's insert and a restore right before its registration, so one whose read comes after the move took the claim lands no row; one whose read came before lands its row whenever its insert does, from that read on: during the move's wait (which lists it), before the rewrite (which skips CREATING rows, and the re-list reports it), during the move's sync (its closing report names it if the insert lands before that report), or after the command (only a later plain sync reports it). Before deleting the old root, run plain sync and check that its report of other stores names no row of it. Nothing under the old root is read or changed, and no object is copied; the reconciler then cleans the leftovers of the moved FAILED and DELETED rows under the configured root. Its peak memory grows with the rows of both roots, since every backend reads a listing's rows whole: it reads the old root's rows, which it keeps without their parsed description, and the configured root's rows, for the check of one backup id per account (it keeps their account and backup id; the plain sync after the move reads them again), and after the rewrite it reads each moved row by its ARN, 16 at a time, rather than listing the configured store. Each read is about 23 KiB per row on PostgreSQL and 27 on SQLite, above about 50 MiB, at 1,452 bytes of table description per row (the figure grows with the description): a move of 20,000 rows of a table with five GSIs into an empty store peaked at 569 MiB on a SQLite debug build and 500 MiB on a PostgreSQL debug build (51 and 49 MiB at 50 rows), 37 s each, and a move of 50 rows on the PostgreSQL debug build at 49 MiB into an empty store and 231 MiB into a store of 8,000 rows (1,569 bytes of description per row; 1.8 and 5.8 s), on the 16-core host above with filesystem roots on a local disk. + +`--forget-store ` removes every row of a root other than the configured one outright, of any status, leaving no DELETED tombstone; if the root is configured again later, sync adopts its backups again. `--forget-backup ` does the same for one backup of any status and any store, and can be given more than once (the ARNs are taken in order, up to the first that is refused; an ARN no row and no in-catalog backup carries is reported as having nothing to remove and counts as done, so a command line can be run again after a stop; every ARN's form is checked before the claim is taken). The removals delete no object and read no store, so both run on a store in any state and with no store configured (`kind = "disabled"` included), though a configured root that cannot be opened (a directory that does not exist) stops them before the claim, as it stops plain sync, and `kind = "disabled"` then runs them; both take the maintenance claim on every backend and wait for running jobs as `purge` does (over the configured root's store ids too when its identity is not known). They open the store writing nothing to it: the identity file is read, never created, and when it agrees with the rows the store's check runs once without writing (no `validate` probe file). A reconciler's FAILED-prefix, DELETED-prefix and tombstone duties read the claim again right before each deletion and start none while it is held; each deletion that read it free ends 30 s after that read was sent (a read answered later starts nothing, and the store checks the deadline before each removal, or each request on S3), and on PostgreSQL, MongoDB and Cassandra the command waits 60 s from taking the claim (the 30 s and one store call's 30 s) before its first removal of a FAILED or DELETED row, printing the wait; on SQLite the serve lock excludes every server and nothing waits. What a duty can still do to a removed row's bytes is the one removal, or one S3 request (up to 1,000 keys), in flight at its deadline, and only a server process paused past the wait (`SIGSTOP`, a frozen container, a VM pause) between its last check and that step runs it after the removal: the bound, like every lease of the design, assumes the server keeps running. A DELETING row of the configured store is the exception: before removing it the command waits as for a DELETED row, then finishes its deletion behind the deletion fence, the deletion its DeleteBackup asked for, which needs the store's check to have passed (a stop during the wait leaves the row DELETING and its objects untouched); with the store unhealthy or its id unknown the row is refused ("let a server finish the DeleteBackup, or run again once the store's check passes"). When the check failed on an identity mismatch that the row's own store id takes part in (the backup manual's copied `extenddb-store.json`), the refusal says so and names the way out: run the same command with `[backup.store] kind = "disabled"`, which removes the row and keeps its bytes, then, with the store configured again, let a sync (the move's own, in the manual's procedure) adopt the backup once the check passes, and DeleteBackup it again. The per-row line says when the command deleted the objects (on SQLite the command's own pass over what the stopped server left may finish it first, under the claim, and the line says so too). A DELETING row of another store keeps its bytes unless a server configured with that store finishes the deletion (the deletion its DeleteBackup asked for), and a sync of that root after its return with the bytes adopts the backup again. Because they wait for running jobs first, as `purge` does, any running job holds the command, not only the jobs of the rows it removes: a CREATING or FAILED row of a store the wait covers whose job is live, and any running restore, whichever backup it reads; the wait lasts up to twice `stale_job_after_secs` (600 s at the default), and the command refuses, exit 1, if a job is still running when it ends; a row whose heartbeat, written after the wait, is within `stale_job_after_secs` (its job perhaps still writing under the prefix), or one of a store the wait does not cover (a `--forget-backup` of another store's row), is left, as is a row a restore target is being restored from (as DeleteBackup refuses it: wait for the restore to end, or, once it failed or its owner is dead, remove the target with `extenddb backup remove-target --account
`), and the command exits 1 naming each. `--forget-store` matches the root as `--move-store` does and refuses the configured root (`purge` and `--prune-missing` are the configured store's removals); on Cassandra it then sweeps the root's status entries under the claim, each checked against its row: one whose row is gone or names another store (a deleter whose entry batch landed after a move of its row, or the earlier-status entry of a row removed above) is deleted, and one whose row is of the root under another status (a status change whose entry batch failed, or whose writer died before it) is written again under the row's status; it reports both counts, and then the store ids the catalog still records for the root (entries written after the root was listed, by a writer still configured with it). `--forget-backup` takes the ARN as the row carries it: a row under another spelling of the ARN (another region) is named and left, and an earlier release's in-catalog backup is reported as one with nothing changed (DeleteBackup removes it). While the configured root has no `extenddb-store.json` and the catalog holds rows for it (an unmounted volume, say), or, on SQLite, the default root is missing while the catalog holds rows for it (moved away, or a symbolic link to a volume that is gone; the command's header then says so), `--forget-backup` refuses every row of that root and changes nothing for it ("mount it, or point `[backup.store]` at the root that holds these backups, then run again"): a running server's check creates the file at a root the catalog has no rows for and reports the store healthy, so removing the root's last rows would have the next CreateBackup write to the empty mount point. After `--forget-backup`, when a store is configured, the store's check runs once, writing nothing to the root (no identity file is created; when the root had no `extenddb-store.json` as the command opened it, the check is not run and the command says so), and its outcome is printed: the check passing, or the problem it finds, which for a missing manifest names the next backup and the command for it; the exit stays 0 once the rows were removed. That is the way out of the unhealthy store a restored catalog is left with when its newest AVAILABLE backup was deleted after the catalog's backup: remove that row, and the next one the check names, then run `--prune-missing`; each running server finds the store healthy at its own next check. The three options conflict with each other, with `--adopt-store` and with `--prune-missing` (exit 2), and each row they change is logged under `extenddb::audit`: + +``` +=== extenddb backup sync === +Store: filesystem (unhealthy: ; the removals read no store; the check after them does) +Took the maintenance claim as "extenddb backup sync on (pid )". + +--- --forget-backup (rows removed outright; no object is deleted, but a DELETING row of the configured store has its deletion finished first): + : removed: the row (AVAILABLE) is gone, and no object was deleted + +=== extenddb backup sync --forget-backup complete: 1 row(s) removed, 0 not === + +The configured store's check passes (store id ); each server finds it so at its next reconciler cycle. +``` Every `extenddb backup` command refuses an in-memory SQLite database (`[storage.sqlite] path = ":memory:"`, or no path on a `sqlite-memory` build) before it opens the catalog, exit 1: it would open a new, empty catalog that ends with the command. A server on such a database syncs with its store at every start instead (the `path` row above). @@ -1056,18 +1078,42 @@ Increase `pool_size` in `extenddb.toml` or check for connection leaks. ### Data Recovery -extenddb stores all data in PostgreSQL. Use standard PostgreSQL backup and recovery tools: +Table backups (`CreateBackup`) hold one table's items and definition; they do not hold the catalog's accounts, IAM users, keys, policies and settings. Back the databases up with the backend's own tools as well; the [Backup and Restore manual](14-backup-and-restore.md) covers table backups, the backup store and recovery from a lost catalog. The databases each backend uses: + +| Backend | Databases | +|---------|-----------| +| PostgreSQL | The catalog database, `extenddb_catalog` unless `init --catalog-db` names another, and the data database, whose name `init` derives from the catalog's by dropping the `_catalog` suffix (`extenddb` for the default; `init --data-db` names another). A catalog name without the `_catalog` suffix needs `--data-db`, since the derived name would be the catalog's own; an `init` that failed this way leaves the catalog database behind and writes no config file, so drop that database before running `init` again with `--data-db`. The catalog records the data database's name in its `settings` table under `data_database_name` | +| SQLite | One database file (`[storage.sqlite] path`) with its `-wal` and `-shm` files beside it; stop the server before copying the three. To restore, stop the server, remove the current `-wal` and `-shm` files, and put the copied files in their place | +| MongoDB | `extenddb_catalog` and `extenddb_data`, fixed names | +| Cassandra | The catalog keyspace `_catalog` and one keyspace per account, `_account_` (`keyspace_prefix` defaults to `extenddb`) | + +On PostgreSQL the catalog and data databases are one copy: dump both with every server of the catalog stopped, or, to back up without stopping the server, take a base backup (`pg_basebackup`) of the PostgreSQL server that holds both. Each `pg_dump` takes its own snapshot of one database, so two dumps taken while a server runs disagree when a table is created or deleted between them. The catalog dump holds the encryption key and the application role's password (in the `data_database_connection_string` setting), so create both files readable by their owner alone, with `umask 077`. With the default names: ```bash -# Backup -pg_dump extenddb_catalog > catalog_backup.sql -pg_dump extenddb_catalog_data > data_backup.sql +# Backup, with the server stopped, with the pg_dump of the server's major version +umask 077 +pg_dump extenddb_catalog > catalog_backup.sql && pg_dump extenddb > data_backup.sql \ + && test -s catalog_backup.sql && test -s data_backup.sql && echo "both dumps written" +``` -# Restore -psql -f catalog_backup.sql extenddb_catalog -psql -f data_backup.sql extenddb_catalog_data +Use the `pg_dump` of the PostgreSQL server's major version (for a server in a container, the container's own: `docker exec pg_dump ...`). An older `pg_dump` refuses, "aborting because of server version mismatch", and exits 1, and the shell has already created its file, empty. Go on only when both commands exited 0 and neither file is empty: `psql -v ON_ERROR_STOP=1` restores an empty file without an error, and the server then refuses to start ("the catalog at ... is not initialized"). + +Restore into new, empty databases owned by the application role (`extenddb` unless `init --extenddb-user` named another), created by the PostgreSQL admin role, with the server stopped. Restoring over the existing databases restores nothing: the dump's statements fail on the objects already there (`psql` prints an `ERROR` line for each and exits 0), and the tables keep their current items. Rename the old databases aside first, and drop them only once the restored server has read every table, so that the old copy stays until the new one is proven. The owner matters: on PostgreSQL 15 and later the `public` schema belongs to the database's owner and the dump does not carry it, so in a database the admin role owns the server cannot create tables (`CreateTable` fails and the server logs "permission denied for schema public"). `-v ON_ERROR_STOP=1` makes `psql` stop at the first error, with exit 3: + +```bash +# Restore, as the PostgreSQL admin role, with the server stopped +psql -c 'ALTER DATABASE extenddb_catalog RENAME TO extenddb_catalog_before_restore' postgres +psql -c 'ALTER DATABASE extenddb RENAME TO extenddb_before_restore' postgres +createdb -O extenddb extenddb_catalog +createdb -O extenddb extenddb +psql -v ON_ERROR_STOP=1 -f catalog_backup.sql extenddb_catalog +psql -v ON_ERROR_STOP=1 -f data_backup.sql extenddb +# Start the server and read every table, then +psql -c 'DROP DATABASE extenddb_catalog_before_restore' -c 'DROP DATABASE extenddb_before_restore' postgres ``` +On a new PostgreSQL server (the old one lost with its databases) create the application role first: a dump holds no roles, and without it `createdb -O extenddb` fails, `role "extenddb" does not exist`. Give it the password that the catalog dump's `data_database_connection_string` row carries (`postgresql://extenddb:@:/extenddb`), the one the config file's `[storage.postgres] connection_string` names as well: `psql -c "CREATE USER extenddb WITH PASSWORD ''" postgres`. That row also records the data database's host and port, and the server connects to the data database there. On a server at another address, point the config file's `connection_string` at it and, after the restore and before the server starts, set the row: `psql -c "UPDATE settings SET value = 'postgresql://extenddb:@:/extenddb' WHERE key = 'data_database_connection_string'" extenddb_catalog`. Without that the server connects to the old address: it fails to start once nothing answers there, and while the old server still answers, the tables created then keep their items in the old server's data database while the new catalog lists them. + --- ## License diff --git a/docs/manuals/07-upgrade-manual.md b/docs/manuals/07-upgrade-manual.md index 7f244409f..035584b69 100755 --- a/docs/manuals/07-upgrade-manual.md +++ b/docs/manuals/07-upgrade-manual.md @@ -8,7 +8,7 @@ This release versions the PostgreSQL, SQLite, Cassandra and MongoDB catalogs by Migration 9, `warm_throughput`, adds a nullable `warm_throughput` column to the `tables` and `indexes` catalog tables on PostgreSQL, SQLite and Cassandra. It holds the `WarmThroughput` a client sets on a table or a global secondary index, which `DescribeTable` reports; nothing acts on the value (see [Differences from DynamoDB](../differences-from-dynamodb.md)). Tables created before the migration report the defaults for their billing mode. On MongoDB it changes nothing, since the catalog collections have no validator, and the ledger records it as not applicable. Migration 9 is breaking: to go back to a release without it, restore the backup taken before the upgrade (see [Rollback Procedure](#rollback-procedure)). -Configure `[backup.store]` before the release that stops creating in-catalog backups. From this release CreateBackup writes to the configured store on PostgreSQL, SQLite and MongoDB, and is asynchronous there as on the service: the response is `CREATING` and the backup becomes `AVAILABLE` once the upload ends, so a client that deletes a backup it just created has to poll DescribeBackup first (the previous release returned `AVAILABLE`). A store-backed backup is listed, described, deleted and restored like any other. `RestoreTableFromBackup` of a backup in the configured store runs the restore job from the store on every backend (PostgreSQL, SQLite, MongoDB and Cassandra); in-catalog backups, those of the previous release included, restore as before. An S3 store needs `spool_path` set (the restore's download area) on PostgreSQL, MongoDB and Cassandra, and that must be a directory of its own, not the database directory; a filesystem store needs no spool. Migration 8 adds the `restore_progress` table to the PostgreSQL data database; run `extenddb migrate --yes` before starting the new binary. The progress rows of a restore in flight live in the data database, so a data database restored from a dump taken before migration 8 has none of them and no `restore_progress` table: the new binary refuses to serve it until `extenddb migrate --yes` has run again, and its reconciler then finds no progress for the restore and resumes it from its first data file into the (empty) target, or `FAILED`s it as `target_not_empty` if the target already holds items, in which case `extenddb backup remove-target --account
` (the admin guide's `backup` section) removes the target and its record and the backup restores again. The same `target_not_empty` state is what a server stopped inside a restore's first data file on MongoDB, or inside its first batch of 10,000 items or 32 MiB on Cassandra, leaves on restart, since both record progress after the write; the command is the remedy there too. `backup remove-target` writes the catalog, so on SQLite the server is stopped for it, as for `purge`. The previous binary resumes no restore (a store-backed restore in flight at a rollback is lost with the catalog rows, as the rollback paragraph below says). Quiesce restores before a downgrade. `kind = "disabled"` is no longer needed to keep a deployment able to restore the backups it takes. Without a store CreateBackup writes the in-catalog format and logs a deprecation warning naming `[backup.store]` on each call; a later release's CreateBackup refuses without one. Cassandra writes the in-catalog format with or without a store. `kind = "disabled"` under `[backup.store]` is the opt-out that keeps a deployment on the in-catalog format on purpose and silences the warning. On file-backed SQLite an absent section is a filesystem store at `backups/` beside the database file, created `0700` at startup; that root must be owned by the extenddb user with mode `0700`, under directories owned by root or that user which others cannot write to. A default root that breaks the rule (a database file in a group-writable directory, say), or whose `extenddb-store.json` names another catalog (a catalog re-created by `destroy` and `init` under the old root), is logged at startup as a warning (`backup store: the default root ... failed its check`) and leaves the store unconfigured, not unhealthy, so the upgrade never stops a server that never configured backups and CreateBackup keeps working on it; the same root configured explicitly is a startup error or an unhealthy store. The warning applies only while the catalog has no backup rows for the default root (an upgrade, a fresh deployment, a catalog re-created by `destroy` and `init`): once a server has written rows for it, the default root is handled as a configured one, so its backups never vanish behind a warning (a failed identity check is an unhealthy store, reported on `/health`; a root that can no longer be opened is a startup error naming it). Instances that share a catalog must share one store root, an S3 bucket and prefix or a shared mount: a second instance pointed at a host-local directory under the same configured path starts with the store unhealthy (`/health` reports `backup_store` unhealthy; the admin guide's `[backup.store]` section says what the identity file and the manifest probe check). `extenddb destroy` leaves the store; `extenddb backup purge --confirm` is what deletes its backups, and a catalog re-created with `init` needs the old root's `extenddb-store.json` removed or a new root, since the file names the old catalog. +Configure `[backup.store]` before the release that stops creating in-catalog backups. The [Backup and Restore manual](14-backup-and-restore.md) covers choosing and securing a store, the recovery figures, the restore drill, catalog-loss recovery and what to do with in-catalog backups; this section says what the upgrade changes. From this release CreateBackup writes to the configured store on PostgreSQL, SQLite and MongoDB, and is asynchronous there as on the service: the response is `CREATING` and the backup becomes `AVAILABLE` once the upload ends, so a client that deletes a backup it just created has to poll DescribeBackup first (the previous release returned `AVAILABLE`). A store-backed backup is listed, described, deleted and restored like any other. `RestoreTableFromBackup` of a backup in the configured store runs the restore job from the store on every backend (PostgreSQL, SQLite, MongoDB and Cassandra); in-catalog backups, those of the previous release included, restore as before. An S3 store needs `spool_path` set (the restore's download area) on PostgreSQL, MongoDB and Cassandra, and that must be a directory of its own, not the database directory; a filesystem store needs no spool. Migration 8 adds the `restore_progress` table to the PostgreSQL data database; run `extenddb migrate --yes` before starting the new binary. The progress rows of a restore in flight live in the data database, so a data database restored from a dump taken before migration 8 has none of them and no `restore_progress` table: the new binary refuses to serve it until `extenddb migrate --yes` has run again, and its reconciler then finds no progress for the restore and resumes it from its first data file into the (empty) target, or `FAILED`s it as `target_not_empty` if the target already holds items, in which case `extenddb backup remove-target --account
` (the admin guide's `backup` section) removes the target and its record and the backup restores again. The same `target_not_empty` state is what a server stopped inside a restore's first data file on MongoDB, or inside its first batch of 10,000 items or 32 MiB on Cassandra, leaves on restart, since both record progress after the write; the command is the remedy there too. `backup remove-target` writes the catalog, so on SQLite the server is stopped for it, as for `purge`. The previous binary resumes no restore (a store-backed restore in flight at a rollback is lost with the catalog rows, as the rollback paragraph below says). Quiesce restores before a downgrade. `kind = "disabled"` is no longer needed to keep a deployment able to restore the backups it takes. Without a store CreateBackup writes the in-catalog format and logs a deprecation warning naming `[backup.store]` on each call; a later release's CreateBackup refuses without one. Cassandra writes the in-catalog format with or without a store. `kind = "disabled"` under `[backup.store]` is the opt-out that keeps a deployment on the in-catalog format on purpose and silences the warning. On file-backed SQLite an absent section is a filesystem store at `backups/` beside the database file, created `0700` at startup; that root must be owned by the extenddb user with mode `0700`, under directories owned by root or that user which others cannot write to. A default root that breaks the rule (a database file in a group-writable directory, say), or whose `extenddb-store.json` names another catalog (a catalog re-created by `destroy` and `init` under the old root), is logged at startup as a warning (`backup store: the default root ... failed its check`) and leaves the store unconfigured, not unhealthy, so the upgrade never stops a server that never configured backups and CreateBackup keeps working on it; the same root configured explicitly is a startup error or an unhealthy store. The warning applies only while the catalog has no backup rows for the default root (an upgrade, a fresh deployment, a catalog re-created by `destroy` and `init`): once a server has written rows for it, the default root is handled as a configured one, so its backups never vanish behind a warning (a failed identity check is an unhealthy store, reported on `/health`; a root that can no longer be opened is a startup error naming it). Instances that share a catalog must share one store root, and a catalog re-created with `init` finds the old root's `extenddb-store.json` naming the old catalog: the [Backup and Restore manual](14-backup-and-restore.md) gives both rules and the recovery with `extenddb backup sync --adopt-store`. A backup that a 0.1.x release took can restore without its global and local secondary indexes: a SQLite backup taken by 0.1.14 records only the table's key schema and attribute definitions, and its restore creates the table with its items and no index. Take a new backup of each table after upgrading to v1.0, and keep the 0.1.x backups for their items only. @@ -70,11 +70,16 @@ extenddb stop --config extenddb.toml 2. **Back up the databases** (PostgreSQL; for SQLite see [SQLite](#sqlite) below, for Cassandra see the [rollback procedure](#rollback-procedure)) ```bash -pg_dump -h -U > catalog_backup_$(date +%Y%m%d).sql -pg_dump -h -U > data_backup_$(date +%Y%m%d).sql +umask 077 +d=$(date +%Y%m%d) +pg_dump -h -U > catalog_backup_$d.sql \ + && pg_dump -h -U > data_backup_$d.sql \ + && test -s catalog_backup_$d.sql && test -s data_backup_$d.sql && echo "both dumps written" ``` -`pg_dump` refuses to dump a server of a newer major version than itself ("aborting because of server version mismatch"); use the client of the server's version. +The catalog dump holds the encryption key and the application role's password, so `umask 077` creates both files readable by their owner alone. + +`pg_dump` refuses to dump a server of a newer major version than itself ("aborting because of server version mismatch"), exits 1 and leaves its file empty; use the client of the server's version. Go on only when the command prints "both dumps written": `psql` restores an empty file without an error. 3. **Build the new version** @@ -169,17 +174,27 @@ SQLite deployments follow the same sequence, backing up the database files as [S The upgrade is not reversible in place when it applies a breaking migration, and every migration in this release is breaking (see [The Catalog Status](#the-catalog-status)). If an upgrade fails or must be undone (PostgreSQL; for SQLite see [SQLite](#sqlite) below): 1. Stop the server -2. Restore the catalog from the backup, connected as `` to the `postgres` database: +2. Restore both databases from the backups into new databases, connected as `` to the `postgres` database, with the upgraded databases renamed aside rather than dropped: ```bash -psql -h -U -d postgres -c "DROP DATABASE ;" +psql -h -U -d postgres -c "ALTER DATABASE RENAME TO _before_rollback;" +psql -h -U -d postgres -c "ALTER DATABASE RENAME TO _before_rollback;" psql -h -U -d postgres -c "CREATE DATABASE OWNER ;" -psql -h -U -d -f catalog_backup_YYYYMMDD.sql +psql -h -U -d postgres -c "CREATE DATABASE OWNER ;" +psql -h -U -d -v ON_ERROR_STOP=1 -f catalog_backup_YYYYMMDD.sql +psql -h -U -d -v ON_ERROR_STOP=1 -f data_backup_YYYYMMDD.sql ``` -To discard data written after the upgrade as well, restore `` from its dump the same way. +`-v ON_ERROR_STOP=1` makes `psql` stop at the first error, with exit 3; go on only when both restores exited 0. The catalog and data databases are one copy, so restore both; every write made after the dumps is lost. A catalog restored alone beside the newer data database is a catalog older than the tables; the Backup and Restore manual's [catalog restore](14-backup-and-restore.md#the-catalog-is-restored-from-a-database-backup) section says what that gives. 3. Rebuild the previous version and start it +4. Once the previous version has read every table, drop the renamed databases: + +```bash +psql -h -U -d postgres -c "DROP DATABASE _before_rollback;" -c "DROP DATABASE _before_rollback;" +``` + +Until then they hold the upgraded copy: if a restore fails, drop the new, partly restored database and rename the old one back. This manual has no tested rollback for Cassandra. Before migrating, with every server stopped, take a snapshot of the catalog keyspace and every account keyspace on every node with your usual Cassandra tooling; a rollback restores those snapshots, again with every server stopped, before the previous binary starts. @@ -218,6 +233,7 @@ The upgrade is not reversible in place: a 0.0.3 PostgreSQL or SQLite binary refu ```bash extenddb stop --config extenddb.toml +umask 077 pg_dump -h -U > catalog_backup_$(date +%Y%m%d).sql export EXTENDDB_PG_PASSWORD='' extenddb migrate --config extenddb.toml --pg-user # exit 3, prints the plan diff --git a/docs/manuals/10-security-model.md b/docs/manuals/10-security-model.md index 24447b981..e43e0add0 100755 --- a/docs/manuals/10-security-model.md +++ b/docs/manuals/10-security-model.md @@ -333,11 +333,12 @@ Import/export file paths are validated: ### Backup and Recovery -extenddb stores all state in PostgreSQL. Use standard PostgreSQL tools for backup and recovery: +Back the databases up with the backend's own tools (the admin guide's [Data Recovery](05-admin-guide.md#data-recovery) entry names each backend's databases), and keep table backups in a backup store whose access you restrict (the [Backup and Restore manual](14-backup-and-restore.md#securing-the-store)). On PostgreSQL the catalog and data databases are one copy: dump both with the server stopped (the admin guide's Data Recovery entry restores the dumps), or, when both databases are on one PostgreSQL server, take a base backup of that server and restore it with PostgreSQL's own procedure (the design guide allows the two databases on different PostgreSQL instances, which one base backup does not cover). With the default database names, and `umask 077` so that only the owner can read the files: ```bash +umask 077 pg_dump extenddb_catalog > catalog_backup.sql -pg_dump extenddb_catalog_data > data_backup.sql +pg_dump extenddb > data_backup.sql ``` Encryption keys are stored in the catalog database. A catalog backup includes the encryption key needed to decrypt access key secrets. diff --git a/docs/manuals/11-deployment-guide.md b/docs/manuals/11-deployment-guide.md index 355a66911..c5e0d3ad0 100755 --- a/docs/manuals/11-deployment-guide.md +++ b/docs/manuals/11-deployment-guide.md @@ -17,7 +17,7 @@ This guide covers deploying extenddb in various environments beyond local develo extenddb is a single Rust binary that connects to a storage backend. All durable state lives in the backend; extenddb keeps only short-lived credential, policy, and table-metadata caches in process. This means: - Multiple extenddb instances can share one backend catalog (with caveats — see Multi-Instance below) -- The backend's own HA, backup, and replication tools provide durability +- The backend's own HA, backup, and replication tools provide durability; with `[backup.store]` configured (file-backed SQLite has one by default), table backups go to the store on PostgreSQL, SQLite and MongoDB; without one, and always on Cassandra, they are in-catalog backups inside the database; the [Backup and Restore manual](14-backup-and-restore.md) covers both - extenddb can run anywhere the storage backend is reachable ## Deployment Models @@ -229,7 +229,8 @@ extenddb init \ - [ ] Use a dedicated PostgreSQL user for extenddb with minimal privileges - [ ] Configure PostgreSQL `max_connections` ≥ `2 * pool_size + catalog_pool_size + max_concurrent_jobs`, plus one per overlapping vector index build and one during a schema migration - [ ] Enable PostgreSQL TLS -- [ ] Set up automated backups (pg_dump, WAL archiving, or managed service snapshots) +- [ ] Set up automated backups that copy the catalog and data databases as one copy: a base backup with WAL archiving or managed service snapshots of the instance (either covers one instance, so only when both databases are on one PostgreSQL instance), or `pg_dump` of both with the server stopped, under `umask 077` (the admin guide's [Data Recovery](05-admin-guide.md#data-recovery) entry) +- [ ] Configure `[backup.store]` (S3 for production) per the [Backup and Restore manual](14-backup-and-restore.md) - [ ] Monitor PostgreSQL disk usage, connection count, and query performance ### Monitoring @@ -272,7 +273,10 @@ RestartSec=5 # large items it reads (the admin guide's [backup] max_concurrent_jobs row # gives the figures). It costs the request path a few percent (3.5 to 9.2% # on GetItem, Query and Scan on SQLite, about 2% on PostgreSQL (the scans, -# 2.1%)), so the line is left to the operator: +# 2.1%)), so the line is left to the operator. On SQLite it also lowers +# the peak of a restore of tables whose items are larger than a few +# hundred bytes, of any attribute types (the Backup and Restore manual +# gives the figures): #Environment=MALLOC_ARENA_MAX=1 [Install] @@ -291,7 +295,7 @@ Run one extenddb instance per catalog. Multiple instances behind a load balancer - Credentials, policies, and table metadata are cached in each process (see the cache TTLs in the admin guide). Nothing invalidates a cache on another instance, so a key revoked or a policy changed through one instance stays in force on the others until the TTL expires. - Every instance runs every background worker (TTL expiry, GSI backfill, stream retention, control-plane transitions). The workers are safe to run twice, but the work is duplicated. - `/health` is a liveness check that does not query the database, so a load balancer cannot use it to take an instance with a broken database connection out of rotation. It does report the backup store: `components.backup_store` is `unhealthy` and `status` is `degraded` while the configured store cannot be reached or fails its identity check, with HTTP 200 kept, so a store outage never drains a replica. -- Instances that share a catalog must share one backup store root (`[backup.store]`: one S3 bucket and prefix, or one shared mount that supports hard links, which the store's conditional create needs; some CIFS and FUSE mounts refuse `link(2)` with `EPERM`, and the server then reports the store as having failed validation, with the `link` error in the log line). A host-local directory configured at the same path on each host is not one root: once the catalog has rows for the root, the second instance finds the root's identity file missing or the newest backup's manifest absent and starts with the store unhealthy. While the catalog has no rows for the root (a fresh deployment, or one whose backups were purged), nothing tells the two directories apart: each instance creates its own identity file and store id and both start healthy, and the first backups then split the rows between two ids, which the next check reports as an identity mismatch that only `backup purge` resolves. Point every instance at one root before the first backup. Each instance on a host needs its own `spool_path`; the spool's `.lock` refuses a second instance. An existing spool directory must be owned by the server's user with mode `0700`, as the root must; a spool created `0755` stops startup naming it. +- Instances that share a catalog must share one backup store root (`[backup.store]`: one S3 bucket and prefix, or one shared mount that supports hard links, which the store's conditional create needs; some CIFS and FUSE mounts refuse `link(2)` with `EPERM`, and the server then reports the store as having failed validation, with the `link` error in the log line). A host-local directory configured at the same path on each host is not one root: once the catalog has rows for the root, the second instance finds the root's identity file missing or the newest backup's manifest absent and starts with the store unhealthy. While the catalog has no rows for the root (a fresh deployment, or one whose backups were purged), nothing tells the two directories apart: each instance creates its own identity file and store id and both start healthy, and the first backups then split the rows between two ids, which the next check reports as an identity mismatch. `backup purge`, plain `backup sync` and `--prune-missing` all refuse while the store is unhealthy, and `--forget-store` refuses the one root label both directories share. To clear it, keep one directory: remove each row that carries the other directory's store id with `extenddb backup sync --forget-backup ` (`backup list` prints each row's `store_id`; the command runs on the unhealthy store and deletes no object), copy that directory's account directories, without its `extenddb-store.json`, into the kept one as the server's user, keeping directories `0700` and files `0600` (`cp -rp` or `rsync -a`; `backup sync` exits 1 on a directory left `0755`), point every instance at the kept directory over a shared mount, and run plain `backup sync`, which adopts them; or restore the catalog from a copy taken before the split, keep the directory whose store id the restored rows carry, and copy the other directory's account directories into it the same way before `backup sync` (the backups that the other directory's instance took after the catalog's copy are otherwise unknown to it). The [Backup and Restore manual](14-backup-and-restore.md#configuration) gives both, with the case of a `DELETING` row among them. Point every instance at one root before the first backup. To change the root later without losing the backups, reconfigure every instance to the new root, copy the old root's tree there without its `extenddb-store.json` (the new root has its own once the instances restart) and run `extenddb backup sync --move-store ` ([Moving a store to a new root](14-backup-and-restore.md#moving-a-store-to-a-new-root)). Each instance on a host needs its own `spool_path`; the spool's `.lock` refuses a second instance. An existing spool directory must be owned by the server's user with mode `0700`, as the root must; a spool created `0755` stops startup naming it. On SQLite this limit is enforced: `extenddb serve` holds an exclusive lock on `.lock` and a second server on the same file refuses to start. On PostgreSQL, MongoDB, and Cassandra it is not enforced. On PostgreSQL, if you start a second instance anyway, size `pool_size × instance_count + 3 × instance_count ≤ PostgreSQL max_connections`. diff --git a/docs/manuals/12-extending-extenddb-storage.md b/docs/manuals/12-extending-extenddb-storage.md index 6c54f3344..7905409fc 100755 --- a/docs/manuals/12-extending-extenddb-storage.md +++ b/docs/manuals/12-extending-extenddb-storage.md @@ -288,8 +288,8 @@ The `storage-postgres` crate provides the default implementation: ### Database Architecture extenddb uses a dual-database architecture: -- **Catalog database** (`extenddb`) — table metadata, IAM data, settings, metrics, stream records, tags, idempotency tokens, login attempts -- **Data database** (`extenddb_data`) — DynamoDB items and index rows +- **Catalog database** (`extenddb_catalog` by default) — table metadata, IAM data, settings, metrics, stream records, tags, idempotency tokens, login attempts +- **Data database** (`extenddb` by default, the catalog's name without `_catalog`) — DynamoDB items and index rows ### Migrations diff --git a/docs/manuals/14-backup-and-restore.md b/docs/manuals/14-backup-and-restore.md new file mode 100644 index 000000000..2a2924440 --- /dev/null +++ b/docs/manuals/14-backup-and-restore.md @@ -0,0 +1,927 @@ +# Backup and Restore + +> See [NOTICE](../NOTICE.md) for important disclaimers. + +A filesystem backup store keeps every backup in the same failure domain as the host that holds it, so S3 is +the production recommendation: a lost host or disk loses a filesystem store's backups with it unless +something copies the root off the host, while an S3 bucket (or an S3-compatible server on other hardware) +survives the loss of the ExtendDB host and of its database. + +This manual is for operators who plan, run and test backups. The admin guide is the reference for every +configuration key and command named here; the +[differences document](../differences-from-dynamodb.md#backup-and-restore) records how each backup operation +compares with Amazon DynamoDB's. + +## Strategy + +What a table backup is. `CreateBackup` takes one table's items and definition at one instant and writes +them to the configured backup store, the `[backup.store]` section: a directory (`kind = "filesystem"`) or +an S3 bucket and prefix (`kind = "s3"`). The request answers `CREATING` and a job of the server uploads the +backup; it becomes `AVAILABLE` when the upload ends. The backup is a consistent snapshot of the table on +PostgreSQL, SQLite and MongoDB. The catalog keeps a row per backup as the index that `ListBackups` and +`DescribeBackup` read, and the store holds the bytes, so a lost catalog loses the index, not the backups +([Recovering from catalog loss](#recovering-from-catalog-loss)). + +Cassandra has no table snapshot. `CreateBackup` there writes the in-catalog format (inside the Cassandra +cluster, in the catalog keyspace and the account's keyspace) whatever the store, and that copy pages the live +table, so it is not point-in-time. A configured store serves restores into Cassandra of backups that another +backend wrote to the store. For off-host copies of a Cassandra deployment use the cluster's own snapshot +tooling. + +What a table backup is not: + +- It does not hold the catalog: accounts, IAM users, groups, roles, policies, access keys, runtime settings + and the encryption key live in the catalog database, which no table backup contains. Back the catalog up + with the database's own tools (the admin guide's [Data Recovery](05-admin-guide.md#data-recovery) entry + names the databases and gives the PostgreSQL commands), and keep that copy off the host as well. On + PostgreSQL the catalog and data databases are one copy: dump both with the server stopped, or take a base + backup of the PostgreSQL server that holds both. A catalog backup holds the encryption key that decrypts the + access key secrets, and on PostgreSQL the application role's password: protect it as a secret, and create + the dump files with `umask 077`. +- It is not scheduled. ExtendDB has no backup scheduler: run `CreateBackup` from cron or your scheduler. As a + crontab line, where every `%` must be escaped because cron turns an unescaped `%` into a newline: + + ``` + 0 * * * * AWS_CA_BUNDLE=/path/to/cert.pem AWS_DEFAULT_REGION=us-east-1 aws dynamodb create-backup --table-name orders --backup-name orders-$(date +\%Y\%m\%d\%H\%M) --endpoint-url https://:18443 + ``` + + or put the command, with its `%` unescaped, in a script that cron runs. `AWS_CA_BUNDLE` (or the CLI's + `--ca-bundle`) names the self-signed certificate `init` generates, `~/.extenddb/tls/cert.pem` of the user + who ran it; without it the CLI fails certificate verification. Cron's environment names no region; + `AWS_DEFAULT_REGION` gives the CLI one, which it otherwise needs from the user's `~/.aws/config`. +- It has no retention. A backup stays until `DeleteBackup` removes it. Your scheduler deletes the backups it + no longer wants. +- It is not point-in-time recovery, which ExtendDB does not provide (below). + +A plan that covers host loss, database loss and operator error therefore has three parts: table backups on a +schedule to an S3 store in another failure domain, database-level backups of the catalog, and a restore drill +run on a schedule ([Restore drill](#restore-drill)). + +## RPO and RTO + +### Recovery point + +The recovery point objective equals the interval between two backups of a table. There is no point-in-time +recovery: `RestoreTableToPointInTime` answers `PointInTimeRecoveryUnavailableException`, and +`UpdateContinuousBackups` with `PointInTimeRecoveryEnabled` true answers +`ContinuousBackupsUnavailableException`. A restore brings the table back as it was at the backup's +`BackupCreationDateTime`, the instant the request opened the table's snapshot; every write acknowledged after +that instant is lost. + +Per store kind: + +| Store | Loss it survives | Recovery point | +|---|---|---| +| S3, a bucket outside the ExtendDB host and the database host | the ExtendDB host, the database, the catalog | the backup interval | +| filesystem, a root on the ExtendDB host | the database and the catalog, while the host's disk survives | the backup interval for database loss; for host loss, the interval of whatever copies the root off the host | +| filesystem, a root on a shared mount | as the mount's own storage | as the mount's own storage | +| in-catalog, inside the database (Cassandra always; PostgreSQL, MongoDB and in-memory SQLite without `[backup.store]`; `kind = "disabled"`) | the loss of a table, not of the database or its host | the backup interval while the database survives; none once it is lost, which for in-memory SQLite is at every restart | + +### Recovery time + +The recovery time is the restore's duration plus the time to bring a server back, and, after catalog loss, +the time to rebuild the catalog and run `extenddb backup sync`. A restore answers `CREATING` at once and the +table accepts reads and writes when it reports `ACTIVE`; every index, vector indexes included, is rebuilt from +the items during the restore, so the table is complete when it is `ACTIVE`. + +Measured restore durations, from the request to `ACTIVE` as the client saw it. Host: 16 cores (Xeon +Platinum 8124M), 30 GiB, Linux 5.10; PostgreSQL 16 (with pgvector for the vector rows), MongoDB 7.0 as a +single-node replica set and Cassandra 4.1 as a single node, each in a container on loopback; SQLite as the +binary bundles it; release builds; a filesystem store on local disk; items of a 200-byte string under a +composite key with no secondary index unless the row says otherwise. Each figure is one run (a range is +several) on a host that ran other work, so the durations are upper bounds. The figures come from the +measurement records under `docs/design/` (`15-backup-restore-bench-*.md`), each on the release build it +names. Those builds predate later changes to the backup and restore code; the PR 11 record's 100k cells, +measured again on a release build after them, reproduce in kind (peaks within 8.4%, durations up to 32% longer +at a host load of 5.3 to 11.0 against 1.3 to 8.5 for the earlier cells; the record's "Reproduced on the +rebased tree"): + +| Table | PostgreSQL | SQLite | MongoDB | Cassandra | +|---|---|---|---|---| +| 100k items | 1.1 s | 2.9 s | 3.9 to 8.9 s (unordered) | 6.1 s | +| 1M items, backup in key order | 9.5 s | 26.8 to 56.3 s | | 49.0 s | +| 1M items, unordered backup | 17.5 s | 66 to 90 s | 33.8 s (up to 5 s more in another run) | | +| 100k items, two GSIs | 6.2 to 9.9 s | 10.2 to 13.9 s | | | +| 300k items, two GSIs | 19.7 to 21.4 s | 51.7 to 51.9 s | | | +| 100k items with a 128-dimension vector and a vector index | 25 s | 22 s (28 s under `MALLOC_ARENA_MAX=1`) | | | +| 1M items with a 128-dimension vector and a vector index | 246 s | 214 s (283 s under `MALLOC_ARENA_MAX=1`) | | | + +A server writes its own backups in key order for hash-only and single-range-key tables on PostgreSQL and +SQLite; MongoDB's backups and those of multi-part-key tables are unordered. On PostgreSQL the order also needs +the table's text key columns to have collation `C` or `POSIX`: tables created by a server of version 0.1.13 or +later declare them `TEXT COLLATE "C"`, and a table created by an older server keeps the database collation on +them (the technical-debt register's C-11 entry), so the server streams it unordered and logs a warning, "table +snapshot: key columns do not have collation "C"; streaming the table unordered (key_ordered = false) rather +than sorting it", naming the columns. Such a backup restores as the "unordered backup" row above: 17.5 s +against 9.5 s for 1M items on PostgreSQL. On MongoDB the table becomes +`ACTIVE` at the next cycle of the backend's backfill worker after the restore's load ends, and the worker +runs every 5 s, so a MongoDB restore takes the load's time plus up to 5 s: 3.9 to 8.9 s at 100k items over +five runs, and at 1M items 33.8 s in one run, which another run may exceed by up to one cycle. The Cassandra rows restore a +SQLite backup copied into the Cassandra deployment's store, since Cassandra's own backups are in-catalog. The +vector rows' data files carry about 2.5 KB of line text per item, 9.5 times the plain rows' 265 bytes (the +plain rows' pad is one repeated character, so their files compress far better: 90 to 176 times smaller than +the vector rows' compressed files, 176 for the 1M key-ordered backup); their duration is not the index's cost +alone. In the runs paired with an indexed restore, the same 100k items without the vector index restored in +18.0 to 22.4 s on SQLite and 15.6 to 18.1 s on PostgreSQL. The record has three paired sets on SQLite. Two, +on a lightly loaded host (1-minute load 2.7 to 5.5), put the index at 18 to 21% of the 100k restore's +duration (18.0 to 18.3 s without it, 21.9 to 23.3 s with it). The third ran at load 5.4 to 13.1 and restored +the items in 20.5 to 22.4 s without the index and in 22.3 and 36.6 s with it; its two indexed runs differ by +14 s under that load, so it gives no share and is left out of the figure. On PostgreSQL the index is 33 to +47% of the duration (five pairs). The runs of each set or pair are from one measurement harness (the +measurement record's "Later runs on the same host" and "Runs on release builds of f78e9bc1"). + +Measured backup durations, `CreateBackup` to `AVAILABLE`, on the same host class: 1M items with two GSIs +36.1 s on PostgreSQL (the host loaded by two other 1M-item loads) and 12.45 s on SQLite; 500k items with two +GSIs 8.81 s on MongoDB; 1M items without a secondary index 13.5 s on MongoDB; 1M items with a 128-dimension +vector 243 s on PostgreSQL and 210 s on SQLite (222 s under `MALLOC_ARENA_MAX=1`). + +Not measured: a database on another host, an S3 store's upload and download, the spool's disk, and the restore +of a Cassandra deployment's own in-catalog backups beyond 60,000 items (11.2 s for 60,000 on an earlier build, +the differences document's [Backup and Restore](../differences-from-dynamodb.md#backup-and-restore) section). +Time a restore drill on your own hardware and store; that figure, not this table, is your recovery time. + +### Memory while a job runs + +Every backup and restore job is bounded per server by `[backup] max_concurrent_jobs` (default 4); a request +over it waits up to 30 s for a slot and is then refused with `LimitExceededException`. Size the host for the +jobs it runs at once. The admin guide's `max_concurrent_jobs` row has the figures for tables of large items. +For the tables above and two more shapes, the process's peak over its idle figure while one job ran: + +| Job | PostgreSQL | SQLite | MongoDB | Cassandra | +|---|---|---|---|---| +| restore, 1M items, key order | 16 MiB | 87 to 98 MiB | | 33 MiB | +| restore, 1M items, unordered | 16 MiB | 144 to 164 MiB | 52 MiB | | +| restore, 1M items with a 128-number vector and a vector index | 111 MiB | 501 to 552 MiB; 204 to 208 MiB under `MALLOC_ARENA_MAX=1` | | | +| backup, 1M items with a 128-number vector and a vector index | 104 MiB | 138 to 141 MiB; 58 MiB under `MALLOC_ARENA_MAX=1` (one run) | | | +| restore, 100k items with a 128-number list and no vector index | 60 to 71 MiB | 397 to 424 MiB; 184 to 186 MiB under `MALLOC_ARENA_MAX=1` | | | +| restore, 100k items with a 2,290-character string and no index | | 253 MiB; 130 MiB under `MALLOC_ARENA_MAX=1` | | | +| restore, 20,000 items with a 1,536-number vector and a vector index | | 669 MiB; 301 to 316 MiB under `MALLOC_ARENA_MAX=1` | | | +| backup, 20,000 items with a 1,536-number vector and a vector index | | 521 MiB | | | + +On SQLite a restore holds its write batch, up to 10,000 items or 32 MiB of item text, and the decode chunks in +front of it, so for items of more than a few hundred bytes its memory grows with the item's size up to that +bound, whatever the attribute types, and the vector index is not what it belongs to. 100k items of about +2.5 KB of strings with no index held 253 MiB over idle, and 130 MiB under `MALLOC_ARENA_MAX=1` (two runs each), +against 38 MiB for 100k items of 200 bytes. Lists of numbers decode larger at the same text size: 100k items +of 128 numbers held 345 to 424 MiB over the measured runs with or without a vector index, and 184 to 195 MiB +under one arena; the figure also grows with the numbers per item (the 1,536-number row). On PostgreSQL the +vector index adds 27 to 53 MiB at 100k items (five pairs of runs with and without it). Under glibc's default allocator the SQLite vector restore's +peak is higher at 1M items than at 100k (552 against 354 MiB), and the 1M run levelled off after about 150 s; +no other size was measured. Under `MALLOC_ARENA_MAX=1` the same restores held 195 MiB at 100k and 204 MiB at +1M, at a cost in duration: 32% for one 1M restore (283 s against 214 s), and for two 100k restores running at +once 57.5 s against 24.9 s under the default arenas (279 against 697 MiB over idle for the pair). The +deployment guide's service unit carries `MALLOC_ARENA_MAX=1` commented out, because one allocator arena costs +the request path a few percent (3.5 to 9.2% on GetItem, Query and Scan on SQLite). Set it on a SQLite host +that restores tables of items of more than a few hundred bytes, or backs up tables of large items, and size +such a host per running restore from the points measured under it: 130 MiB for 100k items of 2.5 KB strings, +184 to 208 MiB for items of 128 numbers (100k and 1M), 301 to 316 MiB for 20,000 items of 1,536 numbers, and +more for larger items (the admin guide's `max_concurrent_jobs` row has the large-item figures). Without it, +size for the default-allocator figures, which these measurements do not bound. The peaks varied by up to about +20% between runs of one cell, idle or loaded (345 to 409 MiB over nine runs of the SQLite 100k vector restore). + +## Configuration + +The `[backup]` section's keys (the job bound, the heartbeat and reconciler intervals, `spool_path`) and the +`[backup.store]` section are documented in the admin guide's [`[backup]`](05-admin-guide.md#backup) and +[`[backup.store]`](05-admin-guide.md#backupstore) sections. The store's keys, defaults and refusals are the +backup store crate's, `crates/backup-store/src/config.rs`. + +A filesystem store: + +```toml +[backup.store] +kind = "filesystem" + +[backup.store.filesystem] +path = "/var/lib/extenddb/backups" +``` + +The configured root must exist: the server does not create it and refuses to start, "backup root is +not usable: No such file or directory (os error 2)". Create it owned by the server's user with mode `0700`, for example +`install -d -m 0700 -o /var/lib/extenddb/backups` ([A filesystem root](#a-filesystem-root) has +the rules for the directories above it). + +An S3 store; a restore from S3 downloads to `spool_path`, which needs disk for the compressed size of the +backups being restored at once (on file-backed SQLite it defaults to `backup-spool/` beside the database file): + +```toml +[backup] +spool_path = "/var/lib/extenddb/backup-spool" + +[backup.store] +kind = "s3" + +[backup.store.s3] +bucket = "my-extenddb-backups" +prefix = "extenddb/" +region = "us-east-1" +``` + +The spool is created `0700` when it is absent. An existing one must be owned by the server's user with mode +`0700`, under directories that follow the root's rules ([A filesystem root](#a-filesystem-root)), or the +server refuses to start, naming it (a spool left `0755` by `mkdir -p` under a `022` umask is refused, +"backup.spool_path ... is not private"). + +Credentials for S3 come from the AWS SDK's default chain (environment, shared config, instance or container +role) and are never in the file. A server with `kind = "s3"` also needs root certificates the SDK can parse +(a debug build with an `http://` endpoint too): the system store (the `ca-certificates` package, which the +deployment guide's image installs) or a bundle named by `SSL_CERT_FILE`. Without them every `https://` +request fails, and a debug build stops at startup, "TrustStore configured to enable native roots but no valid +root certificates parsed!". An S3-compatible server takes `endpoint`, and an `http://` endpoint needs +`force_path_style = true` +and a loopback host unless `allow_insecure_endpoint = true`. The S3-compatible material (the request checksums +some servers reject, TLS trust, conditional writes) is in +[`crates/backup-store/README.md`](../../crates/backup-store/README.md). + +Defaults: on file-backed SQLite an absent section is a filesystem store at `backups/` beside the database +file. On PostgreSQL, MongoDB, Cassandra and an in-memory SQLite database there is no default, and without a +store `CreateBackup` writes the in-catalog format with a deprecation warning ([Legacy +backups](#legacy-backups)). `kind = "disabled"` keeps a deployment on the in-catalog format on purpose. + +Instances that share a catalog must share one store root: one S3 bucket and prefix, or one shared mount. A +host-local directory configured at one path on each host is not one root: once the catalog has backups there, +a second instance pointed at its own directory under that path starts with the store unhealthy. Before the +first backup nothing tells the directories apart: each instance creates its own identity file and store id, +the first backups split the rows between the two ids, and the next check reports an identity mismatch +(`/health` gives `"reason":"identity mismatch"`). `backup purge --confirm` refuses an unhealthy store ("fix +that before purging its backups"), plain `backup sync` and `--prune-missing` refuse it too ("sync changes no +row while the store is unhealthy"), `--adopt-store` takes only a root another catalog owns, and +`--forget-store` does not apply: both directories have the one root label, the configured root, which it +refuses. Two remedies exist. Keep one directory as the root: on a host whose instance uses the kept directory, +remove each row that carries the other directory's store id with `extenddb backup sync --forget-backup ` +(`backup list` runs on an unhealthy store and prints every row of the root with its `store_id`; the option +runs on the unhealthy store, can be given once per ARN, deletes no object, and after the removals runs the +store's check once, printing "The configured store's check passes (store id )" when no row of the other +id is left). A `DELETING` row among them is refused, naming its store id as one the identity check refuses; +remove it with the same command run with `[backup.store] kind = "disabled"`, as step 2 of [Moving a store to +a new root](#moving-a-store-to-a-new-root) does, and `DeleteBackup` it again once its backup is adopted. Then +copy the other directory's account directories, not its `extenddb-store.json`, into the kept directory, under +the [Restore drill](#restore-drill)'s copy rules: as the server's user, keeping directories `0700` and files +`0600` (`cp -rp` or `rsync -a`; a copy left `0755` by `tar`, by `rsync` without permissions or by `aws s3 cp +--recursive` under a `022` umask makes `backup sync` exit 1 naming the directory). Point every instance at +the kept directory over a shared mount, and run plain `backup sync`, which adopts the copied backups (the +deployment guide's [Multi-Instance +Considerations](11-deployment-guide.md#multi-instance-considerations)). Or restore the catalog from a copy +taken before the split ([The catalog is restored from a database +backup](#the-catalog-is-restored-from-a-database-backup)): keep the directory whose store id the restored +rows carry (rows of the root under the other directory's id make the next check report the mismatch again; +with no rows for the root, either directory), copy the other directory's account directories into it under +the same copy rules before `backup sync` (the backups that the other directory's instance took after the +catalog's copy are in neither the restored catalog nor the kept directory, and the sync adopts only what the +kept directory holds), and point every instance at the kept directory before the next backup. With an +`extenddb-store.json` copied from one directory to the other nothing detects the split, and a restart can +delete the only copy of a backup (the admin guide's [`[backup.store]`](05-admin-guide.md#backupstore) +section). Point every instance at the shared root before the first backup, and never copy +`extenddb-store.json` between roots. + +`extenddb destroy` removes the databases and leaves the store: the store is the copy that survives it. +`extenddb backup purge --confirm` is what deletes a store's backups. A catalog re-created with `init` finds +the old root's `extenddb-store.json` naming the old catalog; `extenddb backup sync --adopt-store` makes the +new catalog its owner, and until then the store is unhealthy, or, on file-backed SQLite with the default root, +unconfigured, with `CreateBackup` writing in-catalog backups ([The catalog is lost and +re-created](#the-catalog-is-lost-and-re-created)). Or configure a new root. `destroy` leaves the config file +too, and `init` refuses a config file that exists; `init --overwrite` regenerates the whole file from defaults, +keeping only the storage connection settings (values edited in the default sections revert too), so copy the +file aside first and put the copy back whole, apart from the storage settings, before the server starts. A +root that has to change while the catalog stays (a new volume, a new bucket) keeps its backups when its tree is +copied under the new root and `extenddb backup sync --move-store ` re-points the rows ([Moving a +store to a new root](#moving-a-store-to-a-new-root)). + +`/health` reports the store as the `backup_store` component, for example +`{"components":{"backup_store":{"kind":"filesystem","status":"healthy"}},"status":"healthy"}` (on MongoDB +`status` comes first). An unhealthy store (unreachable, or failing its identity check) is `status` `degraded` +with HTTP 200 kept, so a store outage never takes the data plane out of rotation; `CreateBackup`, and the +restore and delete of a backup in that store, are refused with a `ValidationException` until the next check +passes. + +## Securing the store + +### Encryption and integrity + +Backup contents are DynamoDB JSON in gzip, in plaintext. Encryption at rest is the store's job: SSE-S3 or +SSE-KMS on the bucket (SSE-KMS needs `kms:GenerateDataKey` and `kms:Decrypt` on the key for the server's +role), or an encrypted filesystem for a filesystem root. Bucket versioning and Object Lock add protection +against deletion and ransomware; ExtendDB depends on neither. A restore verifies every data file's checksums +and the manifest's before it writes any item, so a corrupted file fails the restore, naming the file, and +leaves no table behind. + +### A filesystem root + +The root must be owned by the server's user with mode `0700`, and every directory above it must be owned by +root or by that user and not writable by group or others unless it carries the sticky bit. The server +refuses anything else at startup, naming the path, for example: + +``` +[backup.store] (filesystem path /srv/backups/extenddb) cannot be opened: permission denied: /srv/backups (an +ancestor of the backup root) has mode 0775 and is writable by group or others without the sticky bit; another +user could replace the backup root. chmod it to remove group and world write, or move the backup root +``` + +So a root under a directory a storage administrator owns, under an NFS mount owned by `nobody`, or under a +group-writable parent without the sticky bit (a `root:staff 0775` `/srv`) is refused; put the root under a +directory owned by root or by the server's user. A root of mode `0755` is refused with "chmod it to 0700". +The root itself must exist before the server starts; the directories the store creates under it are `0700` +and its files `0600`. + +A copy tool that takes the root off the host (rsync, restic, borg) cannot read the tree as a member of a +group: run it as the server's user or as root, and have it copy the root as a whole while no backup job runs, +or copy whole backup prefixes whose `extenddb-manifest.json` is present (a prefix without that file is a +backup still being written). + +The owner-only checks see the tree as the kernel reports it. They cannot observe a bind mount of the root or +of a directory under it, nor a filesystem without Unix permission semantics: a root on such a filesystem +mounted with a mask that makes it report `0700` passes the check while enforcing nothing. Both are the +operator's responsibility. The root also needs hard links, which the store's create-if-absent uses; some CIFS +and FUSE mounts refuse `link(2)`, and the server then reports the store as having failed validation. + +### An S3 bucket + +The least-privilege IAM policy for the server's role, its actions and the resources each applies to, is in +[`crates/backup-store/README.md`](../../crates/backup-store/README.md) (the IAM item of its Behavior section). +The store's check probes each grant the policy lists and names a missing one. On an SSE-KMS bucket it does not +probe `kms:Decrypt`, which an upload's parts (`UploadPart`) and a restore's download (`GetObject`) use, so a +role without it passes the check and fails at the first job that needs it. Run the restore drill after any +change to the key policy. + +Lifecycle rules for the bucket: + +- `AbortIncompleteMultipartUpload` (one day): a process killed during an upload leaves a multipart upload + that only a later deletion of the same prefix would sweep; the rule removes the rest. +- On a versioned bucket (an Object Lock bucket always is), deletions add delete markers and keep the + noncurrent versions, which are billed: add `NoncurrentVersionExpiration` and `Expiration` with + `ExpiredObjectDeleteMarker: true`, or use an unversioned bucket. + +Objects are keyed `/
//`, so each account's backups sit under a +prefix of their own. Write access to an account's prefix is backup-administrator authority for that account: +whoever can write there can delete the account's backups, or replace one with a backup of other items whose +checksums and manifest agree, which a restore then accepts. Grant the server's role the whole prefix, and +grant any other principal (a tenant's tooling, a copy job) at most its own account's prefix, +`arn:aws:s3::://*`, with IAM prefix policies. + +The AWS SDK's `trace` logs carry credentials: at `trace`, `aws_sigv4` logs the canonical request with the +session token and `aws_smithy_runtime` the `authorization` header. Do not enable those targets at `trace` in +production. + +## What a backup costs the database + +A backup holds its snapshot open from the request until its last data file is written, and each backend pays +for that while it runs. The figures below were measured for this manual; the measurement record +`docs/design/15-backup-restore-bench-pr11.md` ("Figures measured for the manual") has their method, build, +load and repeat runs. + +- PostgreSQL. The snapshot is a read-only `REPEATABLE READ` transaction on the data database. While it is + open, `VACUUM` cannot remove any row version that became dead after it started, in any table of the data + database: every update and delete during the backup leaves its old row version in place until the backup + ends. Measured on the host above: during a 25 s backup (100k items with a 128-dimension vector), 12,156 + `PutItem` overwrites of 1,000 200-byte items in another table grew that table from 0.4 to 4.4 MiB, about + 345 bytes per write, and a `VACUUM` run 10 s into the backup reported its 4,948 dead row versions as "dead + but not yet removable"; the same writer for the same time with no backup left the table at 0.5 MiB. The + first `VACUUM` after the backup removed every dead version, and the table keeps the space for later writes. + Size the data database's disk for the update and delete volume of the longest backup. +- SQLite. The snapshot is a read transaction, which stops the WAL from being checkpointed past it, so the + `-wal` file grows by the pages written during the backup, and how many pages a write changes depends on the + table's indexes; a put that leaves the item's bytes unchanged writes almost no page. Measured on the host + above, beside a 100k-item backup (items with a 128-dimension vector), with `PutItem` overwrites of 200-byte + items in another table, each changing the item: + - a table with no secondary index: 15,472 writes during a 22.8 s backup grew the `-wal` file from 4.0 to + 61.1 MiB, about 3.9 KB per write; + - a table with two GSIs (each overwrite changing both index keys, `ALL` projection): 13,996 writes during + a 25.6 s backup grew it from 4.1 to 1,294 MiB, about 82 to 97 KB per write over four runs, some 20 + to 24 pages per write. A GSI is asynchronous at the default `index_propagation_delay_ms`: the put queues each + index write in `gsi_pending` inside the put's transaction, and later transactions apply and remove it, + each appending every page it touched. + + The same writers for the same time with no backup left the file at 4.0 and 4.1 MiB. The file keeps its size + after the backup, and after the server stops; later writes reuse it. Size the disk for the pages each write + changes, about one 4 KiB page for a table with no index and about 24 for a table with two GSIs, times the + writes during the longest backup. +- MongoDB. The server keeps snapshot history for `minSnapshotHistoryWindowInSeconds` (default 300). A backup + whose read stays open longer may fail with `SnapshotTooOld`; the backup is then `FAILED` and must be taken + again. A table that takes longer than the window to back up needs the parameter raised on the replica set + (`setParameter`), which costs the server memory for the longer history. On the host above, 1M items of 200 + bytes backed up in 13.5 s. + +On PostgreSQL each running job also holds a dedicated connection outside the pool, whose session advisory +lock is the job's claim; startup raises `pool_size` and `catalog_pool_size` to at least +`2 * max_concurrent_jobs + 10`, and `max_connections` must leave room for it (the deployment guide's +connection pool section). The claim's session sets TCP keepalives (30 s idle, then three probes 10 s apart), +so after the loss of the owner's host the database ends the session about a minute later. Another instance's +reconciler takes the job over only once that session has ended and the job's heartbeat is older than +`[backup] stale_job_after_secs` (default 300 s). An owner whose host still answers but whose process is +stopped (a paused VM, `SIGSTOP`) keeps its session, and with it the job, until it resumes. To end the +session, find it by its lock key, `backup:` for a backup (the last component of the backup ARN) +and `/
` for a restore into that table, and terminate it: + +```sql +-- in the catalog database +SELECT l.pid FROM pg_locks l + WHERE l.locktype = 'advisory' AND l.objsubid = 1 AND l.granted + AND (l.classid::bigint << 32 | l.objid::bigint) = hashtextextended('backup:', 4539474) + AND l.database = (SELECT oid FROM pg_database WHERE datname = current_database()); +SELECT pg_terminate_backend(); +``` + +The job keeps writing until its next heartbeat (within one heartbeat period, 15 s by default), then stops +with no further store or catalog +access, and once its heartbeat is older than `stale_job_after_secs` the reconciler resolves it: a backup with +no complete manifest becomes `FAILED` (`DescribeBackup` answers `BackupNotFoundException`) and its partial +objects are deleted; a store-backed restore is resumed from its recorded progress (the differences document's +restore-interruption row). Do not terminate the session whose `application_name` is `extenddb serve`: that is +the server's own registration. The server then refuses requests with `ServiceUnavailable` and stops its +background work while it registers again, and every store-backed backup it is running fails (reason +`shutdown`) and must be taken again. + +A transaction-mode pooler (PgBouncer with `pool_mode = transaction`) is unsupported for the +claim: the lock lands on a pooled server connection and is not released with the job. Reach the catalog +database directly or through a session-mode pooler. + +## Restore drill + +Run a drill on a schedule and after every change to the store, the hosts or the release. It proves that the +backups restore, and its duration is your recovery time. + +Every `aws` command below is for the ExtendDB server: give each one `--endpoint-url https://:18443`, with +`AWS_CA_BUNDLE` naming the server's certificate, as the crontab line above does. Without the flag the CLI sends +the request to Amazon DynamoDB, with whatever credentials the host's profile holds. + +1. Pick a backup: `aws dynamodb list-backups --table-name orders`, or take one with `create-backup` and poll + `aws dynamodb describe-backup --backup-arn ` until `BackupStatus` is `AVAILABLE`. On the server, + `extenddb backup list --config extenddb.toml` prints the store-backed backups with their size, item count + and store; it runs beside a server. +2. Restore it under a new name: + `aws dynamodb restore-table-from-backup --backup-arn --target-table-name orders-drill`. The answer is + `CREATING` at once. +3. Poll `aws dynamodb describe-table --table-name orders-drill` until `TableStatus` is `ACTIVE` and every + index reports `ACTIVE`. Until then the table answers reads with `ResourceNotFoundException`, and + `DeleteTable` on it is refused. Note the elapsed time. +4. Compare: `aws dynamodb scan --table-name orders-drill --select COUNT` (follow `LastEvaluatedKey`) against + the backup's `SourceTableDetails.ItemCount` from `describe-backup`, and read a few known keys with `get-item`. + `DescribeTable` reports `ItemCount` 0 until the statistics pass next runs, as the service does, so count + with a scan. +5. Delete the drill table: `aws dynamodb delete-table --table-name orders-drill --endpoint-url https://:18443`. + +A drill on the same server proves the backup. To prove the off-host copy, run it on a separate deployment: +copy the backup's prefix (`/
//`, every object under it) under the drill +deployment's own store root, with the same relative path, as the drill server's user, keeping directories +`0700` and files `0600`. Create the `/` and `
/` directories with `install -d -m 0700` +first, since `mkdir -p` under a `022` umask makes them `0755`, or copy the account directory whole; copy with +`cp -rp` or `rsync -a`, or run `chmod -R u=rwX,go=` on the account directory (`/`) +afterwards. Then run `extenddb backup sync` there (below), and restore. The drill deployment needs the account +the backup's ARN names. A copy that lands readable by other users, as `aws s3 cp --recursive`, `tar` or +`rsync` without permissions under a `022` umask leave it, is refused: `backup sync` exits 1 naming the +directory ("permission denied: has mode 0755 and is accessible to other users; chmod it to 0700"), +a restore of such a backup is refused (`InternalServerError`; the server log names the directory), and when +the copy is the newest backup the store's next check marks the whole store unhealthy. An account or table +directory left `0755` stops the whole sync the same way, with the backups listed before it adopted. Run +`chmod -R u=rwX,go=` on the account directory and `extenddb backup sync` again. + +A restore that fails (a checksum mismatch, found before any item is written, or an item the target refuses +during the load) removes its target, +and the server logs the reason with the backup ARN. A store-backed restore stopped by the server stopping is +resumed by a server's reconciler from its recorded progress; an in-catalog one is failed and its target +removed. If a resume finds a target holding more +items than the backup and no recorded progress, it fails as `target_not_empty` and leaves the target +`CREATING`; `extenddb backup remove-target --account
` removes it (the admin guide's +`backup` section). + +## Recovering from catalog loss + +The store holds every store-backed backup; the catalog holds the rows that index them, the accounts and IAM. +After a catalog is lost, bring the catalog back, then give it a row for each backup with `extenddb backup +sync`. + +### The catalog is restored from a database backup + +Restore the catalog from its database backup (the admin guide's [Data +Recovery](05-admin-guide.md#data-recovery) entry). On PostgreSQL the catalog and data databases are one copy: +restore both, either from dumps taken together with the server stopped, into new, empty databases owned by the +application role as the entry gives it (restoring over the existing databases restores nothing), or from one +base backup of the PostgreSQL server that held both, with PostgreSQL's own procedure. On a new PostgreSQL +server create the application role before the dumps' restore and, when the server's address changed, set the +data database's address in the restored catalog, as the entry's last paragraph says. On SQLite the database +file holds the catalog and the tables; restore it as the entry's SQLite row says. Either way every table returns to the copy's moment, and a table created after it is gone. The restored catalog has the same catalog id, so it still +owns the store root, and its accounts and IAM are back. Then run, with the servers stopped on SQLite and +beside them on the other backends: + +```bash +extenddb backup sync --config extenddb.toml +``` + +Sync lists the root and inserts an `AVAILABLE` row for each complete backup the catalog has no row for (the +backups taken after the catalog's backup). It reports the prefixes it skipped (an incomplete backup, one the +catalog already has) and the `AVAILABLE` rows whose manifest is no longer in the store (backups deleted after +the catalog's backup); `extenddb backup sync --prune-missing` removes those rows and deletes nothing in the +store. When the newest of those rows is the catalog's newest `AVAILABLE` backup, the store is unhealthy (each +check looks for that backup's manifest, and `/health` reports `"reason":"manifest probe failed"`): both +commands refuse ("sync changes no row while the store is unhealthy"), as do `CreateBackup` and the restore and +delete of every backup in that store, and the state survives a restart. Putting that backup's prefix back from +a copy clears it. When the backup is gone for good, remove its row: + +```bash +extenddb backup sync --forget-backup --config extenddb.toml +``` + +with the ARN the refusal and the server's log line name (`extenddb backup list` lists it too). First check that +the root is the one that holds the backups: a root that is not shared by every server (another host's directory +at the same path) shows the same refusal. An unmounted volume shows another one, "the root has no +extenddb-store.json but the catalog has rows for it", and `--forget-backup` leaves the rows of such a root ("the +configured root has no extenddb-store.json while the catalog holds rows for it (a volume that is not +mounted?)"), so its rows are not removed one by one from under an empty mount point. The command runs on the +unhealthy store, removes that row outright and deletes no +object, then runs the store's check once, writing nothing to the root, and prints it: "The configured store's +check passes", or the next `AVAILABLE` row whose manifest is gone, with the same command for it; repeat for that one (the option can be given more than +once, and an ARN whose row is already gone counts as done, so the same command line can be run again). Once the check +passes, `extenddb backup sync --prune-missing` removes the older rows whose manifest is gone, and each running +server finds the store healthy at its next reconciler cycle. To stay out of this state, take each catalog +backup right after the deletions your scheduler makes, so that the newest backup a catalog backup names is not +one deleted later. + +Tables backed up after the copy can be restored, under their own names or others, once `backup sync` lists +their backups; their writes after their last backup are lost. + +A PostgreSQL catalog database restored alone, beside a data database that survived, is a catalog older than +the tables, and gives this: a table created after the copy is not listed, and creating it again gives an empty +table while its old items stay in a table of the data database that nothing references; a table deleted after +the copy is listed and answers `ResourceNotFoundException` until it is deleted again. Restore the pair +instead. + +### The catalog is lost and re-created + +When no catalog backup exists, `extenddb init` creates a new catalog with a new catalog id. `init` refuses a +config file that exists (exit 255, "Config file "" already exists. Use --overwrite to delete and +regenerate it."), and `extenddb destroy` and a lost database disk both leave that file in place. `init +--overwrite` regenerates the whole file from defaults. Only the storage settings are carried over, as the new +catalog's (on PostgreSQL the connection string, on SQLite the database path); the other default sections +(`[server]`, `[server.tls]`, `[auth]`, `[auth.cache]`, `[logging]`, `[limits]`) come back with default +values, so a `port`, `bind_addr`, TLS path, cache `ttl_seconds` or limit you set in them is gone and the new +server listens on 18443; and `[backup]`, +`[backup.store]`, `[import]`, `[export]` and every other section you added are dropped. Copy the file aside +before running `init`, and before the new server starts put the copy back whole, taking only the regenerated +storage settings from the new file. Without `[backup.store]` a PostgreSQL, MongoDB or Cassandra server has +no store (`/health` shows no `backup_store` component, `CreateBackup` writes in-catalog backups, and `backup +sync --adopt-store` exits 1, "no backup store is configured"), and a SQLite server uses the default root beside +the database; when the store was configured elsewhere, that root is a new, empty directory, and `--adopt-store` +finds nothing there to adopt. + +With the sections back, the store's `extenddb-store.json` still names the old catalog, so the new server +starts with the store unhealthy: + +``` +{"components":{"backup_store":{"kind":"filesystem","reason":"owned by another catalog","status":"unhealthy"}},"status":"degraded"} +``` + +On file-backed SQLite with the default root (no `[backup.store]` section) the server instead logs a warning, +"the default root ... failed its check (extenddb-store.json names catalog ; this catalog is ); the catalog has no backup rows for it, so the store is unconfigured and CreateBackup writes in-catalog +backups", and `/health` shows no `backup_store` component (`{"components":{},"status":"healthy"}`). Until +the adoption below, `CreateBackup` answers `AVAILABLE` at once and writes an in-catalog backup into the +database file, the failure domain this manual's first paragraph warns about; such a backup stays in-catalog +after the adoption ([Legacy backups](#legacy-backups)). Pause scheduled backups until the adoption, or +configure `[backup.store]` explicitly, which gives the unhealthy store above instead. + +In both shapes plain `backup sync` refuses (exit 1, "owned by another catalog"). Then: + +1. Stop every server of the old catalog, on every host. A server that still runs keeps acting on the root + until a later check finds it adopted. +2. Start a server of the new catalog: `extenddb manage` works through its management API, and reads the + admin password from `EXTENDDB_PASSWORD`, which keeps it out of the process list (without it or + `--password` it refuses, "No password provided"). Re-create the accounts with their old account ids, + `extenddb manage --user admin create-account --account-id + --account-name ` (the admin guide's Accounts section), and their IAM users and policies. A backup is + reachable only by callers of the account its ARN names. `create-access-key` makes new key pairs; to keep + the clients' existing ones, import them. The server refuses an import unless the runtime setting + `allow_credential_import` is `true`, and a re-created catalog has no value for it, which counts as + `false`: `import-access-key` exits 1, "Error: HTTP 403: Credential import is disabled + (allow_credential_import = false)", and the management API answers 403 with the same text. Turn it on + first, beside the running server (the next import reads it; no restart): + + ```bash + extenddb settings --config extenddb.toml set allow_credential_import true + ``` + + Then import each pair with `extenddb manage --user admin import-access-key --account-id + --user-name --access-key-id --secret-access-key --yes`. While that command + runs, the secret is in its arguments, which the process list shows to every user of the host. On a shared + host send the pair to the management API instead, in a request body read from a file: + `curl --cacert --netrc-file -H 'Content-Type: application/json' --data @key.json + https://:18443/management/accounts//users//access-keys/import`, where `key.json` holds + `{"access_key_id": "", "secret_access_key": ""}` and the netrc file the admin user's name + and password, both created under `umask 077`; the answer is `Access key imported`. Once every pair is + imported, turn the setting off again, `extenddb settings --config extenddb.toml set allow_credential_import + false` (the deployment guide's production checklist); the imported pairs keep signing requests. On SQLite + stop the server before step 3. +3. Adopt the root and sync it into the new catalog, on SQLite with the server stopped and on the other + backends beside the running servers: + + ```bash + extenddb backup sync --adopt-store --config extenddb.toml + ``` + + It rewrites `extenddb-store.json` to name the new catalog (keeping the store id), logs the change under + `extenddb::audit`, and syncs: + + ``` + Adopted the root: extenddb-store.json named catalog and now names this catalog (store id ). + ... + --- Adopted (an AVAILABLE row inserted for each): + arn:aws:dynamodb:us-east-1::table/orders/backup/ + ... + === extenddb backup sync complete: 1 backup(s) adopted, 0 prefix(es) skipped, 0 AVAILABLE row(s) whose manifest is gone === + ``` + +4. On SQLite start the server again; on the other backends the running servers find the store adopted at a + later check. `/health` reports the `backup_store` component healthy (on SQLite's default root as well), the + backups are `AVAILABLE`, and each restores as in the drill. The new catalog knows none of the old tables; restore each + from its backup. + +On SQLite every `backup sync` needs the server stopped, since it opens the database as the server does. On +PostgreSQL, MongoDB and Cassandra it runs beside servers and holds the catalog's maintenance claim while it +runs, during which `CreateBackup` and `RestoreTableFromBackup` on every instance answer +`LimitExceededException`; `--adopt-store`, `--prune-missing`, `--move-store`, `--forget-store` and +`--forget-backup` first wait, as `purge` does, until no backup +or restore job is running and no backup failed within `stale_job_after_secs` (a job that failed may still be +writing to its prefix), at most `2 * stale_job_after_secs`, and then refuse. Sync exits 0 when it completed +and 1 when it refused or skipped a prefix for a store or catalog error (run it again). It exits 6 when it +cannot open the catalog: beside a running server on SQLite ("another extenddb process is already using ..."), +or on the other backends while a migration runs. On a catalog `serve` would refuse it exits with `serve`'s +code: 3 with migrations pending (run `extenddb migrate`), 4 on a catalog a later release migrated with a +breaking migration, and 5 on one that needs attention (the upgrade manual's [catalog +status](07-upgrade-manual.md#the-catalog-status)). +The admin guide's `backup sync` paragraph has its options and output in full. + +### An in-memory SQLite server + +A server on an in-memory SQLite database (`[storage.sqlite] path = ":memory:"`) starts with a new, empty +catalog every time. With `[backup.store]` configured it adopts, at startup, a root that an earlier in-memory +server owned, and syncs the root's backups into its new catalog before it serves (on a filesystem store; an +S3 store in the reconciler's first cycle). The log line + +``` +backup store (filesystem) startup sync of the in-memory catalog: 1 backup(s) adopted, 0 prefix(es) skipped (0 of them reserved for exports or the change-log archive) +``` + +ends the window in which the earlier backups answer `BackupNotFoundException`. Tables, accounts and IAM do not +survive the restart; only the backups do. A backup is reachable only by callers of the account its ARN names, +and the in-memory server creates its default account with a new id at every start, so until that account +exists again the adopted backups answer `AccessDeniedException` and `ListBackups` lists none of them. After +each start, re-create the accounts with their old ids (`extenddb manage --user admin create-account +--account-id --account-name `, as the admin user, with `EXTENDDB_PASSWORD` set to its +password: the `EXTENDDB_ADMIN_PASSWORD` the server started with or, without it, the one it printed at +start), then their +users, policies and keys; the backups then answer `DescribeBackup` and restore. Each start's catalog also has +no value for `allow_credential_import`, so an import of an old key pair is refused with the 403 of step 2 above, and +`extenddb settings` cannot set it: the command opens an in-memory catalog of its own and fails, "Error: Failed +to set setting: Internal("set_setting: error returned from database: (code: 1) no such table: settings")". +After each start, turn it on through the server's management API as the admin user before the imports, and +off again after them (`"false"`): + +```bash +curl --cacert --netrc-file -X PUT -H 'Content-Type: application/json' \ + --data '{"value": "true"}' https://:18443/management/settings/allow_credential_import +``` + +The answer is `{"key":"allow_credential_import","value":"true"}`. Start a new in-memory server +only after the old one has exited. A root that a durable catalog owns is never adopted by an in-memory server. +Every `extenddb backup` command refuses an in-memory database (exit 1), since it would open a new, empty +catalog that ends with the command. + +## Moving a store to a new root + +Each backup's row names its store's root as the catalog stores it (the canonical directory of a filesystem +store; `s3:///` for S3, with `?endpoint=` when an endpoint is set), so a tree copied to +a new root comes with no rows: configured with the new root, `extenddb backup sync` skips every copied prefix, +naming the row of the old store that holds it, and lists the old root's rows under "Rows of other stores". +`--move-store` re-points those rows at the configured store, and the backups keep their ARNs: + +1. Reconfigure every server of the catalog to the new root, on every host, and restart it (on SQLite stop it: + the move needs the serve lock). A server left on the old root keeps writing its backups there after the + move, and nothing detects it beyond the next sync's report of other stores. +2. Copy the old root's tree under the new root, whole and with the same relative paths, as the server's user, + keeping directories `0700` and files `0600`, and leave the old root's `extenddb-store.json` out of the + copy: the servers restarted in step 1 gave the new root its own (`rsync -a --exclude=/extenddb-store.json + /srv/backups/old/ /srv/backups/new/`; for S3, `aws s3 sync --exclude extenddb-store.json` between the two + prefixes). Finish the copy before the move: the move checks the copy once, and the reconciler later deletes + what the copy brought of the moved `FAILED` and `DELETED` backups. If the old root's `extenddb-store.json` + was copied over the new root's, every running server's store turns unhealthy (an identity mismatch). With + no backup taken under the new root since the restart, the move still runs; restart every server after it. + With one taken (a server kept serving between the steps), the move refuses with the same mismatch: + `extenddb backup list` names those backups under the new root's first store id; remove each row with + `extenddb backup sync --forget-backup ` (its bytes stay), run the move, whose sync adopts them again, + and restart every server. A `DELETING` row among them (a `DeleteBackup` that had not finished) is refused, + naming its store id as one the identity check refuses: its deletion waits on a check that cannot pass while + the copied file and that row's id disagree. Remove it with the same command run with `[backup.store] kind = + "disabled"` (its bytes stay), configure the new root again, run the move, whose sync adopts that backup + too, restart every server, and `DeleteBackup` it again. +3. With the configuration naming the new root, run: + + ```bash + extenddb backup sync --move-store /srv/backups/old --config extenddb.toml + ``` + + Give the old root byte for byte, as `extenddb backup list` printed it before the change or as plain sync's + report of other stores prints it. A root that differs by a trailing `/`, or by an S3 root's `?endpoint=` + suffix, has no rows; the command says so and lists the roots the catalog holds rows for. + +The move needs the configured store healthy. It takes the maintenance claim, waits for the running jobs of the +configured store and of the old root, and checks every row of the old root against the copy: + +- an `AVAILABLE` row needs its prefix complete under the new root: the manifest and its sidecar, the row's ARN, + every data file at its recorded size, and the row's size and item count; +- a `CREATING` row whose job is dead is resolved `AVAILABLE` when the copy holds its complete backup. A copy + without the complete backup stops the move and the row is left `CREATING`: a copy that lacks the manifest + or a data file says nothing about the old root's prefix, and a `FAILED` row would have the reconciler delete + the copy. Run a server configured with the old root (on SQLite, plain `extenddb backup sync` there), whose + reconciler resolves the row against the old root, then copy that prefix again; or remove the row with + `--forget-backup`. A `CREATING` row whose job still runs stops the move; +- a `DELETING` row has its deletion finished under the new root, which deletes what the copy brought of it; +- a `DELETING`, `DELETED` or `FAILED` row whose copied prefix holds a complete backup stops the move: moving + that row would have the reconciler delete the backup. For a `DELETING` or `DELETED` row the backup was + deleted: remove that prefix from the new root and run the move again. Removing the row with + `--forget-backup` instead would have the next sync adopt the prefix, so the backup a client deleted would be + `AVAILABLE` again and would have to be deleted again. For a `FAILED` row, remove the prefix, or remove the + row with `--forget-backup` and let the next sync adopt the prefix. + +Every row that stops the move is named with its reason in one run, and no row moves while any does (exit 1): + +``` +--- Stopping the move (no row is moved while any is listed here): + arn:aws:dynamodb:us-east-1::table/orders/backup/ (AVAILABLE, account ): not a complete copy under the configured root: data file data/000001.json.gz is not in the store; copy the prefix again (or remove the row with --forget-backup) +``` + +Copy what is missing and run the command again, or remove the row with `extenddb backup sync --forget-backup +` and run the move again; the prefix is then reported as an incomplete backup by every sync until the +missing file is copied, and the next sync adopts it. With no row stopping it, the move rewrites the rows, +logs each under `extenddb::audit`, and syncs the new root: + +``` +=== extenddb backup sync --move-store complete: 3 row(s) moved, 0 resolved, 0 deletion(s) finished, 0 row(s) stopping the move, 0 gone during the move, 0 left under the old root === +``` + +`DescribeBackup`, `ListBackups` and restores of the moved backups then work against the new root. A row still +under the old root after the rewrite (a `CreateBackup` of a server left on the old root) is listed and the +command exits 1: its prefix was written after the copy, so reconfigure that server, copy its prefix under the +new root (or `--forget-backup` it), and run the move again. The move rewrites the rows 500 at a time (on +Cassandra, which rewrites key by key, one row at a time), confirms its maintenance claim before each 500 rows and +reads the claim's state, with no write, before every call: a move whose claim another command took over (on +MongoDB and Cassandra, a move stopped for longer than `stale_job_after_secs`) writes nothing after the one call +it had sent or was about to send when it stopped (one row on Cassandra, up to 500 on MongoDB), and exits 1 with +the earlier rows moved and the rest listed under the old root; a second run moves them. A first Ctrl-C or +`SIGTERM` during the rewrite stops it the same way before its next call. On PostgreSQL +each slice is one statement in a transaction of its own, under a 30 s statement timeout (the admin guide's +`backup sync` paragraph has the figures). A move interrupted after its rewrite (a signal, a lost connection) +has moved its rows: run plain `extenddb backup sync`, since a second `--move-store` then finds no row under +the old root and says so. A row of the old root in the report of other stores the move's own sync prints (a +`CreateBackup` whose insert landed during that sync) is named and the command exits 1 ("a row landed under the +old root during the move"). Nothing under the old root is read or changed, and no object is copied. Before you +delete the old root, run plain `extenddb backup sync` and check that its report of other stores lists no row +of the old root: a server still configured with it, or a `CreateBackup` that read the maintenance claim before +the move took it and inserted its row after the move's report, can leave a row there whose bytes exist only +under the old root. On Cassandra that report reads each root's rows whatever status their status entry is +under, so a row of the old root is listed as a row; a line that counts only "status entries with no row of +that status behind them" names no row, and `extenddb backup sync --forget-store ` removes those +entries. A restore that failed because it read a store other than the configured one is run again once its +backup's rows are moved. The move's memory grows with the rows of both roots: every backend reads a listing's +rows whole, and the move reads the old root's rows and the configured root's (for the check of one backup id +per account; the sync after the move reads them again), each about 23 KiB per row on PostgreSQL and 27 on +SQLite, above about 50 MiB, for a table with five GSIs, measured at 1,452 bytes of table description per row +(the figure grows with the description; the admin guide's `backup sync` paragraph has the figures). + +To drop the rows of a root without moving them (a root that is gone, or one retired with its backups), +`extenddb backup sync --forget-store ` removes every row of that root outright, leaving no tombstone, +and deletes no object; it refuses the configured root. It runs on a store in any state and with no store +configured; a configured root that cannot be opened (a directory that does not exist: "[backup.store] ... +cannot be opened") stops it before anything is read, as it stops plain sync, and the same command with +`[backup.store] kind = "disabled"` runs it. Beside servers it waits a minute after taking the maintenance claim before its first removal of a +`FAILED` or `DELETED` row, and prints the wait: a reconciler cycle that began before the claim may be deleting +such a row's prefix, and its deletion stops within that time, checking its deadline before each removal, so +what it can still do once the command removes the row is the one removal (or one S3 request) in flight at that +deadline. That holds while the servers' processes keep running: a server paused for longer than the wait (a +stopped process, a frozen container or VM) in the middle of a deletion takes up that step when it resumes. A +`DELETING` row of another store keeps its bytes unless a server configured with that store finishes the +deletion (its reconciler does, the deletion the `DeleteBackup` asked for): if that root comes back with the +bytes, `backup sync` adopts the backup again. It waits for running jobs as `purge` does, and any running job +holds it, not only the jobs of the rows it removes: a backup job of the configured store (a `CREATING` or +`FAILED` row whose heartbeat is live), or a running restore from any backup. The wait lasts up to twice +`stale_job_after_secs` (600 s at the default), and the command refuses, exit 1, if a job is still running when +it ends; a `CREATING` or `FAILED` row whose job writes a heartbeat after the wait, and a row a restore is +reading, are left and named (exit 1): run it again once the job ends, and for the restore once it ends or, if +it failed, once its target is removed with `extenddb backup remove-target --account
`. On +Cassandra it also sweeps the root's status entries: it deletes those that no row of the root stands behind, +writes again under its row's status one left under an earlier status (a status change whose entry batch +failed), and says how many of each. If the root comes back and is configured again, `backup sync` adopts its +backups again. `--forget-backup ` does the same for one backup; for a `DELETING` row of the configured +store it first waits as for a `DELETED` row, then finishes the deletion its `DeleteBackup` asked for (the +per-row line says the command deleted its objects), which needs the store's check to pass, and refuses the row +otherwise. While the configured root has no `extenddb-store.json` and the catalog holds rows for it (an +unmounted volume), or on SQLite the default root is missing while the catalog holds rows for it (moved away, +or a link to a volume that is gone), `--forget-backup` refuses that root's rows and writes nothing to it: +mount the volume, or point `[backup.store]` at the root that holds the backups, then run it again. + +## Legacy backups + +A backup written in the in-catalog format (by an earlier release, by a server without a configured store or +with `kind = "disabled"`, and on Cassandra always) lives in the database the server uses: on PostgreSQL in +the catalog database, on SQLite in the database file, on MongoDB in the catalog and data databases, on +Cassandra in the catalog keyspace and the account's keyspace. It shares the database's failure domain, and a +lost database loses it. + +Legacy backups stay listable, describable, restorable and deletable whether or not a store is configured: +`ListBackups` lists both kinds side by side. `extenddb backup list` lists only the store's rows. The +in-catalog format is deprecated: without a store `CreateBackup` logs a warning on each call ("is writing the +in-catalog backup format, which a later release stops creating; configure [backup.store] ..."), and a later +release stops creating it. + +This release has no command that converts a legacy backup to the store format (`backup migrate-legacy` is +planned, #821). To move one into the store today, with the store configured: + +1. Restore it under a new name: `aws dynamodb restore-table-from-backup --backup-arn + --target-table-name orders-moved --endpoint-url https://:18443`, and wait for `ACTIVE`. +2. `aws dynamodb create-backup --table-name orders-moved --backup-name --endpoint-url + https://:18443` and wait for `AVAILABLE`. This is a store-backed backup with a new ARN and a new + `BackupCreationDateTime`, of a table named `orders-moved`. +3. Check the new backup with a drill, then `aws dynamodb delete-backup` the legacy one and `aws dynamodb + delete-table` the interim table, each with `--endpoint-url https://:18443`. + +To keep the original table name, restore into it only once that table is gone: deleting a live table to make +room loses every write since the legacy backup. While the table exists, `create-backup` of it is a +store-backed backup of that name with its current items; keep the legacy backup only for its older contents. +A backup taken by an earlier release may restore without its secondary indexes +(the upgrade manual's Current Status section); take a new backup of each table after an upgrade rather than +moving the old ones. A table with a vector index has no legacy backup: the in-catalog format refuses it. + +## What a restore preserves + +A restore reproduces what Amazon DynamoDB's does, on every backend, with one gap (a GSI's on-demand throughput): + +| Carried across | Not carried across | +|---|---| +| items, with every number's 38 digits | table class (a restore is `STANDARD`) | +| key schema and attribute definitions | stream settings | +| global and local secondary indexes, rebuilt from the items | TTL settings | +| billing mode and provisioned throughput, table and GSI | tags | +| the table's on-demand throughput | a GSI's on-demand throughput (ExtendDB records none) | +| SSE settings | deletion protection | +| vector indexes, into PostgreSQL and SQLite | auto scaling, IAM policies, CloudWatch settings | + +Reapply what is not carried by hand after the restore. The overrides of `RestoreTableFromBackup` +(`BillingModeOverride`, `GlobalSecondaryIndexOverride`, `LocalSecondaryIndexOverride`, +`SSESpecificationOverride`, `OnDemandThroughputOverride`, `ProvisionedThroughputOverride`) apply as on the +service; `VectorIndexOverride` is refused. + +Across backends: a store-backed backup restores into any backend, with these refusals, each before any table +is created or with the target removed: + +- A backup with vector indexes is refused into MongoDB and Cassandra, which have no vector support. +- Into MongoDB, an `N` sort-key value of more than 34 significant digits fails the restore (the sort-key + column is Decimal128); other numbers keep their 38 digits. + +`DescribeTable` on a restored table reports `ItemCount` and `TableSizeBytes` 0 until the statistics pass +rewrites them. The differences document's [Backup and Restore](../differences-from-dynamodb.md#backup-and-restore) +section has each of these rows in full, with the multi-part-key preview and the restore's error texts. + +## Moving data to and from Amazon DynamoDB + +The backup format is the DynamoDB S3 export layout: each backup's `data/` directory holds gzip files of +DynamoDB JSON, one `{"Item": {...}}` object per line, beside the export's `manifest-summary.json` and +`manifest-files.json`, plus ExtendDB's own `extenddb-manifest.json` and its checksum file. + +From ExtendDB to Amazon DynamoDB: upload a backup's `data/` files to an S3 bucket and run the service's +`ImportTable` with `InputFormat` `DYNAMODB_JSON`, `InputCompressionType` `GZIP`, an `S3BucketSource` whose +`S3KeyPrefix` is that directory, and `TableCreationParameters` built from the backup: the key schema from +`describe-backup`'s `SourceTableDetails`, the GSIs from its `SourceTableFeatureDetails` (for an on-demand +table give no `ProvisionedThroughput`, the table's or a GSI's: the backup reports each as 0 and 0, which +the SDK's parameter validation refuses; `BillingMode` comes from `SourceTableDetails`), and the attribute +types (`AttributeDefinitions`, which neither carries) from the +`source_table` object of the backup's `extenddb-manifest.json`. `TableCreationParameters` has no member for +local secondary indexes, so the service's import creates none; a table with LSIs arrives without them. That is +the layout the service's import reads; this release has not run the import against the service. + +From Amazon DynamoDB to ExtendDB: export the table to S3 with `DYNAMODB_JSON`, download the export's data +files, decompress them and concatenate them into one file under an `[import]` path, in the calling account's +directory (`//`), then call ExtendDB's `ImportTable` with a `FileSource` naming that +file and `InputFormat` `DYNAMODB_JSON`. Name the file by its canonical path in `FileSource.Path`: a path +through a symbolically linked directory is refused, "Path must resolve under one of the configured allowed +paths", whether or not the `[import] paths` entry is itself written through that link, and a file that is +itself a symbolic link is refused with `ValidationException` "Symbolic links are not allowed in import/export +paths". ExtendDB's `ImportTable` reads one uncompressed local file (a gzip file is refused, "I/O error reading +import file at line 1"), creates the table from `TableCreationParameters`, reads the whole file before it +writes the first item, completes before it answers, and skips the items that fail validation (keys that do not +match the key schema, an item over the size limit), counting them in `ErrorCount`; the file is bounded by +`max_import_file_bytes` (10 GiB) and `max_import_item_count` (10 million). The same works for one of +ExtendDB's own backup data files. + +ExtendDB's `ExportTableToPointInTime` writes one uncompressed DynamoDB JSON file under an `[export]` path, in +the calling account's directory, at the path the request's `FilePath` member names (an ExtendDB member; a +request without it reads `S3Prefix` as the path, and `S3Bucket` is accepted and ignored, so a request naming +neither path is refused, "Either FilePath or S3Prefix must be provided"). It reads the table as it changes, so it is not a consistent snapshot. Reading a store prefix or a +DynamoDB export directly, with `ImportTable` or a restore, is planned (#748) and not in this release; `backup +sync` adopts only prefixes with an ExtendDB manifest. + +## License + +Copyright 2026 ExtendDB contributors. Licensed under the Apache License, Version 2.0. +See [LICENSE](../../LICENSE) for the full text. + +This software is provided "as is" without warranty of any kind. ExtendDB is not +affiliated with, endorsed by, or sponsored by Amazon Web Services. "DynamoDB" is a trademark +of Amazon.com, Inc. diff --git a/docs/rfcs/draft-cassandra-backend.md b/docs/rfcs/draft-cassandra-backend.md index d40f74118..4433af568 100644 --- a/docs/rfcs/draft-cassandra-backend.md +++ b/docs/rfcs/draft-cassandra-backend.md @@ -190,6 +190,12 @@ high expiration rates will accumulate backlog. ### Backup and restore +> Superseded in part. This section describes the first implementation. Restores now run as background jobs that +> restore the secondary indexes, and a backup in a configured backup store restores into Cassandra; +> `CreateBackup` on Cassandra still writes the in-catalog backup, before it returns. See the +> [Backup and Restore manual](../manuals/14-backup-and-restore.md) and the design, +> [15-backup-restore.md](../design/15-backup-restore.md). + Cassandra's native snapshot mechanism (`nodetool snapshot`) produces node-local SSTable artifacts managed outside CQL. Creating, enumerating, or restoring a distributed snapshot requires cluster-level orchestration that is not available through the driver session, so the backend implements on-demand backups as logical snapshots instead. @@ -234,6 +240,11 @@ writes can race the scan. `RestoreTableToPointInTime` is not supported. and `ImportTable` execute synchronously and block the API response until complete. Amazon DynamoDB performs these asynchronously. +> Superseded in part: `RestoreTableFromBackup` now answers `CREATING` and copies in the background on every backend, +> and `CreateBackup` with a backup store does so on PostgreSQL, SQLite and MongoDB. On Cassandra `CreateBackup` still +> completes before it returns, as do `ExportTableToPointInTime` and `ImportTable` on every backend. See the note under +> "Backup and restore". + **Transaction recovery is conservative.** Transactions found in PREPARING state during recovery are always rolled back, even if they could theoretically have been committed. This is safe but means a transaction that completed PREPARE and then encountered a server crash will be rolled back rather than committed. @@ -252,8 +263,15 @@ path is suitable at scale. `ImportTable` to background jobs with status polling is planned but not yet implemented. This applies to all current ExtendDB backends. +> Superseded in part: the restore, and `CreateBackup` with a backup store on PostgreSQL, SQLite and MongoDB, now run +> as background jobs (the note under "Data movement operations are synchronous" above). + **Restore does not recover GSI/LSI definitions, tags, TTL configuration, or stream history.** +> Superseded in part: a restore now recovers the GSI and LSI definitions, rebuilt from the items, on every backend, of a backup +> that recorded them; a backup an earlier release took may not have (the upgrade manual's Current Status section). +> Tags, TTL configuration and stream history are still not restored. + **Transaction fence uses item data equality.** The TTL deletion protocol conditions the final base item delete on the stored item image rather than a version counter. A monotonic version column would be a more robust fence and remove the dependency on stable JSON encoding. diff --git a/docs/technical-debt.md b/docs/technical-debt.md index fc125ff08..45e42a6ec 100755 --- a/docs/technical-debt.md +++ b/docs/technical-debt.md @@ -72,13 +72,13 @@ Last updated: 2026-11-09 | # | Item | Location | Priority | Origin | |---|------|----------|----------|--------| -| A-1 | ~~Catalog/data database separation not implemented (REQ-CAT-001/002)~~ Implemented at the location this row cites: the runtime reads the data connection string from the catalog and opens a second pool, and the item, GSI-queue, and vector-index paths run on it. Residual: the pool selection falls back to the catalog pool when the setting is absent, so a hand-written configuration can run both roles on one database without a warning, which makes the separation enforced by `extenddb init` rather than by the code path | `storage-postgres/src/lib.rs:238-256` (the second pool and its fallback) | ~~High~~ Low | P40 | +| A-1 | ~~Catalog/data database separation not implemented (REQ-CAT-001/002)~~ Implemented at the location this row cites: the runtime reads the data connection string from the catalog and opens a second pool, and the item, GSI-queue, and vector-index paths run on it. Residual: the pool selection falls back to the catalog pool when the setting is absent, so a hand-written configuration can run both roles on one database without a warning, which makes the separation enforced by `extenddb init` rather than by the code path | `PostgresEngine::new` in `storage-postgres/src/lib.rs` (the second pool and its fallback) | ~~High~~ Low | P40 | ### A-1: Catalog/Data Database Separation -**Design requirement:** Two databases — catalog (`extenddb`) for metadata, data (`extenddb_data`) for user items (REQ-CAT-001, REQ-CAT-002). +**Design requirement:** Two databases — a catalog for metadata and a data database for user items (REQ-CAT-001, REQ-CAT-002); on PostgreSQL `init` names them `extenddb_catalog` and `extenddb` by default. -**Current state:** Implemented on PostgreSQL. `extenddb init` creates both databases, records the data connection string in the catalog `settings` table (`storage/src/bootstrapper.rs:139`, called at `app/src/cmd_init.rs:334`), and initialises the data schema. The runtime reads that setting and opens a second pool (`storage-postgres/src/lib.rs:238-256`). Table creation, the GSI queue, the item-write paths, vector index data, and the backup and stream engines all take that pool (`create_table.rs:311`, `gsi_queue.rs`, `data/put_item.rs`, `data/vector_index.rs`, `backup_engine.rs`, `stream_engine.rs`), and `create_table.rs:300` commits catalog metadata before creating data tables. `extenddb destroy` drops both databases (`storage-postgres/src/bootstrapper.rs:725` and `:734`). SQLite co-locates catalog and data by design, which its worker module states (`storage-sqlite/src/workers.rs:11`). +**Current state:** Implemented on PostgreSQL. `extenddb init` creates both databases, records the data connection string in the catalog `settings` table (`PostgresBootstrapper::record_data_connection` in `storage-postgres/src/bootstrapper.rs`, a `Bootstrapper` method of `storage/src/bootstrapper.rs`, called from `app/src/cmd_init.rs`), and initialises the data schema. The runtime reads that setting and opens a second pool (`PostgresEngine::new` in `storage-postgres/src/lib.rs`). Table creation, the GSI queue, the item-write paths, vector index data, and the backup and stream engines all take that pool (`create_table.rs`, `gsi_queue.rs`, `data/put_item.rs`, `data/vector_index.rs`, `backup_engine.rs`, `stream_engine.rs`), and `create_table_impl_inner` (`create_table.rs`) commits catalog metadata before creating data tables. `extenddb destroy` drops both databases (`PostgresBootstrapper::drop_databases` in `storage-postgres/src/bootstrapper.rs`). SQLite co-locates catalog and data by design, which its worker module states (`storage-sqlite/src/workers.rs:11`). **Residual, which is a note rather than an architectural gap:** the pool selection ends in `_ => pool.clone()`, so a catalog with no `data_database_connection_string` runs both roles on one database and nothing says so. The separation is enforced by `init` rather than by the code path. That is also what the connection formula in `docs/getting-started.md` assumes: it says the collapsed shape is the only one where the two `pool_size` pools become one, and that `extenddb init` does not produce it. Fix, if wanted: warn or refuse when the setting is absent, instead of collapsing silently. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index d9f2680bf..799ed66c0 100755 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -932,7 +932,7 @@ Keys imported earlier keep working while import is disabled. pool_size = 50 # default is 20 ``` -If the problem persists, check for long-running queries or connection leaks with `SELECT * FROM pg_stat_activity WHERE datname = 'extenddb_data';`. +If the problem persists, check for long-running queries or connection leaks with `SELECT * FROM pg_stat_activity WHERE datname = 'extenddb';` (the data database's default name). **Known limitation:** The HTTP status code should be 503 with a `Retry-After` header. This is tracked as technical debt. diff --git a/tests/test_backup_job.py b/tests/test_backup_job.py index 1ba68d83d..c83f1bef4 100644 --- a/tests/test_backup_job.py +++ b/tests/test_backup_job.py @@ -81,7 +81,8 @@ - an identity file rewritten to name another catalog while a plain sync is in its prefix loop ends the command with exit 1, naming that catalog; - (beside a server) while `--prune-missing` holds the maintenance claim, `backup - purge` and `--adopt-store` fail at once, and `--adopt-store` on a root + purge`, `--adopt-store`, `--move-store`, `--forget-store` and + `--forget-backup` fail at once, and `--adopt-store` on a root the catalog owns waits for a running job as `--prune-missing` does; - (beside a server) on host-local roots, `--prune-missing` on the host that lacks a backup's bytes removes its row only, the other host keeps the @@ -98,6 +99,46 @@ deletion fence and leaves the bytes, and the adopting catalog's store stays healthy. +`backup sync --move-store`, `--forget-store` and `--forget-backup`, on the +three deployments (the steps that edit a row at the catalog on SQLite and +PostgreSQL): + +- a tree copied to a new root: sync there names the old store's rows, + `--move-store` refuses the configured root and a root that differs by a + trailing `/`, and stops on a FAILED row whose copy is a complete backup + and on a copy missing a data file, naming each and moving none; + `--forget-backup` of the incomplete one lets the move finish, the sync + after it skips that prefix until the file is copied, and a server on the + new root restores a moved backup; +- the rows of another root are left alone by a server's reconciler (a copy + of that root's tree under the configured root included) and reported by + sync; `--forget-store` removes them outright and deletes + nothing, after which the root's return has its prefixes adopted; +- a store left unhealthy by its newest backups' missing manifests is cleared + by `--forget-backup`, which prints the store's check, after which + `--prune-missing` runs; +- `--forget-backup` leaves a live CREATING row (PostgreSQL), an ARN spelt + with another region and an in-catalog backup, and runs with no store + configured; +- (PostgreSQL, MongoDB) a CreateBackup during `--move-store`'s wait for a + running job is refused by the maintenance claim, and the move finishes + once the job ends; +- (PostgreSQL) a CreateBackup of a server left on the old root whose row + lands during the move's sync is named under "Rows that landed under the + old root during the move" and the move exits 1; +- (PostgreSQL, MongoDB) the backup manual's move procedure in its order + beside a running server, the tree copied without its identity file and a + CreateBackup between the restart and the copy: the server stays healthy, + restores a moved backup and backs up again; with the identity file + copied, the manual's remedy (`--forget-backup` of the backup taken + meanwhile, the move, a restart) ends in the same state; +- `--forget-backup` refuses a string that is no ARN before the claim, goes + past an ARN whose row is gone, and on an empty root under the rows runs no + store check and writes nothing there; +- (PostgreSQL) a forget option started while a DELETING row and a dead + RUNNING restore exist proceeds once a server's held-claim cycle has + finished the one and failed the other. + The binary at `EXTENDDB_BINARY` decides which deployment runs: a SQLite build, a PostgreSQL build (with `EXTENDDB_TEST_PG_CONNECTION_STRING`) or a MongoDB build (with `EXTENDDB_TEST_MONGODB_URI`, a replica set of this suite's own: the @@ -1816,8 +1857,15 @@ def test_sync_rebuilds_dropped_rows_and_adopts_only_verified_prefixes(deployment others[label] = arn if d.backend == "sqlite": # The serve lock held is the catalog refusal `serve` gives for - # it: exit 6, before any store or row is read. - for args in (("sync",), ("sync", "--prune-missing")): + # it: exit 6, before any store or row is read. The options that + # move or remove rows of other stores take it too. + for args in ( + ("sync",), + ("sync", "--prune-missing"), + ("sync", "--move-store", os.path.join(d.cwd, "elsewhere")), + ("sync", "--forget-store", os.path.join(d.cwd, "elsewhere")), + ("sync", "--forget-backup", kept), + ): beside = d.run("backup", *args) assert beside.returncode == 6, beside.stdout + beside.stderr assert "another extenddb process" in beside.stdout + beside.stderr, beside.stdout + beside.stderr @@ -1941,8 +1989,8 @@ def test_prune_missing_and_the_unhealthy_store_refusals(deployment, tmp_path): sync reports it with the `--prune-missing` hint and leaves it AVAILABLE; `--prune-missing` takes the maintenance claim and removes the row outright (no tombstone, BackupNotFound), deleting no object: the rest of - its prefix stays. An unhealthy store refuses plain sync and - `--prune-missing`, exit 1, with no row changed and nothing written to the + its prefix stays. An unhealthy store refuses plain sync, `--prune-missing` + and `--move-store`, exit 1, with no row changed and nothing written to the root, for each shape an unmounted volume gives: an empty root under the rows, a root holding only the identity file (the newest AVAILABLE row's manifest absent), and a root whose identity file carries another store @@ -2009,7 +2057,11 @@ def test_prune_missing_and_the_unhealthy_store_refusals(deployment, tmp_path): if content is not None: _private_file(os.path.join(d.root, "extenddb-store.json"), content) tree = _tree(d.root) - for args in (("sync",), ("sync", "--prune-missing")): + for args in ( + ("sync",), + ("sync", "--prune-missing"), + ("sync", "--move-store", os.path.join(d.cwd, "elsewhere")), + ): res = d.run("backup", *args) out = res.stdout + res.stderr assert res.returncode == 1, (shape, args, out) @@ -2043,8 +2095,10 @@ def test_a_second_maintenance_command_fails_at_once(deployment, tmp_path): """`backup sync --prune-missing` takes the maintenance claim and waits for a CREATING row whose owner beat just now (the row edited at the catalog, its heartbeat the database's now); while it waits, `backup - purge --confirm` and `backup sync --adopt-store` each fail within - seconds, exit 1, naming the one-at-a-time rule, and change nothing. The + purge --confirm`, `backup sync --adopt-store`, and `--move-store`, + `--forget-store` (of a root a second backup's row was edited to name) + and `--forget-backup` each fail within seconds, exit 1, naming the + one-at-a-time rule, and change nothing. The first command, signalled, stops before any change and releases the claim. `--adopt-store` on the root this catalog owns then takes the claim and waits for the same live job ("Waiting for running jobs to @@ -2061,8 +2115,23 @@ def test_a_second_maintenance_command_fails_at_once(deployment, tmp_path): server, ddb, creds = _started(d, str(tmp_path / "serve.log")) try: name, busy = _small_backed_up_table(ddb, "claim") + moved = ddb.create_backup(TableName=name, BackupName="elsewhere")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, moved, timeout=60) finally: server.stop() + elsewhere = os.path.join(d.cwd, "elsewhere") + if d.backend == "mongodb": + n = _mongosh( + f"print(db.backup_index.updateOne({_mongodb_row_filter(moved)}, " + f"{{$set: {{store_root: {json.dumps(elsewhere)}}}}}).matchedCount)" + ) + else: + n = _catalog( + d, + "UPDATE backup_index SET store_root = ? WHERE account_id = ? AND backup_id = ?", + (elsewhere, ACCOUNT, _backup_id(moved)), + ) + assert n == 1, f"{n} backup_index rows match {moved}" _set_status(d, busy, "CREATING") out_path = str(tmp_path / "first.out") with open(out_path, "wb") as out: @@ -2077,7 +2146,13 @@ def test_a_second_maintenance_command_fails_at_once(deployment, tmp_path): assert time.monotonic() < deadline, open(out_path).read() time.sleep(0.2) assert busy in open(out_path).read() or name in open(out_path).read(), open(out_path).read() - for args in (("purge", "--confirm"), ("sync", "--adopt-store")): + for args in ( + ("purge", "--confirm"), + ("sync", "--adopt-store"), + ("sync", "--move-store", elsewhere), + ("sync", "--forget-store", elsewhere), + ("sync", "--forget-backup", moved), + ): started = time.monotonic() res = d.run("backup", *args) took = time.monotonic() - started @@ -2087,6 +2162,7 @@ def test_a_second_maintenance_command_fails_at_once(deployment, tmp_path): assert took < 15, f"{args} took {took:.1f} s to refuse" print(f"{' '.join(args)}: refused in {took:.1f} s") assert first.poll() is None, "the first command stopped while the others ran" + assert _rows_present(d, [moved]) == 1, "a refused command removed the other root's row" first.send_signal(15) assert first.wait(timeout=30) == 1 text = open(out_path).read() @@ -2832,3 +2908,1112 @@ def test_an_adopter_whose_put_lands_during_the_sync_ends_it_with_exit_1(deployme ) in text, text[-4000:] assert f" {arn}" in _adopted_section(text), text[-4000:] assert "=== extenddb backup sync complete" not in text, text[-4000:] + + +# --- `backup sync --move-store`, `--forget-store`, `--forget-backup` --------------- +# +# These cells change rows only through the binary, so they run on the MongoDB +# deployment too; the steps that edit a row at the catalog (`_catalog`) run on +# SQLite and PostgreSQL only and say so. + +_STORE_SECTION = re.compile( + r"^\[backup\.store\]\n(?:kind = .*\n)(?:\[backup\.store\.filesystem\]\npath = .*\n)?", re.M +) + + +def _set_root(d: _Deployment, path: str) -> None: + """Configure the deployment's filesystem store at `path`, replacing the + `[backup.store]` section the fixture or an earlier call wrote (SQLite's + fixture writes none: its root is the defaulted one).""" + with open(d.config) as f: + text = f.read() + section = f'[backup.store]\nkind = "filesystem"\n[backup.store.filesystem]\npath = "{path}"\n' + found = list(_STORE_SECTION.finditer(text)) + assert len(found) <= 1, text + if found: + text = text[: found[0].start()] + section + text[found[0].end() :] + else: + text += section + with open(d.config, "w") as f: + f.write(text) + + +def _set_store_disabled(d: _Deployment) -> None: + """Configure `[backup.store] kind = "disabled"`: CreateBackup writes an + in-catalog backup, and no store is configured.""" + with open(d.config) as f: + text = f.read() + found = list(_STORE_SECTION.finditer(text)) + section = '[backup.store]\nkind = "disabled"\n' + if found: + text = text[: found[0].start()] + section + text[found[0].end() :] + else: + text += section + with open(d.config, "w") as f: + f.write(text) + + +def _copy_root(src: str, dst: str) -> None: + """Copy a store tree as `cp -rp` does (contents, modes and times; the + identity file included).""" + shutil.copytree(src, dst, copy_function=shutil.copy2) + os.chmod(dst, 0o700) + + +def _copy_tree_into(src: str, dst: str, identity: bool = False) -> None: + """Copy a store tree into an existing root as the backup manual's move + procedure says, `rsync -a --exclude=/extenddb-store.json src/ dst/`: + contents, modes and times, the identity file at the top left out unless + `identity`.""" + + def top_identity(directory: str, names: list[str]) -> list[str]: + if not identity and os.path.samefile(directory, src): + return [n for n in names if n == "extenddb-store.json"] + return [] + + shutil.copytree(src, dst, copy_function=shutil.copy2, ignore=top_identity, dirs_exist_ok=True) + + +def _tree_without_identity(root: str) -> list[str]: + """`_tree` without the identity file at the top.""" + return [entry for entry in _tree(root) if not entry.startswith("extenddb-store.json")] + + +def _section(stdout: str, header: str) -> str: + """One `--- header` section of a command's report: the header's line + and the lines under it, up to a blank line or the next `---` or `===` + line.""" + assert header in stdout, f"{header!r} not in\n{stdout}" + lines = stdout.split(header, 1)[1].split("\n") + kept = [lines[0]] + for line in lines[1:]: + if not line or line.startswith(("---", "===")): + break + kept.append(line) + return "\n".join(kept) + + +OTHER_STORES = "--- Rows of other stores (left as they are; sync reads and changes none of them):" + + +def test_move_store_moves_a_copied_tree_and_stops_on_an_incomplete_copy(deployment, tmp_path): + """Three backups under the root; the tree copied to a new root and the + configuration pointed at it. Plain sync there skips each prefix, naming + the row of the old store that holds it, and its report of other stores + counts the old root's three AVAILABLE rows and names `--move-store` and + `--forget-store`. `--move-store` refuses the configured root and a root + that differs from the old one by a trailing `/` (printing the stores the + catalog knows). (SQLite and PostgreSQL) A FAILED row whose copied prefix + holds a complete backup stops the move, naming `--forget-backup` and the + adopting sync; a DELETED one stops it saying the backup was deleted and + to remove the prefix. With + one data file of the second backup left out of the copy, the move names + that row and that file, exits 1 and moves none; `--forget-backup` of it + removes the row and prints the store's check; the move then moves the + other two (audit lines), syncs the configured root, which skips the + incomplete prefix naming the missing file, and its report of other + stores is empty. Once the file is copied, sync adopts that backup. A + server on the new root describes and restores a moved backup. Nothing + under the old root changes.""" + d = deployment + old = os.path.realpath(d.root) + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, b1 = _small_backed_up_table(ddb, "move") + b2 = ddb.create_backup(TableName=name, BackupName="two")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, b2, timeout=60) + b3 = ddb.create_backup(TableName=name, BackupName="three")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, b3, timeout=60) + finally: + server.stop() + store_id = _store_id(old) + res = d.run("backup", "sync") + assert res.returncode == 0, res.stdout + res.stderr + assert _section(res.stdout, OTHER_STORES).splitlines()[1:] == [" (none)"], res.stdout + old_tree = _tree(old) + + new = os.path.join(d.cwd, "backups-new") + _copy_root(old, new) + new = os.path.realpath(new) + _set_root(d, new) + res = d.run("backup", "sync") + out = res.stdout + res.stderr + assert res.returncode == 0, out + print(f"branch: {d.backend}\n{res.stdout}") + for arn in (b1, b2, b3): + line = ( + f" {_key_prefix(name, arn)}: a row of another store holds it (AVAILABLE; filesystem {old}, " + f"store id {store_id}); it is left as it is" + ) + assert line in res.stdout, f"{line!r} not in\n{res.stdout}" + others = _section(res.stdout, OTHER_STORES).splitlines()[1:] + assert others[0] == f" filesystem {old} (store id {store_id}): 3 AVAILABLE", others + assert "--move-store " in others[1] and "--forget-store " in others[1], others + assert len(others) == 2, others + assert _rows(d) == {}, "the configured store has no row before the move" + + res = d.run("backup", "sync", "--move-store", new) + out = res.stdout + res.stderr + assert res.returncode == 1 and "that is the configured store's root, so nothing would move" in out, out + res = d.run("backup", "sync", "--move-store", old + "/") + out = res.stdout + res.stderr + assert res.returncode == 1 and "the catalog has no backup row under that root" in out, out + known = _section(res.stdout, "--- The stores the catalog holds backup rows for:").splitlines()[1:] + assert known == [f" filesystem {old} (store id {store_id}): 3 AVAILABLE"], res.stdout + assert "Took the maintenance claim" not in res.stdout, res.stdout + + if d.catalog: + # A tombstone whose copied prefix holds a complete backup (its owner + # long dead: the move's wait passes over it). A FAILED row's text + # offers adopting the prefix; a DELETED row's says the backup was + # deleted and to remove the prefix. + for status, remedy in ( + ("FAILED", "let backup sync adopt the prefix"), + ("DELETED", "The backup was deleted: remove that prefix from the configured root"), + ): + _age_row(d, b3, status) + tree = _tree(new) + res = d.run("backup", "sync", "--move-store", old) + out = res.stdout + res.stderr + assert res.returncode == 1, out + stopping = _section(res.stdout, "--- Stopping the move (no row is moved while any is listed here):") + assert ( + f" {b3} ({status}, account {ACCOUNT}): {status} whose prefix under the configured root holds a " + "verified manifest" + ) in stopping and "--forget-backup" in stopping and remedy in stopping, stopping + assert len(stopping.splitlines()) == 2, stopping + assert "--move-store moved no row: 1 row(s) stop the move" in out, out + assert _tree(new) == tree, "the refused move changed the copy" + _set_status(d, b3, "AVAILABLE") + + # One data file of b2 left out of the copy. + data = os.path.join(_prefix(d, name, b2).replace(old, new, 1), "data") + dropped = sorted(os.listdir(data))[0] + aside = os.path.join(d.cwd, dropped) + os.rename(os.path.join(data, dropped), aside) + res = d.run("backup", "sync", "--move-store", old) + out = res.stdout + res.stderr + assert res.returncode == 1, out + stopping = _section(res.stdout, "--- Stopping the move (no row is moved while any is listed here):") + assert stopping.splitlines()[1:] == [ + f" {b2} (AVAILABLE, account {ACCOUNT}): not a complete copy under the configured root: data file " + f"data/{dropped} is not in the store; copy the prefix again (or remove the row with --forget-backup)" + ], stopping + assert "--move-store moved no row: 1 row(s) stop the move" in out, out + assert "=== extenddb backup sync complete" not in res.stdout, "a stopped move ran the sync" + assert _rows(d) == {} + + res = d.run("backup", "sync", "--forget-backup", b2) + out = res.stdout + res.stderr + assert res.returncode == 0, out + assert f" {b2}: removed: the row (AVAILABLE) is gone, and no object was deleted" in res.stdout, out + assert f"The configured store's check passes (store id {store_id})" in res.stdout, out + assert "removed a backup row outright; no object was deleted" in out, "no audit line" + + res = d.run("backup", "sync", "--move-store", old) + out = res.stdout + res.stderr + assert res.returncode == 0, out + assert ( + "=== extenddb backup sync --move-store complete: 2 row(s) moved, 0 resolved, 0 deletion(s) finished, " + "0 row(s) stopping the move, 0 gone during the move, 0 left under the old root ===" + ) in res.stdout, out + moved = _section(res.stdout, "--- Moved (each row now names the configured store):").splitlines()[1:] + assert sorted(moved) == sorted([f" {b1}", f" {b3}"]), moved + assert out.count("moved a backup row to the configured store; no object was copied or deleted") == 2, out + skipped = _section(res.stdout, "--- Skipped (not adopted):") + assert f" {_key_prefix(name, b2)}: " in skipped and f"data file data/{dropped} is not in the store" in skipped, skipped + assert _section(res.stdout, OTHER_STORES).splitlines()[1:] == [" (none)"], res.stdout + assert _rows(d) == {b1: "AVAILABLE", b3: "AVAILABLE"} + + os.rename(aside, os.path.join(data, dropped)) + res = d.run("backup", "sync") + assert res.returncode == 0, res.stdout + res.stderr + assert _adopted_section(res.stdout).splitlines()[1:-1] == [f" {b2}"], res.stdout + + server, ddb, creds = _started(d, str(tmp_path / "serve-new.log")) + try: + details = ddb.describe_backup(BackupArn=b1)["BackupDescription"]["BackupDetails"] + assert details["BackupStatus"] == "AVAILABLE" and details["BackupName"] == "nightly", details + target = f"{name}-r" + ddb.restore_table_from_backup(TargetTableName=target, BackupArn=b1) + _wait_active(ddb, target) + assert _count(ddb, target) == HAND_BUILT_ITEMS + finally: + server.stop() + assert _tree(old) == old_tree, "a command changed the old root" + + +def _age_row(d: _Deployment, arn: str, status: str) -> None: + """Set the backup's row to `status` with its heartbeat and completion in + 2000: a FAILED row whose owner is long dead and past every staleness + window, a DELETED tombstone past its retention.""" + n = _catalog( + d, + "UPDATE backup_index SET status = ?, heartbeat_at = ?, completed_at = ? " + "WHERE account_id = ? AND backup_id = ?", + (status, "2000-01-01T00:00:00.000Z", "2000-01-01T00:00:00.000Z", ACCOUNT, _backup_id(arn)), + ) + assert n == 1, f"{n} backup_index rows match {arn}" + + +def test_another_roots_rows_are_left_alone_until_forget_store_drops_them(deployment, tmp_path): + """Four backups under root B; the configuration then names a new, empty + root A. (SQLite and PostgreSQL: the rows of B are edited at the catalog + to a FAILED row and a DELETED tombstone dated 2000, past every window, + and a DELETING row.) A server on A runs three reconciler cycles: B's + tree is byte-identical after them and B's rows keep their statuses. + Plain sync on A reports B's rows per status and names the DELETING row. + `--forget-store` refuses the configured root A; `--forget-store B` + removes every row of B outright (no tombstone: `backup list` on B is + empty), deletes nothing under B and logs each removal; a second run + finds no row there. Configured with B again, sync adopts the four + prefixes as AVAILABLE backups.""" + d = deployment + b_root = os.path.realpath(d.root) + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, a1 = _small_backed_up_table(ddb, "forget") + arns = [a1] + for label in ("two", "three", "four"): + arn = ddb.create_backup(TableName=name, BackupName=label)["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, arn, timeout=60) + arns.append(arn) + finally: + server.stop() + store_id = _store_id(b_root) + _, a2, a3, a4 = arns + if d.catalog: + _age_row(d, a2, "FAILED") + _set_status(d, a3, "DELETING") + _age_row(d, a4, "DELETED") + expected = {a1: "AVAILABLE", a2: "FAILED", a3: "DELETING", a4: "DELETED"} + counts = "1 AVAILABLE, 1 DELETING, 1 DELETED, 1 FAILED" + else: + expected = {arn: "AVAILABLE" for arn in arns} + counts = "4 AVAILABLE" + print(f"branch: {d.backend}, the rows of B: {counts}") + assert _rows(d) == expected + b_tree = _tree(b_root) + + # A holds a copy of B's tree (without B's identity file), as before a + # move: a duty of A's server acting on B's rows would find the bytes. + a_root = os.path.join(d.cwd, "backups-a") + os.mkdir(a_root, 0o700) + a_root = os.path.realpath(a_root) + _copy_tree_into(b_root, a_root) + a_tree = _tree_without_identity(a_root) + _set_root(d, a_root) + server, ddb, creds = _started(d, str(tmp_path / "serve-a.log")) + try: + time.sleep(3.5) # three cycles at reconcile_interval_secs = 1 + assert server.proc.poll() is None + finally: + server.stop() + assert _tree(b_root) == b_tree, "a reconciler on another root changed B" + assert _tree_without_identity(a_root) == a_tree, "a reconciler deleted the copy of another root's backups" + _set_root(d, b_root) + assert _rows(d) == expected, "a reconciler on another root changed B's rows" + _set_root(d, a_root) + + res = d.run("backup", "sync") + assert res.returncode == 0, res.stdout + res.stderr + others = _section(res.stdout, OTHER_STORES).splitlines()[1:] + assert others[0] == f" filesystem {b_root} (store id {store_id}): {counts}", others + named = [line for line in others[1:] if line.startswith(" ")] + assert named == ([f" DELETING {a3} (account {ACCOUNT})"] if d.catalog else []), others + + res = d.run("backup", "sync", "--forget-store", a_root) + out = res.stdout + res.stderr + assert res.returncode == 1 and "that is the configured store's root, which --forget-store refuses" in out, out + res = d.run("backup", "sync", "--forget-store", b_root) + out = res.stdout + res.stderr + assert res.returncode == 0, out + assert "=== extenddb backup sync --forget-store complete: 4 row(s) removed, 0 left ===" in res.stdout, out + # Beside servers the FAILED and DELETED removals wait out a deletion a + # server's reconciler may have started before the claim; on SQLite the + # serve lock excludes every server. + waited = "before the first removal of a FAILED or DELETED row" in res.stdout + assert waited == (d.catalog != "" and d.backend != "sqlite"), res.stdout + listed = _section(res.stdout, f"--- --forget-store {b_root} (filesystem): rows removed outright; no object is deleted:") + for arn in arns: + assert f" {arn} (" in listed, listed + assert listed.count(": removed: the row (") == 4, listed + assert out.count("removed a backup row outright; no object was deleted") == 4, out + assert _tree(b_root) == b_tree, "--forget-store changed B" + res = d.run("backup", "sync", "--forget-store", b_root) + out = res.stdout + res.stderr + assert res.returncode == 1 and "the catalog has no backup row under that root" in out, out + + _set_root(d, b_root) + assert _rows(d) == {}, "a tombstone was left" + res = d.run("backup", "sync") + assert res.returncode == 0, res.stdout + res.stderr + assert sorted(_adopted_section(res.stdout).splitlines()[1:-1]) == sorted(f" {arn}" for arn in arns), res.stdout + assert _rows(d) == {arn: "AVAILABLE" for arn in arns} + + +def test_forget_backup_clears_a_store_whose_newest_manifests_are_gone(deployment, tmp_path): + """The state a catalog restored from a dump can be in: of its four + backups, the manifests of the two newest and of the oldest are gone from + the store. The store is unhealthy (the manifest probe of the newest + AVAILABLE row), so plain sync refuses, naming `--forget-backup`, and + `--prune-missing` refuses, with no row changed. `--forget-backup + ` runs on the unhealthy store: it removes that row and prints + the store's check, which now names the next one and the command for it; + `--forget-backup ` removes that one and the check passes (beside + servers, the running server's `/health` turns healthy at its next + cycle). `--prune-missing` then removes the oldest row, and the second + one, whose manifest is there, stays. No object is deleted.""" + d = deployment + root = os.path.realpath(d.root) + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, c1 = _small_backed_up_table(ddb, "heal") + arns = [c1] + for label in ("second", "third", "fourth"): + arn = ddb.create_backup(TableName=name, BackupName=label)["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, arn, timeout=60) + arns.append(arn) + _, c2, c3, c4 = arns + if d.backend == "sqlite": + server.stop() + print(f"branch: {d.backend}, the server {'stopped' if d.backend == 'sqlite' else 'running'}") + store_id = _store_id(root) + for arn in (c1, c3, c4): + os.remove(os.path.join(_prefix(d, name, arn), "extenddb-manifest.json")) + tree = _tree(root) + before = _rows(d) + for args in (("sync",), ("sync", "--prune-missing")): + res = d.run("backup", *args) + out = res.stdout + res.stderr + assert res.returncode == 1 and "is unhealthy" in out and c4 in out, (args, out) + assert f"remove its row with extenddb backup sync --forget-backup {c4}," in out, (args, out) + assert _rows(d) == before + + res = d.run("backup", "sync", "--forget-backup", c4) + out = res.stdout + res.stderr + assert res.returncode == 0, out + assert f"Store: filesystem {root} (unhealthy: " in res.stdout, out + assert f" {c4}: removed: the row (AVAILABLE) is gone, and no object was deleted" in res.stdout, out + assert "The configured store's check fails: " in res.stdout, out + assert f"run extenddb backup sync --forget-backup {c3}" in res.stdout, out + + res = d.run("backup", "sync", "--forget-backup", c3) + out = res.stdout + res.stderr + assert res.returncode == 0, out + assert f"The configured store's check passes (store id {store_id})" in res.stdout, out + if d.backend != "sqlite": + health = _wait_health( + d.port, + lambda h: h.get("components", {}).get("backup_store", {}).get("status") == "healthy", + server=server, + ) + print(f"the running server's store after the removals: {health['components']['backup_store']}") + + res = d.run("backup", "sync", "--prune-missing") + out = res.stdout + res.stderr + assert res.returncode == 0, out + assert f" {c1}: removed: the row is gone, and no object was deleted" in res.stdout, out + assert _rows(d) == {c2: "AVAILABLE"} + assert _tree(root) == tree, "an object was deleted" + finally: + server.stop() + + +def test_forget_backup_leaves_a_live_creating_row_and_an_in_catalog_backup(deployment, tmp_path): + """`--forget-backup` changes nothing for an ARN it cannot act on: (on + PostgreSQL, beside servers) a CREATING row of another root whose owner + beat just now is left with exit 1, and removed once its heartbeat is an + hour old; an ARN spelt with another region than the row's is refused, + naming the row's ARN; with `[backup.store] kind = "disabled"` (no store + configured, which both forget options accept) an earlier release's + in-catalog backup is reported as one with exit 1, the ARN after it in + the same command is not reached and keeps its row, and a second command + removes that row. The in-catalog backup is still AVAILABLE afterwards.""" + d = deployment + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, e1 = _small_backed_up_table(ddb, "refuse") + e2 = ddb.create_backup(TableName=name, BackupName="two")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, e2, timeout=60) + finally: + server.stop() + print(f"branch: {d.backend}") + if d.backend == "postgres": + elsewhere = os.path.join(d.cwd, "elsewhere") + n = _catalog( + d, + "UPDATE backup_index SET store_root = ?, status = 'CREATING', completed_at = NULL, " + "heartbeat_at = {now} WHERE account_id = ? AND backup_id = ?", + (elsewhere, ACCOUNT, _backup_id(e2)), + ) + assert n == 1 + res = d.run("backup", "sync", "--forget-backup", e2) + out = res.stdout + res.stderr + assert res.returncode == 1, out + assert f" {e2}: left: the CREATING row has a live owner" in res.stdout, out + assert f"--forget-backup did not remove the row of {e2}" in out, out + assert _rows_present(d, [e2]) == 1 + n = _catalog( + d, + "UPDATE backup_index SET heartbeat_at = {now} - interval '1 hour' WHERE account_id = ? AND backup_id = ?", + (ACCOUNT, _backup_id(e2)), + ) + assert n == 1 + res = d.run("backup", "sync", "--forget-backup", e2) + out = res.stdout + res.stderr + assert res.returncode == 0, out + assert f" {e2}: removed: the row (CREATING) is gone, and no object was deleted" in res.stdout, out + assert _rows_present(d, [e2]) == 0 + + region = e1.split(":")[3] + respelt = e1.replace(f":{region}:", ":eu-west-3:", 1) + assert respelt != e1 + res = d.run("backup", "sync", "--forget-backup", respelt) + out = res.stdout + res.stderr + assert res.returncode == 1, out + assert f" {respelt}: left: no row carries this ARN as written; the row of this backup is {e1}" in res.stdout, out + assert e1 in _rows(d) + + _set_store_disabled(d) + server = _Server(d.config, str(tmp_path / "serve-disabled.log")) + try: + _wait_health(d.port, lambda h: True, server=server) + ddb = _client(d.port, _provision(d.port, d.cert, ADMIN_PASSWORD)) + legacy = ddb.create_backup(TableName=name, BackupName="in-catalog")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, legacy, timeout=60) + finally: + server.stop() + res = d.run("backup", "sync", "--forget-backup", legacy, "--forget-backup", e1) + out = res.stdout + res.stderr + assert res.returncode == 1, out + assert "Store: none (" in res.stdout, out + assert f" {legacy}: an in-catalog backup; --forget-backup changes nothing; DeleteBackup removes it" in res.stdout, out + assert f" {e1}: not reached" in res.stdout, out + assert "=== extenddb backup sync --forget-backup stopped: 0 row(s) removed, 2 not ===" in res.stdout, out + if d.catalog: + assert _rows_present(d, [e1]) == 1 + res = d.run("backup", "sync", "--forget-backup", e1) + out = res.stdout + res.stderr + assert res.returncode == 0, out + assert f" {e1}: removed: the row (AVAILABLE) is gone, and no object was deleted" in res.stdout, out + if d.catalog: + assert _rows_present(d, [e1]) == 0 + + server = _Server(d.config, str(tmp_path / "serve-disabled2.log")) + try: + _wait_health(d.port, lambda h: True, server=server) + ddb = _client(d.port, _provision(d.port, d.cert, ADMIN_PASSWORD)) + details = ddb.describe_backup(BackupArn=legacy)["BackupDescription"]["BackupDetails"] + assert details["BackupStatus"] == "AVAILABLE", details + finally: + server.stop() + + +def test_a_create_backup_during_the_moves_wait_is_refused(deployment, tmp_path): + """Beside servers (PostgreSQL and MongoDB): a CREATING row of the old + root whose owner beat just now; the tree copied to a new root, a server + running on it. `--move-store ` takes the claim and waits for + that job; while it waits, a CreateBackup on the server is refused with + LimitExceededException (the maintenance claim) and writes no row. Once + the job's row is AVAILABLE the move goes on and moves both rows, and the + server's next CreateBackup and a DescribeBackup of a moved backup + work.""" + _sync_cell(deployment) + d = deployment + if d.backend == "sqlite": + pytest.skip("on SQLite the serve lock keeps every server stopped during the move") + old = os.path.realpath(d.root) + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, m1 = _small_backed_up_table(ddb, "race") + m2 = ddb.create_backup(TableName=name, BackupName="two")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, m2, timeout=60) + finally: + server.stop() + _set_status(d, m2, "CREATING") + new = os.path.join(d.cwd, "backups-new") + _copy_root(old, new) + new = os.path.realpath(new) + _set_root(d, new) + server, ddb, creds = _started(d, str(tmp_path / "serve-new.log")) + out_path = str(tmp_path / "move.out") + def every_row() -> int: + if d.backend == "mongodb": + return _mongosh(f"print(db.backup_index.countDocuments({{account_id: {json.dumps(ACCOUNT)}}}))") + return _catalog(d, "SELECT backup_id FROM backup_index WHERE account_id = ?", (ACCOUNT,)) + + try: + before = every_row() + assert before == 2, before + with open(out_path, "wb") as out: + moving = subprocess.Popen( + [EXTENDDB_BINARY, "backup", "--config", d.config, "sync", "--move-store", old], + cwd=d.cwd, stdout=out, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL, + ) + try: + deadline = time.monotonic() + 60 + while "Waiting for running jobs to finish" not in open(out_path).read(): + assert moving.poll() is None, open(out_path).read() + assert time.monotonic() < deadline, open(out_path).read() + time.sleep(0.2) + assert _backup_id(m2) in open(out_path).read(), open(out_path).read() + with pytest.raises(ClientError) as refused: + ddb.create_backup(TableName=name, BackupName="during") + error = refused.value.response["Error"] + print(f"CreateBackup during the wait: {error}") + assert error["Code"] == "LimitExceededException", error + assert "maintenance claim" in error["Message"], error + assert every_row() == before, "the refused CreateBackup wrote a row" + _set_status(d, m2, "AVAILABLE") + assert moving.wait(timeout=60) == 0, open(out_path).read() + finally: + if moving.poll() is None: + moving.kill() + moving.wait(timeout=10) + text = open(out_path).read() + assert "=== extenddb backup sync --move-store complete: 2 row(s) moved" in text, text + assert _rows(d) == {m1: "AVAILABLE", m2: "AVAILABLE"} + after = ddb.create_backup(TableName=name, BackupName="after")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, after, timeout=60) + details = ddb.describe_backup(BackupArn=m1)["BackupDescription"]["BackupDetails"] + assert details["BackupStatus"] == "AVAILABLE", details + finally: + server.stop() + + +def test_a_row_that_lands_during_the_moves_sync_exits_1(deployment, tmp_path): + """Beside servers (PostgreSQL): server A stays on the old root, server B + runs on a new root holding a copy of the old tree and 2,000 junk + prefixes (so the move's sync lists for a while). A CreateBackup on A + passes its claim reads and is held in its CREATING insert by a trigger; + `--move-store ` then moves the old row, and the trigger lets + the insert go once the move prints that it syncs the configured store. + The job's row lands under the old root during that sync: the closing + report names it under "Rows that landed under the old root during the + move" and the command exits 1, naming the remedy.""" + d = deployment + if d.backend == "sqlite": + pytest.skip("on SQLite the serve lock keeps every server stopped during the move") + if d.backend == "mongodb": + pytest.skip( + "on MongoDB this cell's catalog trigger is SQL, and the deployment's catalog is not " + "an SQL database" + ) + old = os.path.realpath(d.root) + config_a = os.path.join(d.cwd, "extenddb-a.toml") + shutil.copy(d.config, config_a) + a = _Server(config_a, str(tmp_path / "serve-a.log")) + b = None + held = False + try: + _wait_health( + d.port, + lambda h: h.get("components", {}).get("backup_store", {}).get("status") == "healthy", + server=a, + ) + creds = _provision(d.port, d.cert, ADMIN_PASSWORD) + ddb_a = _client(d.port, creds) + name, moved = _small_backed_up_table(ddb_a, "landed") + new = os.path.join(d.cwd, "backups-new") + os.mkdir(new, 0o700) + new = os.path.realpath(new) + _set_root(d, new) + port_b = lh._find_free_port() + lh._patch_config_port(d.config, port_b) + b = _Server(d.config, str(tmp_path / "serve-b.log")) + _wait_health( + port_b, + lambda h: h.get("components", {}).get("backup_store", {}).get("status") == "healthy", + server=b, + ) + _copy_tree_into(old, new) + for i in range(2000): + _private_file( + os.path.join(new, ACCOUNT, "junk", f"0179{i:010d}-0000abcd", "data", "000001.json.gz"), + b"x", + ) + _catalog(d, "CREATE TABLE landed_release (x int)", ()) + _catalog( + d, + "CREATE OR REPLACE FUNCTION landed_hold() RETURNS trigger AS $$ BEGIN " + "WHILE NOT EXISTS (SELECT 1 FROM landed_release) LOOP PERFORM pg_sleep(0.05); END LOOP; " + "RETURN NEW; END $$ LANGUAGE plpgsql", + (), + ) + _catalog( + d, + "CREATE TRIGGER landed_hold BEFORE INSERT ON backup_index FOR EACH ROW " + "WHEN (NEW.status = 'CREATING') EXECUTE FUNCTION landed_hold()", + (), + ) + held = True + result: dict = {} + + def create() -> None: + try: + result["arn"] = _client(d.port, creds).create_backup( + TableName=name, BackupName="late" + )["BackupDetails"]["BackupArn"] + except Exception as e: # noqa: BLE001 - reported by the assertion below + result["error"] = repr(e) + + creating = threading.Thread(target=create) + creating.start() + deadline = time.monotonic() + 30 + while _catalog( + d, + "SELECT pid FROM pg_stat_activity WHERE datname = current_database() " + "AND wait_event = 'PgSleep' AND query LIKE ?", + ("%backup_index%",), + ) == 0: + assert time.monotonic() < deadline, "the CreateBackup's insert is not held" + time.sleep(0.1) + out_path = str(tmp_path / "move.out") + with open(out_path, "wb") as out: + moving = subprocess.Popen( + [EXTENDDB_BINARY, "backup", "--config", d.config, "sync", "--move-store", old], + cwd=d.cwd, stdout=out, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL, + ) + released = False + try: + deadline = time.monotonic() + 120 + while moving.poll() is None: + assert time.monotonic() < deadline, open(out_path).read()[-4000:] + if not released and "syncing it." in open(out_path).read(): + _catalog(d, "INSERT INTO landed_release VALUES (1)", ()) + released = True + time.sleep(0.02) + rc = moving.wait(timeout=60) + finally: + if moving.poll() is None: + moving.kill() + moving.wait(timeout=10) + if not released: + _catalog(d, "INSERT INTO landed_release VALUES (1)", ()) + creating.join(timeout=60) + text = open(out_path).read() + late = result.get("arn") + assert late is not None, result + assert released, "the move's sync line was printed before it ended" + assert f"Moved (each row now names the configured store):\n {moved}" in text, text[-4000:] + assert "--- Rows that landed under the old root during the move:\n " + late in text, text[-4000:] + assert rc == 1, text[-4000:] + assert "a row landed under the old root during the move (1 listed above)" in text, text[-4000:] + finally: + if held: + _catalog(d, "DROP TRIGGER IF EXISTS landed_hold ON backup_index", ()) + _catalog(d, "DROP FUNCTION IF EXISTS landed_hold()", ()) + _catalog(d, "DROP TABLE IF EXISTS landed_release", ()) + a.stop() + if b is not None: + b.stop() + + +def test_the_manuals_move_procedure_beside_a_running_server(deployment, tmp_path): + """The backup manual's "Moving a store to a new root" in its order, + beside a running server (PostgreSQL, MongoDB): one backup under the old + root; the configuration pointed at a new, empty root and the server + restarted (step 1), which gives the new root its own identity file; a + CreateBackup there before the copy (the server keeps serving); the old + tree copied in without its `extenddb-store.json` (step 2); then + `--move-store` (step 3), which moves the old row and keeps the new one. + The running server's store stays healthy throughout, it restores the + moved backup, and its next CreateBackup is AVAILABLE.""" + d = deployment + if d.backend == "sqlite": + pytest.skip("on SQLite the serve lock keeps every server stopped during the move") + old = os.path.realpath(d.root) + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, b1 = _small_backed_up_table(ddb, "proc") + finally: + server.stop() + old_tree = _tree(old) + new = os.path.join(d.cwd, "backups-new") + os.mkdir(new, 0o700) + new = os.path.realpath(new) + _set_root(d, new) + healthy = lambda h: h.get("components", {}).get("backup_store", {}).get("status") == "healthy" # noqa: E731 + server, ddb, creds = _started(d, str(tmp_path / "serve-new.log")) + try: + _wait_health(d.port, healthy, server=server) + new_id = _store_id(new) + assert new_id != _store_id(old) + b2 = ddb.create_backup(TableName=name, BackupName="between")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, b2, timeout=60) + _copy_tree_into(old, new) + assert _store_id(new) == new_id, "the copy brought the old identity file" + res = d.run("backup", "sync", "--move-store", old) + out = res.stdout + res.stderr + assert res.returncode == 0, out + assert ( + "=== extenddb backup sync --move-store complete: 1 row(s) moved, 0 resolved, 0 deletion(s) " + "finished, 0 row(s) stopping the move, 0 gone during the move, 0 left under the old root ===" + ) in res.stdout, out + time.sleep(2) # two cycles' checks of the running server + _wait_health(d.port, healthy, server=server) + target = f"{name}-r" + ddb.restore_table_from_backup(TargetTableName=target, BackupArn=b1) + _wait_active(ddb, target) + assert _count(ddb, target) == HAND_BUILT_ITEMS + b3 = ddb.create_backup(TableName=name, BackupName="after")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, b3, timeout=60) + finally: + server.stop() + assert _rows(d) == {b1: "AVAILABLE", b2: "AVAILABLE", b3: "AVAILABLE"} + res = d.run("backup", "list") + assert res.stdout.count(f"store_id={new_id}") == 3, res.stdout + assert _tree(old) == old_tree, "the move changed the old root" + + +def test_a_copied_identity_file_is_undone_by_the_manuals_remedy(deployment, tmp_path): + """The manual's way out when `extenddb-store.json` was copied with the + tree over the new root's own (PostgreSQL, MongoDB): one backup under the + old root, the server restarted on the new root and a CreateBackup there, + then the whole tree copied, identity file included. The running server's + store turns unhealthy (an identity mismatch) and `--move-store` refuses + (the copied file's id against the row the new root's backup carries). + `extenddb backup list` names that backup under the new root's first id; + `--forget-backup` of it removes the row and deletes nothing; the move + then moves the old row and its sync adopts the forgotten backup again + under the copied file's id. After a restart every server's store is + healthy, both backups restore or describe, and a new CreateBackup is + AVAILABLE.""" + d = deployment + if d.backend == "sqlite": + pytest.skip("on SQLite the serve lock keeps every server stopped during the move") + old = os.path.realpath(d.root) + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, b1 = _small_backed_up_table(ddb, "copied") + finally: + server.stop() + old_id = _store_id(old) + new = os.path.join(d.cwd, "backups-new") + os.mkdir(new, 0o700) + new = os.path.realpath(new) + _set_root(d, new) + status = lambda h: h.get("components", {}).get("backup_store", {}).get("status") # noqa: E731 + server, ddb, creds = _started(d, str(tmp_path / "serve-new.log")) + try: + _wait_health(d.port, lambda h: status(h) == "healthy", server=server) + first_id = _store_id(new) + b2 = ddb.create_backup(TableName=name, BackupName="between")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, b2, timeout=60) + _copy_tree_into(old, new, identity=True) + assert _store_id(new) == old_id + health = _wait_health(d.port, lambda h: status(h) == "unhealthy", server=server) + print(f"the running server's store after the copy: {health['components']['backup_store']}") + res = d.run("backup", "sync", "--move-store", old) + out = res.stdout + res.stderr + assert res.returncode == 1 and "is unhealthy" in out and first_id in out, out + res = d.run("backup", "list") + assert f"{b2} store_root={new} store_id={first_id}" in res.stdout, res.stdout + tree = _tree(new) + res = d.run("backup", "sync", "--forget-backup", b2) + out = res.stdout + res.stderr + assert res.returncode == 0, out + assert f" {b2}: removed: the row (AVAILABLE) is gone, and no object was deleted" in res.stdout, out + assert f"The configured store's check passes (store id {old_id})" in res.stdout, out + assert _tree(new) == tree, "--forget-backup changed the root" + res = d.run("backup", "sync", "--move-store", old) + out = res.stdout + res.stderr + assert res.returncode == 0, out + assert "=== extenddb backup sync --move-store complete: 1 row(s) moved" in res.stdout, out + assert _adopted_section(res.stdout).splitlines()[1:-1] == [f" {b2}"], res.stdout + finally: + server.stop() + server, ddb, creds = _started(d, str(tmp_path / "serve-restarted.log")) + try: + _wait_health(d.port, lambda h: status(h) == "healthy", server=server) + target = f"{name}-r" + ddb.restore_table_from_backup(TargetTableName=target, BackupArn=b1) + _wait_active(ddb, target) + assert _count(ddb, target) == HAND_BUILT_ITEMS + details = ddb.describe_backup(BackupArn=b2)["BackupDescription"]["BackupDetails"] + assert details["BackupStatus"] == "AVAILABLE", details + b3 = ddb.create_backup(TableName=name, BackupName="after")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, b3, timeout=60) + finally: + server.stop() + res = d.run("backup", "list") + assert res.stdout.count(f"store_id={old_id}") == 3, res.stdout + + +def test_forget_backup_checks_its_arns_first_goes_past_removed_ones_and_writes_no_store(deployment, tmp_path): + """`--forget-backup` refuses a string that is no ARN before it takes the + claim, with nothing changed; given again after a run that removed its + first ARN, it says that ARN carries no row and removes the next, exit 0. + With the root replaced by an empty directory (a volume not mounted under + the rows), it refuses the root's last row, exit 1, naming the unmounted + volume, and writes nothing to the root; run again, the same, where a + removal would have left a root with no rows that a server's check gives + a new identity file.""" + d = deployment + root = os.path.realpath(d.root) + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, a = _small_backed_up_table(ddb, "again") + arns = [a] + for label in ("bbb", "ccc"): + arn = ddb.create_backup(TableName=name, BackupName=label)["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, arn, timeout=60) + arns.append(arn) + finally: + server.stop() + _, b, c = arns + + for given in ("", "not-an-arn"): + res = d.run("backup", "sync", "--forget-backup", given, "--forget-backup", a) + out = res.stdout + res.stderr + assert res.returncode == 1, out + assert f" {given!r}: not a backup ARN" in res.stdout.replace('"', "'"), out + assert "nothing was changed" in out, out + assert "Took the maintenance claim" not in res.stdout, out + assert set(_rows(d)) == set(arns) + + res = d.run("backup", "sync", "--forget-backup", a) + assert res.returncode == 0, res.stdout + res.stderr + res = d.run("backup", "sync", "--forget-backup", a, "--forget-backup", b) + out = res.stdout + res.stderr + assert res.returncode == 0, out + assert f" {a}: no row carries this ARN; nothing to remove" in res.stdout, out + assert f" {b}: removed: the row (AVAILABLE) is gone, and no object was deleted" in res.stdout, out + assert "=== extenddb backup sync --forget-backup complete: 1 row(s) removed, 0 not ===" in res.stdout, out + assert set(_rows(d)) == {c} + + aside = os.path.join(d.cwd, "aside") + os.rename(root, aside) + os.mkdir(root, 0o700) + res = d.run("backup", "sync") + out = res.stdout + res.stderr + assert res.returncode == 1 and "has no extenddb-store.json but the catalog has rows for it" in out, out + for run in ("first", "again"): + res = d.run("backup", "sync", "--forget-backup", c) + out = res.stdout + res.stderr + assert res.returncode == 1, f"{run}: {out}" + assert ( + f" {c}: left: the configured root has no extenddb-store.json while the catalog holds rows for " + "it (a volume that is not mounted?); mount it, or point [backup.store] at the root that holds " + "these backups, then run again" + ) in res.stdout, f"{run}: {out}" + assert ( + f"The configured store's check was not run: the root {root} had no extenddb-store.json when the " + "command opened it" + ) in res.stdout, f"{run}: {out}" + assert os.listdir(root) == [], f"{run}: the forget wrote to the empty root" + assert set(_rows(d)) == {c}, run + + +@pytest.mark.parametrize("shape", ["renamed away", "dangling link"]) +def test_forget_refuses_the_rows_of_a_missing_default_root(deployment, tmp_path, shape): + """SQLite's defaulted root missing while the catalog holds rows for it: + renamed away, or a symbolic link to a volume directory that is gone + (the rows carry the link's target). `--forget-backup` refuses the root's + row as an unmounted root's, exit 1, and says the root is missing with + rows rather than that no server has run with the database file; + `--forget-store` of the root refuses it as the configured root. Nothing + is created at the root's path, and the row is kept.""" + d = deployment + if d.backend != "sqlite": + pytest.skip("the defaulted root is SQLite's") + root = d.root + if shape == "dangling link": + volume = os.path.join(d.cwd, "volume") + os.mkdir(volume, 0o700) + label = os.path.join(volume, "backups") + os.mkdir(label, 0o700) + os.symlink(label, root) + else: + label = root + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + _, arn = _small_backed_up_table(ddb, "gone") + finally: + server.stop() + if shape == "dangling link": + os.rename(volume, os.path.join(d.cwd, "volume-gone")) + assert os.path.islink(root) and not os.path.exists(root) + else: + os.rename(root, os.path.join(d.cwd, "aside")) + res = d.run("backup", "sync", "--forget-backup", arn) + out = res.stdout + res.stderr + assert res.returncode == 1, out + assert f"the default root {label} is missing while the catalog holds rows for it" in res.stdout, out + assert "no server has run with this database file" not in out, out + assert ( + f" {arn}: left: the configured root has no extenddb-store.json while the catalog holds rows for it" + ) in res.stdout, out + res = d.run("backup", "sync", "--forget-store", label) + out = res.stdout + res.stderr + assert res.returncode == 1 and "that is the configured store's root" in out, out + assert not os.path.exists(root), "the forget created the default root" + assert _rows_present(d, [arn]) == 1, "the row is kept" + + +def test_the_forget_options_write_nothing_to_a_fresh_root(deployment, tmp_path): + """A configured root with no identity file and no rows (a fresh root): + `--forget-backup` of an ARN no row carries, and `--forget-store` of a + root with no rows, open the store without writing to it; the root is + still empty afterwards, where the writing check of plain sync would + have created its identity file.""" + d = deployment + fresh = os.path.join(d.cwd, "fresh-root") + os.mkdir(fresh, 0o700) + fresh = os.path.realpath(fresh) + _set_root(d, fresh) + absent = f"arn:aws:dynamodb:us-east-1:{ACCOUNT}:table/x/backup/01700000000000-deadbeef" + res = d.run("backup", "sync", "--forget-backup", absent) + out = res.stdout + res.stderr + assert res.returncode == 0, out + assert f" {absent}: no row carries this ARN; nothing to remove" in res.stdout, out + assert ( + f"The configured store's check was not run: the root {fresh} had no extenddb-store.json when the " + "command opened it" + ) in res.stdout, out + assert os.listdir(fresh) == [], "the forget wrote to the fresh root" + res = d.run("backup", "sync", "--forget-store", os.path.join(d.cwd, "nowhere")) + out = res.stdout + res.stderr + assert res.returncode == 1 and "the catalog has no backup row under that root" in out, out + assert os.listdir(fresh) == [], "the forget wrote to the fresh root" + + +def _store_status(health: dict) -> "str | None": + """The backup store component's status in a /health answer.""" + return health.get("components", {}).get("backup_store", {}).get("status") + + +def test_forget_backup_beside_a_server_keeps_the_rows_of_an_unmounted_root(deployment, tmp_path): + """Beside a running server: the root replaced by an empty directory (a + volume not mounted under the rows), the server reports the identity file + missing. `--forget-backup` of the root's last row refuses it, exit 1, + and writes nothing; the server's checks then find the rows still there, + give the root no identity file and keep the store unhealthy, so no + CreateBackup lands on the empty mount point.""" + d = deployment + if d.backend == "sqlite": + pytest.skip("beside a server") + root = os.path.realpath(d.root) + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, a = _small_backed_up_table(ddb, "unm") + os.rename(root, os.path.join(d.cwd, "aside")) + os.mkdir(root, 0o700) + _wait_health(d.port, lambda h: _store_status(h) == "unhealthy", server=server) + res = d.run("backup", "sync", "--forget-backup", a) + out = res.stdout + res.stderr + assert res.returncode == 1, out + assert f" {a}: left: the configured root has no extenddb-store.json" in res.stdout, out + assert os.listdir(root) == [], out + time.sleep(4) + assert os.listdir(root) == [], "a server's check wrote to the empty mount point" + assert _store_status(_health(d.port)) == "unhealthy" + assert set(_rows(d)) == {a} + finally: + server.stop() + + +def test_forget_backup_finishes_a_deleting_row_of_the_configured_store(deployment, tmp_path): + """A DELETING row of the configured store (its DeleteBackup's state + before the prefix is gone), no server running: `--forget-backup` deletes + the prefix first, says so, and removes the row; the next plain sync + adopts nothing. On SQLite the command's own pass over what the stopped + server left may finish the deletion first, and the line says so too.""" + d = deployment + if not d.catalog: + pytest.skip("edits the catalog") + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, a = _small_backed_up_table(ddb, "deling") + finally: + server.stop() + _set_status(d, a, "DELETING") + prefix = _prefix(d, name, a) + assert os.listdir(prefix) + res = d.run("backup", "sync", "--forget-backup", a) + out = res.stdout + res.stderr + assert res.returncode == 0, out + assert ( + f" {a}: removed: the row is gone, and this command deleted its objects first (the row was " + "DELETING in the configured store: the deletion its DeleteBackup asked for)" + ) in res.stdout, out + assert "no object was deleted" not in res.stdout, out + assert not os.path.isdir(prefix) or os.listdir(prefix) == [], os.listdir(prefix) + assert a not in _rows(d) + res = d.run("backup", "sync") + out = res.stdout + res.stderr + assert res.returncode == 0, out + assert "=== extenddb backup sync complete: 0 backup(s) adopted" in res.stdout, out + assert a not in _rows(d) + + +def test_a_maintenance_command_waits_out_a_deleting_row_and_a_dead_restore(deployment, tmp_path): + """Beside a server (PostgreSQL): a DELETING row and a RUNNING restore + record whose owner is long dead (its target CREATING, both edited at the + catalog with the server stopped). `--forget-backup` of the restored + backup takes the claim and waits for the restore; a server started + while it waits runs its held-claim cycle, which finishes the DELETING + row's deletion and fails the dead restore and removes its target, and + the command then removes the backup's row (no longer in use), exit 0.""" + d = deployment + if d.backend != "postgres": + pytest.skip("beside servers, the restore record edited at the PostgreSQL catalog") + root = os.path.realpath(d.root) + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, b1 = _small_backed_up_table(ddb, "dead") + b2 = ddb.create_backup(TableName=name, BackupName="two")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, b2, timeout=60) + target = f"{name}-r" + ddb.restore_table_from_backup(TargetTableName=target, BackupArn=b1) + _wait_active(ddb, target) + finally: + server.stop() + n = _catalog( + d, + "UPDATE tables SET table_status = 'CREATING' WHERE account_id = ? AND table_name = ?", + (ACCOUNT, target), + ) + assert n == 1 + n = _catalog( + d, + "UPDATE table_restores SET status = 'RUNNING', file_index = NULL, items_applied = NULL, " + "failure_reason = NULL, completed_at = NULL, heartbeat_at = '1970-01-01T00:00:00Z' " + "WHERE table_id = (SELECT table_id FROM tables WHERE account_id = ? AND table_name = ?)", + (ACCOUNT, target), + ) + assert n == 1 + _set_status(d, b2, "DELETING") + out_path = str(tmp_path / "forget.out") + with open(out_path, "wb") as out: + forgetting = subprocess.Popen( + [EXTENDDB_BINARY, "backup", "--config", d.config, "sync", "--forget-backup", b1], + cwd=d.cwd, stdout=out, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL, + ) + server = None + try: + deadline = time.monotonic() + 60 + while "Waiting for running jobs to finish" not in open(out_path).read(): + assert forgetting.poll() is None, open(out_path).read() + assert time.monotonic() < deadline, open(out_path).read() + time.sleep(0.2) + assert target in open(out_path).read(), open(out_path).read() + server, ddb, creds = _started(d, str(tmp_path / "serve-held.log")) + assert forgetting.wait(timeout=120) == 0, open(out_path).read() + text = open(out_path).read() + assert f" {b1}: removed: the row (AVAILABLE) is gone, and no object was deleted" in text, text + with pytest.raises(ClientError) as gone: + ddb.describe_table(TableName=target) + assert gone.value.response["Error"]["Code"] == "ResourceNotFoundException", gone.value.response + log = server.log_text() + assert "a maintenance claim is held" in log, log[-4000:] + finally: + if forgetting.poll() is None: + forgetting.kill() + forgetting.wait(timeout=10) + if server: + server.stop() + assert _rows(d) == {b2: "DELETED"} + assert not os.path.exists(_prefix(d, name, b2)), "the DELETING row's deletion was not finished" + assert os.path.isdir(root) diff --git a/tests/test_backup_store_config.py b/tests/test_backup_store_config.py index 1924addd3..25783f20a 100644 --- a/tests/test_backup_store_config.py +++ b/tests/test_backup_store_config.py @@ -1596,6 +1596,16 @@ def test_backup_list_and_purge_on_an_empty_store_and_the_serve_lock(self, sqlite # it: exit 6, the code `migrate` uses for a locked catalog. assert res.returncode == 6, res.stdout + res.stderr assert "another extenddb process" in (res.stdout + res.stderr), res.stdout + res.stderr + # So is every option of `backup sync` that moves or removes rows, + # with or without a store to read. + for args in ( + ("sync", "--move-store", "/srv/elsewhere"), + ("sync", "--forget-store", "/srv/elsewhere"), + ("sync", "--forget-backup", "arn:aws:dynamodb:us-east-1:123456789012:table/t/backup/01700000000000-00000000"), + ): + res = d.run("backup", *args) + assert res.returncode == 6, (args, res.stdout + res.stderr) + assert "another extenddb process" in (res.stdout + res.stderr), (args, res.stdout + res.stderr) assert server.proc.poll() is None, "the server kept running" finally: server.stop()