fix(storage-postgres): store attribute names containing U+0000 in the catalog

.github/workflows/integration.yml

@@ -876,6 +876,7 @@ jobs:
           cargo test --release -p extenddb-storage-postgres
           --test migrations_fresh --test migrations_released --test migrations_crash
           --test migrations_golden --test migrations_guards --test migrations_runs
+          --test migrations_name_escape
 
       # The daemonized server logs to syslog; dump it so server-side failures
       # are diagnosable from the job log.

CONTRIBUTING.md

@@ -359,28 +359,29 @@ SQLite, PostgreSQL, Cassandra and MongoDB migrate through this framework.
 
 ### Example: a nullable column on `tables`
 
-Migration 10, `table_note` (10 being the next free id at the time of writing;
-migrations 6, 7, 8 and 9 are `backup_catalog_index`, `stream_pending`,
-`restore_progress` and `warm_throughput`), adds a nullable `note TEXT` column
-to the catalog's `tables` table on SQLite and PostgreSQL.
+Migration 11, `table_note` (11 being the next free id at the time of writing;
+migrations 6, 7, 8, 9 and 10 are `backup_catalog_index`, `stream_pending`,
+`restore_progress`, `warm_throughput` and `attribute_name_escape`), adds a
+nullable `note TEXT` column to the catalog's `tables` table on SQLite and
+PostgreSQL.
 
 In `crates/storage/src/migrations/registry.rs`, at the end of the
 `migration_ids!` list:
 
 ```rust
     /// Adds a nullable `note` column to the `tables` catalog table.
-    TableNote = 10, "table_note",
+    TableNote = 11, "table_note",
         "add a nullable note column to the tables catalog";
 ```
 
 PostgreSQL has `ADD COLUMN IF NOT EXISTS`, so its body is a file,
-`crates/storage-postgres/migrations/0010_table_note.sql`:
+`crates/storage-postgres/migrations/0011_table_note.sql`:
 
 ```sql
 -- Copyright 2026 ExtendDB contributors
 -- SPDX-License-Identifier: Apache-2.0
 --
--- Migration 10, table_note, PostgreSQL catalog database: a nullable note
+-- Migration 11, table_note, PostgreSQL catalog database: a nullable note
 -- column on tables. IF NOT EXISTS makes a second run change nothing.
 
 ALTER TABLE tables ADD COLUMN IF NOT EXISTS note TEXT;
@@ -395,7 +396,7 @@ In `crates/storage-postgres/src/migrations/mod.rs`, the arm in `migration()`:
             bodies: vec![file_body!(
                 StoreKind::Catalog,
                 "migrations",
-                "0010_table_note.sql"
+                "0011_table_note.sql"
             )],
         },
 ```
@@ -411,19 +412,20 @@ and the id in the transactional arm of `runs_in()`:
         | MigrationId::StreamPending
         | MigrationId::RestoreProgress
         | MigrationId::WarmThroughput
+        | MigrationId::AttributeNameEscape
         | MigrationId::TableNote => RunsIn::Transaction,
 ```
 
 SQLite has no `ADD COLUMN IF NOT EXISTS`, and a file with a plain
 `ADD COLUMN` fails when it runs a second time, so the SQLite body is code
 that checks for the column first.
-`crates/storage-sqlite/src/migrations/m0010_table_note.rs`:
+`crates/storage-sqlite/src/migrations/m0011_table_note.rs`:
 
 ```rust
 // Copyright 2026 ExtendDB contributors
 // SPDX-License-Identifier: Apache-2.0
 
-//! Migration 10, `table_note`, SQLite: a nullable `note` column on `tables`.
+//! Migration 11, `table_note`, SQLite: a nullable `note` column on `tables`.
 //!
 //! SQLite has no `ADD COLUMN IF NOT EXISTS`, so the body reads the table's
 //! columns first and adds the column only when it is missing. A second run
@@ -435,10 +437,10 @@ use extenddb_storage::migrations::{CodeBody, Done, MigrationError};
 use super::SqliteStores;
 
 /// This file, whose SHA-256 the checksum test pins.
-pub(crate) const SOURCE: &str = include_str!("m0010_table_note.rs");
+pub(crate) const SOURCE: &str = include_str!("m0011_table_note.rs");
 
 /// Recorded as this migration's checksum. Changes only with a new migration.
-const FINGERPRINT: &str = "0010_table_note/sqlite/1";
+const FINGERPRINT: &str = "0011_table_note/sqlite/1";
 
 pub(crate) struct TableNote;
 
@@ -477,7 +479,7 @@ beside the others:
 ```rust
 #[rustfmt::skip]
 #[allow(clippy::all, deprecated)]
-mod m0010_table_note;
+mod m0011_table_note;
 ```
 
 and the arm in `migration()`:
@@ -488,8 +490,8 @@ and the arm in `migration()`:
             convergent: true,
             bodies: vec![StoreBody::code(
                 StoreKind::Catalog,
-                &m0010_table_note::TableNote,
-                m0010_table_note::SOURCE,
+                &m0011_table_note::TableNote,
+                m0011_table_note::SOURCE,
             )],
         },
 ```
@@ -501,8 +503,8 @@ rows of `tables`:
 
 ```rust
 #[tokio::test]
-async fn migration_10_adds_the_note_column_to_tables() {
-    let db = temp_db("m10");
+async fn migration_11_adds_the_note_column_to_tables() {
+    let db = temp_db("m11");
     load_dump(&db, DEV_0_1_14).await;
     run(&db).await.expect("migrate");
     let mut text = dump(
@@ -512,12 +514,12 @@ async fn migration_10_adds_the_note_column_to_tables() {
     )
     .await;
     text.push_str(&dump(&db, "SELECT table_name, note FROM tables ORDER BY table_name").await);
-    golden("0010_table_note", &text);
+    golden("0011_table_note", &text);
 }
 ```
 
-`EXTENDDB_BLESS_EXPECTED=1 cargo test -p extenddb-storage-sqlite --lib migration_10`
-writes `crates/storage-sqlite/tests/fixtures/golden/0010_table_note.txt`, one
+`EXTENDDB_BLESS_EXPECTED=1 cargo test -p extenddb-storage-sqlite --lib migration_11`
+writes `crates/storage-sqlite/tests/fixtures/golden/0011_table_note.txt`, one
 tab-separated line per row under each query:
 
 ```
@@ -532,15 +534,15 @@ The PostgreSQL body is a file, so it has no golden test.
 The example declares `Kind::Breaking`. A nullable column can be additive by
 the rule in step 4, but declaring it additive needs the old-binary job, which
 this example does not add. As a breaking migration it makes a 1.x binary
-without migration 10 refuse a catalog that has it, and a rollback restores the
+without migration 11 refuse a catalog that has it, and a rollback restores the
 backup.
 
 The same PR changes the code that reads and writes `tables.note` in each
 backend's catalog store, and commits the `docs/schema/sqlite.txt` and
 `docs/schema/postgres.txt` that the integration jobs regenerate. It changes
 no per-table DDL, because `tables` is a catalog table, and adds no `RELEASED`
-entry. `extenddb migrate --list` then shows migration 10, and on a catalog at
-migration 9 `extenddb migrate` lists migration 10 as pending and exits 3.
+entry. `extenddb migrate --list` then shows migration 11, and on a catalog at
+migration 10 `extenddb migrate` lists migration 11 as pending and exits 3.
 
 ### Running the migration tests
 

crates/storage-cassandra/src/migrations/mod.rs

@@ -168,6 +168,11 @@ pub fn migration(id: MigrationId) -> Migration<CassandraStores> {
                 "0009_warm_throughput.cql"
             )],
         },
+        MigrationId::AttributeNameEscape => Migration::NotApplicable(
+            "Cassandra stores the documents that carry attribute names as JSON text in text \
+             columns, where U+0000 is the six-character escape \\u0000, so every name reads back \
+             as it was written",
+        ),
     }
 }
 

crates/storage-mongodb/src/migrations/mod.rs

@@ -173,6 +173,10 @@ pub fn migration(id: MigrationId) -> Migration<MongoStores> {
             "the catalog collections have no validator, so a table or index document carries \
              a warm_throughput field without a schema change",
         ),
+        MigrationId::AttributeNameEscape => Migration::NotApplicable(
+            "MongoDB stores the catalog's attribute names as BSON string values, which carry their \
+             length and so can hold U+0000, and every name reads back as it was written",
+        ),
     }
 }
 

crates/storage-mongodb/tests/migrations_fresh.rs

@@ -325,6 +325,7 @@ async fn a_fresh_store_gets_every_migration_and_the_expected_definitions() {
             (7, d.target.catalog.clone()),
             (8, d.target.catalog.clone()),
             (9, d.target.catalog.clone()),
+            (10, d.target.catalog.clone()),
         ]
     );
     let not_applicable: Vec<u32> = report
@@ -333,7 +334,7 @@ async fn a_fresh_store_gets_every_migration_and_the_expected_definitions() {
         .filter(|s| s.outcome == Outcome::NotApplicable)
         .map(|s| s.id)
         .collect();
-    assert_eq!(not_applicable, [3, 4, 7, 8, 9], "{report:?}");
+    assert_eq!(not_applicable, [3, 4, 7, 8, 9, 10], "{report:?}");
     assert_eq!(
         d.status().await,
         CatalogStatus::Current {
@@ -354,6 +355,7 @@ async fn a_fresh_store_gets_every_migration_and_the_expected_definitions() {
             (7, Outcome::NotApplicable),
             (8, Outcome::NotApplicable),
             (9, Outcome::NotApplicable),
+            (10, Outcome::NotApplicable),
         ]
     );
     assert!(
@@ -449,7 +451,7 @@ async fn a_second_run_applies_nothing_and_reconverge_keeps_the_rows() {
     let reconverged = d.reconverge().await.expect("reconverge");
     // Migration 1's steps in the catalog and the data database, and
     // migrations 5's and 6's in the catalog: 2 is not convergent, and 3, 4,
-    // 7, 8 and 9 do not apply on MongoDB.
+    // 7, 8, 9 and 10 do not apply on MongoDB.
     let ran: Vec<(u32, String)> = reconverged
         .applied
         .iter()
@@ -540,24 +542,24 @@ async fn a_changed_checksum_a_newer_catalog_and_an_older_release_are_reported()
         "notes": "", "applied_at": bson::DateTime::now(), "applied_by": "9.9.9" }
     };
     ledger
-        .insert_one(row(10, "breaking"))
+        .insert_one(row(11, "breaking"))
         .await
         .expect("insert");
     let status = d.status().await;
     assert_eq!(status.label(), "AheadIncompatible");
     assert_eq!(status.exit_code(), exit::AHEAD_INCOMPATIBLE);
     ledger
-        .delete_one(doc! { "_id": 10_i64 })
+        .delete_one(doc! { "_id": 11_i64 })
         .await
         .expect("delete");
     ledger
-        .insert_one(row(10, "additive"))
+        .insert_one(row(11, "additive"))
         .await
         .expect("insert");
     assert_eq!(d.status().await.label(), "AheadCompatible");
     assert!(d.migrate().await.expect("migrate").applied.is_empty());
     ledger
-        .delete_one(doc! { "_id": 10_i64 })
+        .delete_one(doc! { "_id": 11_i64 })
         .await
         .expect("delete");
 
@@ -638,6 +640,6 @@ async fn a_ledger_document_this_binary_cannot_read_stops_the_status() {
         .expect_err("an error");
     migrations.close().await;
     assert!(error.to_string().contains("\"sideways\""), "{error}");
-    assert_eq!(MigrationId::LATEST.number(), 9);
+    assert_eq!(MigrationId::LATEST.number(), 10);
     d.drop_databases().await;
 }

crates/storage-mongodb/tests/migrations_locks.rs

@@ -53,7 +53,7 @@ async fn two_migrators_started_together_apply_once_and_the_other_finds_nothing()
     applied.sort_unstable();
     assert_eq!(
         applied,
-        [0, 10],
+        [0, 11],
         "one applies every step and the other finds nothing"
     );
     assert_eq!(

crates/storage-mongodb/tests/migrations_released.rs

@@ -161,7 +161,7 @@ async fn every_released_catalog_upgrades_to_the_definitions_of_a_fresh_install()
             panic!("{state} is not behind");
         };
         assert_eq!(version, None, "{state} has no ledger");
-        assert_eq!(pending.len(), 10, "{state}");
+        assert_eq!(pending.len(), 11, "{state}");
         let report = d.migrate().await.expect("migrate");
         assert_eq!(
             (report.from, report.to),

crates/storage-postgres/src/backup_catalog.rs

@@ -39,6 +39,13 @@
 //! [`extenddb_storage::backup_catalog::JOB_HEARTBEAT_INTERVAL`] (15 s), the
 //! constant every backend runs at until `[backup] heartbeat_interval_secs`
 //! reaches the jobs in a later release.
+//!
+//! `backup_index.source_table_details_json` is the manifest's source table,
+//! whose key schema, attribute definitions and indexes carry attribute
+//! names. It is stored through the escape of `catalog_json`, as the other
+//! catalog documents that carry names are: `claim_backup` and `set_status`
+//! write it through `escape_json`, and every read reverses the escape in
+//! `row_from_tuple`.
 
 use std::time::Duration;
 
@@ -61,6 +68,7 @@ use crate::backup_engine::{
     ClaimSession, RESTORE_HEARTBEAT, backup_lock_key, generation_column, restore_lock_key,
     try_claim, try_claim_with,
 };
+use crate::catalog_json::{escape_json, unescape_json};
 use crate::create_table::RestoreSource;
 
 /// The name of a CREATING restore target registered from `backup_arn`, if
@@ -167,7 +175,7 @@ fn row_from_tuple(t: RowTuple) -> Result<BackupRow, StorageError> {
         created_at,
         completed_at,
         failure_reason,
-        source_table_details_json,
+        source_table_details_json: unescape_json(source_table_details_json),
         owner_generation: u64::try_from(owner_generation).unwrap_or_default(),
         heartbeat_at,
     })
@@ -270,7 +278,7 @@ impl JobClaim for PostgresBackupClaim {
             .bind(fields.failure_reason.as_deref())
             .bind(fields.completed)
             .bind(from.as_str())
-            .bind(fields.source_table_details_json.as_ref())
+            .bind(fields.source_table_details_json.clone().map(escape_json))
             .execute(session.connection())
             .await;
             match written {
@@ -520,7 +528,7 @@ impl BackupCatalog for PostgresEngine {
             .bind(row.size_bytes)
             .bind(row.item_count)
             .bind(row.created_at)
-            .bind(&row.source_table_details_json)
+            .bind(escape_json(row.source_table_details_json.clone()))
             .bind(generation_column(session.generation()))
             .execute(session.connection())
             .await