diff --git a/AGENTS.md b/AGENTS.md index 8d19d159..1376a7fa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,3 +39,9 @@ V2 内置 ACL、CE 原生 Tag 和已安装 Guard 的 Checkout 不应重新受旧 **纯文档变更不编译、不跑测试、不构建镜像,也不部署。** 小范围代码改动优先检查受影响路径、复用已验证的证据;全量 C/Go/前端构建、三仓联验及镜像发布只在明确必要的阶段或发布任务开展。不要因一次文档提交自动要求全仓编译。 不要提交 `deploy/compose/.env`、任何真实凭据、密钥、持久数据卷或镜像构建产物。文档中文,代码和 commit message 英文;上游既有风格优先。 + +## 开发期数据库变更(2026-10-21) + +- 新版当前只明确新增四张业务表:`cf_library_storage`、`cf_dir_acl`、`cf_sso_group_map`、`cf_edit_guard`。历史表仅作备份或迁移输入,不属于新版运行库及运行依赖;增加业务表须先确认,历史数据单独归档,不直接删除。 +- 开发期建表、升级及数据迁移通过 Docker 仓 `database/` 下独立、人工执行的 SQL 交付,DEV 验证与 PROD 发布复用固定脚本及校验和。启动/重启仅校验,不自动执行扩展 DDL、旧迁移链或兼容补丁。 +- 仅明确审核的成熟版本之间才考虑自动升级;须定义精确源/目标范围和数据/中断恢复验证。不得根据 dev 分支、表名存在或旧台账自动推断升级路径,不自动回滚。 diff --git a/database/README.md b/database/README.md new file mode 100644 index 00000000..7c0b9714 --- /dev/null +++ b/database/README.md @@ -0,0 +1,31 @@ +# CloudFile 开发期数据库交付 + +2026-10-21 决策:CloudFile 仍处于开发阶段,数据库结构升级和数据迁移使用**独立、人工执行的 SQL**。DEV 验证与 PROD 发布使用同一份固定 SQL 包;成熟版本之间才允许另行设计、审核自动升级机制。开发分支、构建成功或标签变化都不自动取得迁移资格。 + +## 唯一新增业务表清单 + +截至本次决策,新版只明确新增四张表: + +| 表 | 作用 | +| --- | --- | +| `cf_library_storage` | 库与存储绑定 | +| `cf_dir_acl` | 目录权限及分享属性 | +| `cf_sso_group_map` | 外部身份与原生组映射 | +| `cf_edit_guard` | 编辑 Guard;普通上传下载不创建 Checkout/Checkin | + +其它 `cf_*` 名称属于历史实现或待确认依赖,仅用于备份或数据迁移,不属于新版运行库或运行依赖。不得因旧源码、JSON 迁移、旧开关或当前数据库存在便将其带入新版。不得将旧迁移链、`cloudfile.sql` 或 Django 的自动迁移直接作为新版初始化入口,也不据此删除现有表和数据。增加第五张业务表必须先单独确认设计。SQL 的候选表、归档表使用 `stage_`/`backup_` 前缀,属于受控运维产物,不能作为新业务模型。 + +## SQL 包与执行规则 + +当前包:[20261021-four-table-baseline](v2/20261021-four-table-baseline/README.md)。每个包说明源结构、目标结构、适用的 Server/Hub/Docker 提交、数据转换限制、执行顺序、验收与故障处理,并固定 SQL SHA256。 + +- 新安装执行独立建表 SQL;已有实例按精确源结构选择升级 SQL。发现缺表、未知结构或非空 Guard 时停止并审查,不自动猜测映射。 +- 应用启动/重启检查运行库不混入历史 CloudFile 表;基础三表必须匹配,Guard 仅在显式启用 Checkout 时检查,不执行 CloudFile 开发期 DDL、数据回填或旧兼容修复;不使用开发期开关恢复自动升级。结构不匹配就停止并给出手动 SQL 提示。 +- 发布前备份、停止所有写入者;DEV 验证后将同一份 SQL 及校验和用于对应源结构的 PROD,不能把 DEV 业务数据覆盖 PROD。两环境结构不同则另写并审核迁移。 +- 数据库结构与业务数据迁移分别写清,禁止静默截断、猜测身份、重置真实锁 generation 或依赖 HTTP 200 作为落盘证明。业务对象引用须可核验。 +- 默认不自动回滚。MySQL/MariaDB DDL 可能隐式提交,中断后先查看候选表、归档表及数据;停止写入后由人工决定修复或恢复。不能仅因脚本失败就重跑或删除备份。 +- 已交付并执行的 SQL 不改写;后续修正用新包记录。以脚本校验和、实际表结构和执行记录为依据,不伪造旧 `cf_schema_migration` 的成功状态。 + +成熟版本自动升级必须另外给出明确的源/目标版本范围、完整依赖表清单、并发/停机边界、备份与中断恢复证据、全量数据转换验收以及发布审批;不能默认遍历历史迁移目录。上游成熟 Seafile 版本的原有升级流程与 CloudFile 开发期扩展 SQL 分开管理。 + +历史 `cloudfile-directory-schema.py`、`SchemaRunner.apply()` 与 `tools/repair-v2-four-tables.py` 保留用于追溯,不作为新版开发期发布入口。已运行的容器仍使用旧启动代码,代码变更须随匹配版本发布后才能生效;此次规范和 SQL 交付不意味着已经替换 DEV 镜像或执行 PROD 升级。 diff --git a/database/v2/20261021-four-table-baseline/00-create.sql b/database/v2/20261021-four-table-baseline/00-create.sql new file mode 100644 index 00000000..02475c41 --- /dev/null +++ b/database/v2/20261021-four-table-baseline/00-create.sql @@ -0,0 +1,83 @@ +-- New installation only. Select the native Seafile database explicitly. +-- Do not use IF NOT EXISTS: a colliding old layout must stop installation. +-- CloudFile V2 fresh schema. Native CE tables remain the authoritative facts. +-- This file is independent of cloudfile.sql and the legacy migration chain. + +CREATE TABLE cf_library_storage ( + repo_key VARCHAR(80) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + repo_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + state VARCHAR(8) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + PRIMARY KEY (repo_key), + UNIQUE KEY library_storage_repo (repo_id), + CONSTRAINT library_storage_state CHECK (state IN ('reserved','active','retired')) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +CREATE TABLE cf_sso_group_map ( + provider VARCHAR(32) COLLATE utf8mb4_bin NOT NULL, + subject_type VARCHAR(4) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + namespace VARCHAR(128) COLLATE utf8mb4_bin NOT NULL, + external_id VARCHAR(225) COLLATE utf8mb4_bin NOT NULL, + group_id BIGINT NOT NULL, + PRIMARY KEY (provider,subject_type,namespace,external_id), + UNIQUE KEY sso_native_group (group_id), + CONSTRAINT sso_group_type CHECK (subject_type IN ('org','role')), + CONSTRAINT sso_group_id CHECK (group_id > 0) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +CREATE TABLE cf_dir_acl ( + repo_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + path_hash BINARY(32) NOT NULL, + path VARCHAR(4096) COLLATE utf8mb4_bin NOT NULL, + subject_type VARCHAR(4) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + provider VARCHAR(32) COLLATE utf8mb4_bin NOT NULL, + namespace VARCHAR(128) COLLATE utf8mb4_bin NOT NULL, + external_id VARCHAR(225) COLLATE utf8mb4_bin NOT NULL, + permission VARCHAR(9) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + inherit TINYINT NOT NULL, + share_mode TINYINT NOT NULL, + revision CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + PRIMARY KEY (repo_id,path_hash,subject_type,provider,namespace,external_id), + KEY acl_shared_subject (provider,subject_type,namespace,external_id,share_mode,repo_id), + CONSTRAINT acl_subject_type CHECK (subject_type IN ('user','org','role')), + CONSTRAINT acl_permission CHECK (permission IN ('r','rw','invisible')), + CONSTRAINT acl_inherit CHECK (inherit IN (0,1)), + CONSTRAINT acl_share CHECK (share_mode IN (0,1) AND + (share_mode=0 OR (permission IN ('r','rw') AND path <> '/'))) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +-- CloudFile V2 optional Rust local editing. Fresh isolated CF-EDIT installations ONLY. +-- The three-table V2 MVP remains unchanged unless editing is explicitly enabled. +-- There is NO server-side Intent or editing Session table. Seafile keeps native +-- File/Block/Commit/Branch versions; Agent Rust keeps its local pending Journal. +-- One row per physical file path preserves generation across Checkout cycles. +CREATE TABLE cf_edit_guard ( + repo_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + path_hash BINARY(32) NOT NULL, + path VARCHAR(4096) COLLATE utf8mb4_bin NOT NULL, + guard_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NULL, + generation BIGINT UNSIGNED NOT NULL DEFAULT 0, + owner_uid VARCHAR(225) COLLATE utf8mb4_bin NULL, + owner_native_user VARCHAR(255) COLLATE utf8mb4_bin NULL, + terminal_type VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NULL, + terminal_name VARCHAR(128) COLLATE utf8mb4_bin NULL, + checkout_ip VARCHAR(45) CHARACTER SET ascii COLLATE ascii_bin NULL, + proof_digest CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NULL, + base_file_id CHAR(40) CHARACTER SET ascii COLLATE ascii_bin NULL, + lease_expires_at DATETIME(6) NULL, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + PRIMARY KEY (repo_id, path_hash), + UNIQUE KEY cf_edit_guard_id (guard_id), + KEY cf_edit_guard_active (repo_id, guard_id), + KEY cf_edit_guard_owner (owner_uid, guard_id), + CONSTRAINT cf_edit_terminal_type CHECK ( + terminal_type IS NULL OR terminal_type IN ('windows','macos','linux','other') + ), + CONSTRAINT cf_edit_active_shape CHECK ( + (guard_id IS NULL AND owner_uid IS NULL AND owner_native_user IS NULL + AND proof_digest IS NULL AND base_file_id IS NULL AND lease_expires_at IS NULL) + OR + (guard_id IS NOT NULL AND owner_uid IS NOT NULL AND owner_native_user IS NOT NULL + AND proof_digest IS NOT NULL AND base_file_id IS NOT NULL AND lease_expires_at IS NOT NULL) + ) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/database/v2/20261021-four-table-baseline/01-preflight.sql b/database/v2/20261021-four-table-baseline/01-preflight.sql new file mode 100644 index 00000000..4d1e12f6 --- /dev/null +++ b/database/v2/20261021-four-table-baseline/01-preflight.sql @@ -0,0 +1,18 @@ +-- Read-only inspection, usable before/after the four-table conversion. +-- Select the native Seafile database. Output metadata and aggregates only. +SELECT DATABASE() AS database_name, VERSION() AS engine_version; +SELECT TABLE_NAME,ENGINE,TABLE_COLLATION FROM information_schema.TABLES + WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME IN + ('cf_library_storage','cf_dir_acl','cf_sso_group_map','cf_edit_guard') ORDER BY TABLE_NAME; +SELECT TABLE_NAME,COLUMN_NAME,COLUMN_TYPE,IS_NULLABLE,COLUMN_DEFAULT,COLLATION_NAME + FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME IN + ('cf_library_storage','cf_dir_acl','cf_sso_group_map','cf_edit_guard') ORDER BY TABLE_NAME,ORDINAL_POSITION; +SELECT TABLE_NAME,INDEX_NAME,NON_UNIQUE,SEQ_IN_INDEX,COLUMN_NAME,SUB_PART + FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME IN + ('cf_library_storage','cf_dir_acl','cf_sso_group_map','cf_edit_guard') ORDER BY TABLE_NAME,INDEX_NAME,SEQ_IN_INDEX; +SELECT 'cf_library_storage' AS table_name,COUNT(*) AS row_count FROM cf_library_storage + UNION ALL SELECT 'cf_dir_acl',COUNT(*) FROM cf_dir_acl + UNION ALL SELECT 'cf_sso_group_map',COUNT(*) FROM cf_sso_group_map + UNION ALL SELECT 'cf_edit_guard',COUNT(*) FROM cf_edit_guard; +SELECT COUNT(*) AS guard_rows,SUM(guard_id IS NOT NULL) AS occupied_guard_rows, + SUM(generation>0) AS rows_with_generation FROM cf_edit_guard; diff --git a/database/v2/20261021-four-table-baseline/05-archive-legacy.sql b/database/v2/20261021-four-table-baseline/05-archive-legacy.sql new file mode 100644 index 00000000..06cdd92a --- /dev/null +++ b/database/v2/20261021-four-table-baseline/05-archive-legacy.sql @@ -0,0 +1,18 @@ +-- Read-only archive plan. Historical tables must not remain in V2 runtime. +-- Select the runtime DB and set @cf_upgrade_archive_database to a separately +-- backed-up archive DB. Review the generated statements and dependencies, +-- stop all writers, then execute the reviewed RENAME statements explicitly. +-- This file itself never moves or deletes data; FK/views require DBA review. +SELECT TABLE_NAME,TABLE_TYPE,ENGINE + FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() + AND LOWER(LEFT(TABLE_NAME,2))='cf' + AND TABLE_NAME NOT IN ('cf_library_storage','cf_dir_acl','cf_sso_group_map','cf_edit_guard') + ORDER BY TABLE_NAME; +SELECT CONCAT('RENAME TABLE `',REPLACE(TABLE_SCHEMA,'`','``'),'`.`',REPLACE(TABLE_NAME,'`','``'), + '` TO `',REPLACE(@cf_upgrade_archive_database,'`','``'),'`.`',REPLACE(TABLE_NAME,'`','``'),'`;') + AS reviewed_archive_sql + FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_TYPE='BASE TABLE' + AND @cf_upgrade_archive_database IS NOT NULL AND @cf_upgrade_archive_database<>DATABASE() + AND LOWER(LEFT(TABLE_NAME,2))='cf' + AND TABLE_NAME NOT IN ('cf_library_storage','cf_dir_acl','cf_sso_group_map','cf_edit_guard') + ORDER BY TABLE_NAME; diff --git a/database/v2/20261021-four-table-baseline/10-upgrade-029-042.sql b/database/v2/20261021-four-table-baseline/10-upgrade-029-042.sql new file mode 100644 index 00000000..3265b705 --- /dev/null +++ b/database/v2/20261021-four-table-baseline/10-upgrade-029-042.sql @@ -0,0 +1,143 @@ +-- Development release SQL: reviewed 029/042 layout -> four-table V2 baseline. +-- Execute OFFLINE in the explicitly selected native Seafile database. +-- Set @cf_upgrade_writers_stopped=1 after stopping ALL writers. +-- If old lifecycle-bound ACLs exist, separately review/acknowledge the target +-- path-based semantics with @cf_upgrade_accept_path_acl=1. +-- Back up the database first. Never use mysql --force or automatic startup. +-- A single RENAME retains original tables; no DROP or automatic rollback. +DELIMITER $$ +DROP PROCEDURE IF EXISTS cfv2_manual_upgrade_20261021$$ +CREATE PROCEDURE cfv2_manual_upgrade_20261021() +main: BEGIN + DECLARE acquired INT DEFAULT 0; + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + DO RELEASE_LOCK('cloudfile.schema.manual.v2'); + RESIGNAL; + END; + IF DATABASE() IS NULL OR COALESCE(@cf_upgrade_writers_stopped,0) <> 1 THEN + SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='Select the database, back up and stop all writers before manual SQL'; + END IF; + IF COALESCE(@cf_upgrade_archive_database,'') NOT REGEXP '^[A-Za-z][A-Za-z0-9_]{0,63}$' + OR @cf_upgrade_archive_database=DATABASE() + OR NOT EXISTS(SELECT 1 FROM information_schema.schemata WHERE schema_name=@cf_upgrade_archive_database) THEN + SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='Create and select a separate controlled archive database first'; + END IF; + SELECT GET_LOCK('cloudfile.schema.manual.v2',0) INTO acquired; + IF acquired <> 1 THEN SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='Another manual schema operation is running'; END IF; + -- No implicit conversion of unknown or already-upgraded schemas. + IF (SELECT COUNT(*) FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_library_storage')<>3 + OR (SELECT COUNT(*) FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_sso_group_map')<>10 + OR (SELECT COUNT(*) FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_dir_acl')<>15 + OR (SELECT COUNT(*) FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_edit_guard')<>16 + OR NOT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_edit_guard' AND column_name='resource_uid') + OR NOT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_dir_acl' AND column_name='subject_hash') THEN + SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='Unsupported source layout or already upgraded; inspect instead of replaying'; + END IF; + IF EXISTS(SELECT 1 FROM information_schema.key_column_usage WHERE referenced_table_schema=DATABASE() AND referenced_table_name IN ('cf_library_storage','cf_sso_group_map','cf_dir_acl','cf_edit_guard')) THEN + SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='Foreign key dependency requires a separately reviewed migration'; + END IF; + IF EXISTS(SELECT 1 FROM information_schema.tables WHERE (table_schema=DATABASE() AND table_name LIKE 'stage\_20261021\_cf\_%') OR (table_schema=@cf_upgrade_archive_database AND table_name LIKE 'backup\_20261021\_cf\_%')) THEN + SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='Candidate or backup exists; inspect partial state and do not overwrite'; + END IF; + IF EXISTS(SELECT 1 FROM cf_edit_guard) THEN + SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='Guard data is not empty; preserve generation and design explicit lease conversion'; + END IF; + IF EXISTS(SELECT 1 FROM cf_dir_acl WHERE kind<>'dir' OR CHAR_LENGTH(path)>4096 OR CHAR_LENGTH(namespace)>128 OR CHAR_LENGTH(external_id)>225 OR BINARY path_hash<>BINARY SHA2(path,256)) + OR EXISTS(SELECT 1 FROM cf_sso_group_map WHERE CHAR_LENGTH(namespace)>128 OR CHAR_LENGTH(external_id)>225 OR subject_type NOT IN ('org','role') OR group_id<=0) THEN + SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='Source rows do not fit the target contract; no truncation or identity inference'; + END IF; + IF EXISTS(SELECT 1 FROM cf_dir_acl WHERE lifecycle_ref<>'') AND COALESCE(@cf_upgrade_accept_path_acl,0)<>1 THEN + SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='Review lifecycle-bound ACL conversion to path semantics before continuing'; + END IF; + CREATE TABLE stage_20261021_cf_library_storage ( + repo_key VARCHAR(80) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + repo_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + state VARCHAR(8) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + PRIMARY KEY (repo_key), + UNIQUE KEY library_storage_repo (repo_id), + CONSTRAINT library_storage_state CHECK (state IN ('reserved','active','retired')) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + CREATE TABLE stage_20261021_cf_sso_group_map ( + provider VARCHAR(32) COLLATE utf8mb4_bin NOT NULL, + subject_type VARCHAR(4) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + namespace VARCHAR(128) COLLATE utf8mb4_bin NOT NULL, + external_id VARCHAR(225) COLLATE utf8mb4_bin NOT NULL, + group_id BIGINT NOT NULL, + PRIMARY KEY (provider,subject_type,namespace,external_id), + UNIQUE KEY sso_native_group (group_id), + CONSTRAINT sso_group_type CHECK (subject_type IN ('org','role')), + CONSTRAINT sso_group_id CHECK (group_id > 0) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + CREATE TABLE stage_20261021_cf_dir_acl ( + repo_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + path_hash BINARY(32) NOT NULL, + path VARCHAR(4096) COLLATE utf8mb4_bin NOT NULL, + subject_type VARCHAR(4) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + provider VARCHAR(32) COLLATE utf8mb4_bin NOT NULL, + namespace VARCHAR(128) COLLATE utf8mb4_bin NOT NULL, + external_id VARCHAR(225) COLLATE utf8mb4_bin NOT NULL, + permission VARCHAR(9) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + inherit TINYINT NOT NULL, + share_mode TINYINT NOT NULL, + revision CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + PRIMARY KEY (repo_id,path_hash,subject_type,provider,namespace,external_id), + KEY acl_shared_subject (provider,subject_type,namespace,external_id,share_mode,repo_id), + CONSTRAINT acl_subject_type CHECK (subject_type IN ('user','org','role')), + CONSTRAINT acl_permission CHECK (permission IN ('r','rw','invisible')), + CONSTRAINT acl_inherit CHECK (inherit IN (0,1)), + CONSTRAINT acl_share CHECK (share_mode IN (0,1) AND + (share_mode=0 OR (permission IN ('r','rw') AND path <> '/'))) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + CREATE TABLE stage_20261021_cf_edit_guard ( + repo_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + path_hash BINARY(32) NOT NULL, + path VARCHAR(4096) COLLATE utf8mb4_bin NOT NULL, + guard_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NULL, + generation BIGINT UNSIGNED NOT NULL DEFAULT 0, + owner_uid VARCHAR(225) COLLATE utf8mb4_bin NULL, + owner_native_user VARCHAR(255) COLLATE utf8mb4_bin NULL, + terminal_type VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NULL, + terminal_name VARCHAR(128) COLLATE utf8mb4_bin NULL, + checkout_ip VARCHAR(45) CHARACTER SET ascii COLLATE ascii_bin NULL, + proof_digest CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NULL, + base_file_id CHAR(40) CHARACTER SET ascii COLLATE ascii_bin NULL, + lease_expires_at DATETIME(6) NULL, + created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + PRIMARY KEY (repo_id, path_hash), + UNIQUE KEY cf_edit_guard_id (guard_id), + KEY cf_edit_guard_active (repo_id, guard_id), + KEY cf_edit_guard_owner (owner_uid, guard_id), + CONSTRAINT cf_edit_terminal_type CHECK ( + terminal_type IS NULL OR terminal_type IN ('windows','macos','linux','other') + ), + CONSTRAINT cf_edit_active_shape CHECK ( + (guard_id IS NULL AND owner_uid IS NULL AND owner_native_user IS NULL + AND proof_digest IS NULL AND base_file_id IS NULL AND lease_expires_at IS NULL) + OR + (guard_id IS NOT NULL AND owner_uid IS NOT NULL AND owner_native_user IS NOT NULL + AND proof_digest IS NOT NULL AND base_file_id IS NOT NULL AND lease_expires_at IS NOT NULL) + ) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + INSERT INTO stage_20261021_cf_library_storage (repo_key,repo_id,state) SELECT repo_key,repo_id,state FROM cf_library_storage; + INSERT INTO stage_20261021_cf_sso_group_map (provider,subject_type,namespace,external_id,group_id) SELECT provider,subject_type,namespace,external_id,group_id FROM cf_sso_group_map; + INSERT INTO stage_20261021_cf_dir_acl (repo_id,path_hash,path,subject_type,provider,namespace,external_id,permission,inherit,share_mode,revision) SELECT repo_id,UNHEX(path_hash),path,subject_type,provider,namespace,external_id,permission,inherit,share_mode,revision FROM cf_dir_acl; + IF (SELECT COUNT(*) FROM cf_library_storage)<>(SELECT COUNT(*) FROM stage_20261021_cf_library_storage) THEN SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='Candidate row count mismatch'; END IF; + IF (SELECT COUNT(*) FROM cf_sso_group_map)<>(SELECT COUNT(*) FROM stage_20261021_cf_sso_group_map) THEN SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='Candidate row count mismatch'; END IF; + IF (SELECT COUNT(*) FROM cf_dir_acl)<>(SELECT COUNT(*) FROM stage_20261021_cf_dir_acl) THEN SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='Candidate row count mismatch'; END IF; + IF (SELECT COUNT(*) FROM cf_edit_guard)<>(SELECT COUNT(*) FROM stage_20261021_cf_edit_guard) THEN SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='Candidate row count mismatch'; END IF; + IF EXISTS(SELECT 1 FROM cf_library_storage s LEFT JOIN stage_20261021_cf_library_storage n ON n.repo_key=s.repo_key WHERE n.repo_key IS NULL OR NOT ((n.repo_key <=> s.repo_key) AND (n.repo_id <=> s.repo_id) AND (n.state <=> s.state))) THEN SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='Candidate data mismatch'; END IF; + IF EXISTS(SELECT 1 FROM cf_sso_group_map s LEFT JOIN stage_20261021_cf_sso_group_map n ON n.provider=s.provider AND n.subject_type=s.subject_type AND n.namespace=s.namespace AND n.external_id=s.external_id WHERE n.provider IS NULL OR NOT ((n.provider <=> s.provider) AND (n.subject_type <=> s.subject_type) AND (n.namespace <=> s.namespace) AND (n.external_id <=> s.external_id) AND (n.group_id <=> s.group_id))) THEN SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='Candidate data mismatch'; END IF; + IF EXISTS(SELECT 1 FROM cf_dir_acl s LEFT JOIN stage_20261021_cf_dir_acl n ON n.repo_id=s.repo_id AND n.path_hash=UNHEX(s.path_hash) AND n.subject_type=s.subject_type AND n.provider=s.provider AND n.namespace=s.namespace AND n.external_id=s.external_id WHERE n.repo_id IS NULL OR NOT ((n.repo_id <=> s.repo_id) AND (n.path_hash <=> UNHEX(s.path_hash)) AND (n.path <=> s.path) AND (n.subject_type <=> s.subject_type) AND (n.provider <=> s.provider) AND (n.namespace <=> s.namespace) AND (n.external_id <=> s.external_id) AND (n.permission <=> s.permission) AND (n.inherit <=> s.inherit) AND (n.share_mode <=> s.share_mode) AND (n.revision <=> s.revision))) THEN SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='Candidate data mismatch'; END IF; + -- All four names move atomically; old rows remain available for inspection. + SET @cfv2_cutover_sql=CONCAT('RENAME TABLE ','cf_library_storage TO `',@cf_upgrade_archive_database,'`.`backup_20261021_cf_library_storage`',', ','stage_20261021_cf_library_storage TO cf_library_storage',', ','cf_sso_group_map TO `',@cf_upgrade_archive_database,'`.`backup_20261021_cf_sso_group_map`',', ','stage_20261021_cf_sso_group_map TO cf_sso_group_map',', ','cf_dir_acl TO `',@cf_upgrade_archive_database,'`.`backup_20261021_cf_dir_acl`',', ','stage_20261021_cf_dir_acl TO cf_dir_acl',', ','cf_edit_guard TO `',@cf_upgrade_archive_database,'`.`backup_20261021_cf_edit_guard`',', ','stage_20261021_cf_edit_guard TO cf_edit_guard'); + PREPARE cfv2_cutover FROM @cfv2_cutover_sql; + EXECUTE cfv2_cutover; + DEALLOCATE PREPARE cfv2_cutover; + DO RELEASE_LOCK('cloudfile.schema.manual.v2'); + SELECT 'four-table upgrade complete; originals archived; archive other legacy tables then run 50-verify.sql' AS result; +END$$ +CALL cfv2_manual_upgrade_20261021()$$ +DROP PROCEDURE cfv2_manual_upgrade_20261021$$ +DELIMITER ; diff --git a/database/v2/20261021-four-table-baseline/20-seahub-compat.sql b/database/v2/20261021-four-table-baseline/20-seahub-compat.sql new file mode 100644 index 00000000..8a8ce04b --- /dev/null +++ b/database/v2/20261021-four-table-baseline/20-seahub-compat.sql @@ -0,0 +1,33 @@ +-- Explicit compatibility columns formerly written by bootstrap. +-- Select the SEAHUB database, not the native Seafile database. +-- Apply offline only when adopting this CloudFile code onto CE14. +-- Existing model data remains unchanged; object-ID backfill requires RPC and +-- is a separate reviewed operation, never a fictitious SQL backfill. +DELIMITER $$ +DROP PROCEDURE IF EXISTS cfv2_seahub_compat_20261021$$ +CREATE PROCEDURE cfv2_seahub_compat_20261021() +BEGIN + IF DATABASE() IS NULL OR COALESCE(@cf_upgrade_writers_stopped,0)<>1 THEN + SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='Select Seahub database and stop all writers'; + END IF; + IF (SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=DATABASE() + AND table_name IN ('repo_metadata','repo_tags_repotags','base_userstarredfiles'))<>3 THEN + SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='Expected CE14 Seahub tables are missing'; + END IF; + IF NOT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='repo_metadata' AND column_name='summary_enabled') THEN + ALTER TABLE repo_metadata ADD COLUMN summary_enabled TINYINT(1) NOT NULL DEFAULT 0, + ADD KEY key_repo_metadata_summary_enabled(summary_enabled); + END IF; + IF NOT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='repo_tags_repotags' AND column_name='is_system') THEN + ALTER TABLE repo_tags_repotags ADD COLUMN is_system TINYINT(1) NOT NULL DEFAULT 0, + ADD KEY repo_tags_repotags_is_system(is_system); + END IF; + IF NOT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='base_userstarredfiles' AND column_name='obj_id') THEN + ALTER TABLE base_userstarredfiles ADD COLUMN obj_id VARCHAR(64) NULL, + ADD KEY base_userstarredfiles_obj_id(obj_id); + END IF; + SELECT 'Seahub compatibility columns ready; run model/API validation' AS result; +END$$ +CALL cfv2_seahub_compat_20261021()$$ +DROP PROCEDURE cfv2_seahub_compat_20261021$$ +DELIMITER ; diff --git a/database/v2/20261021-four-table-baseline/50-verify.sql b/database/v2/20261021-four-table-baseline/50-verify.sql new file mode 100644 index 00000000..1cba94f1 --- /dev/null +++ b/database/v2/20261021-four-table-baseline/50-verify.sql @@ -0,0 +1,93 @@ +-- Explicit target verification. Read-only against business tables. +-- A short-lived stored routine provides SIGNAL on mismatch, not a new ledger. +-- Run with the mysql/mariadb client in batch mode WITHOUT --force. +DELIMITER $$ +DROP PROCEDURE IF EXISTS cfv2_verify_20261021$$ +CREATE PROCEDURE cfv2_verify_20261021() +BEGIN + IF DATABASE() IS NULL THEN SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='Select the native Seafile database'; END IF; + IF EXISTS(SELECT 1 FROM information_schema.tables WHERE table_schema=DATABASE() + AND LOWER(LEFT(table_name,2))='cf' AND table_name NOT IN + ('cf_library_storage','cf_dir_acl','cf_sso_group_map','cf_edit_guard')) THEN + SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='Historical CloudFile tables must be outside the runtime database'; + END IF; + + IF NOT ((SELECT COUNT(*) FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_library_storage')=3 + AND EXISTS(SELECT 1 FROM information_schema.tables WHERE table_schema=DATABASE() AND table_name='cf_library_storage' AND engine='InnoDB' AND table_collation='utf8mb4_bin') + AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_library_storage' AND column_name='repo_key' AND data_type='varchar' AND is_nullable='NO' AND character_maximum_length=80 AND collation_name='ascii_bin') + AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_library_storage' AND column_name='repo_id' AND data_type='char' AND is_nullable='NO' AND character_maximum_length=36 AND collation_name='ascii_bin') + AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_library_storage' AND column_name='state' AND data_type='varchar' AND is_nullable='NO' AND character_maximum_length=8 AND collation_name='ascii_bin') + AND (SELECT COUNT(*) FROM information_schema.statistics WHERE table_schema=DATABASE() AND table_name='cf_library_storage' AND index_name='PRIMARY')=1 + AND (SELECT COUNT(*) FROM information_schema.statistics WHERE table_schema=DATABASE() AND table_name='cf_library_storage' AND index_name='PRIMARY' AND non_unique=0 AND sub_part IS NULL AND ((seq_in_index=1 AND column_name='repo_key')))=1 + AND (SELECT COUNT(*) FROM information_schema.statistics WHERE table_schema=DATABASE() AND table_name='cf_library_storage' AND index_name='library_storage_repo')=1 + AND (SELECT COUNT(*) FROM information_schema.statistics WHERE table_schema=DATABASE() AND table_name='cf_library_storage' AND index_name='library_storage_repo' AND non_unique=0 AND sub_part IS NULL AND ((seq_in_index=1 AND column_name='repo_id')))=1 + AND EXISTS(SELECT 1 FROM information_schema.table_constraints WHERE constraint_schema=DATABASE() AND table_name='cf_library_storage' AND constraint_name='library_storage_state' AND constraint_type='CHECK')) THEN SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='Target contract mismatch: cf_library_storage'; END IF; + SELECT 'cf_library_storage verified' AS result; + IF NOT ((SELECT COUNT(*) FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_sso_group_map')=5 + AND EXISTS(SELECT 1 FROM information_schema.tables WHERE table_schema=DATABASE() AND table_name='cf_sso_group_map' AND engine='InnoDB' AND table_collation='utf8mb4_bin') + AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_sso_group_map' AND column_name='provider' AND data_type='varchar' AND is_nullable='NO' AND character_maximum_length=32 AND collation_name='utf8mb4_bin') + AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_sso_group_map' AND column_name='subject_type' AND data_type='varchar' AND is_nullable='NO' AND character_maximum_length=4 AND collation_name='ascii_bin') + AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_sso_group_map' AND column_name='namespace' AND data_type='varchar' AND is_nullable='NO' AND character_maximum_length=128 AND collation_name='utf8mb4_bin') + AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_sso_group_map' AND column_name='external_id' AND data_type='varchar' AND is_nullable='NO' AND character_maximum_length=225 AND collation_name='utf8mb4_bin') + AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_sso_group_map' AND column_name='group_id' AND data_type='bigint' AND is_nullable='NO') + AND (SELECT COUNT(*) FROM information_schema.statistics WHERE table_schema=DATABASE() AND table_name='cf_sso_group_map' AND index_name='PRIMARY')=4 + AND (SELECT COUNT(*) FROM information_schema.statistics WHERE table_schema=DATABASE() AND table_name='cf_sso_group_map' AND index_name='PRIMARY' AND non_unique=0 AND sub_part IS NULL AND ((seq_in_index=1 AND column_name='provider') OR (seq_in_index=2 AND column_name='subject_type') OR (seq_in_index=3 AND column_name='namespace') OR (seq_in_index=4 AND column_name='external_id')))=4 + AND (SELECT COUNT(*) FROM information_schema.statistics WHERE table_schema=DATABASE() AND table_name='cf_sso_group_map' AND index_name='sso_native_group')=1 + AND (SELECT COUNT(*) FROM information_schema.statistics WHERE table_schema=DATABASE() AND table_name='cf_sso_group_map' AND index_name='sso_native_group' AND non_unique=0 AND sub_part IS NULL AND ((seq_in_index=1 AND column_name='group_id')))=1 + AND EXISTS(SELECT 1 FROM information_schema.table_constraints WHERE constraint_schema=DATABASE() AND table_name='cf_sso_group_map' AND constraint_name='sso_group_type' AND constraint_type='CHECK') + AND EXISTS(SELECT 1 FROM information_schema.table_constraints WHERE constraint_schema=DATABASE() AND table_name='cf_sso_group_map' AND constraint_name='sso_group_id' AND constraint_type='CHECK')) THEN SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='Target contract mismatch: cf_sso_group_map'; END IF; + SELECT 'cf_sso_group_map verified' AS result; + IF NOT ((SELECT COUNT(*) FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_dir_acl')=11 + AND EXISTS(SELECT 1 FROM information_schema.tables WHERE table_schema=DATABASE() AND table_name='cf_dir_acl' AND engine='InnoDB' AND table_collation='utf8mb4_bin') + AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_dir_acl' AND column_name='repo_id' AND data_type='char' AND is_nullable='NO' AND character_maximum_length=36 AND collation_name='ascii_bin') + AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_dir_acl' AND column_name='path_hash' AND data_type='binary' AND is_nullable='NO' AND character_maximum_length=32) + AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_dir_acl' AND column_name='path' AND data_type='varchar' AND is_nullable='NO' AND character_maximum_length=4096 AND collation_name='utf8mb4_bin') + AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_dir_acl' AND column_name='subject_type' AND data_type='varchar' AND is_nullable='NO' AND character_maximum_length=4 AND collation_name='ascii_bin') + AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_dir_acl' AND column_name='provider' AND data_type='varchar' AND is_nullable='NO' AND character_maximum_length=32 AND collation_name='utf8mb4_bin') + AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_dir_acl' AND column_name='namespace' AND data_type='varchar' AND is_nullable='NO' AND character_maximum_length=128 AND collation_name='utf8mb4_bin') + AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_dir_acl' AND column_name='external_id' AND data_type='varchar' AND is_nullable='NO' AND character_maximum_length=225 AND collation_name='utf8mb4_bin') + AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_dir_acl' AND column_name='permission' AND data_type='varchar' AND is_nullable='NO' AND character_maximum_length=9 AND collation_name='ascii_bin') + AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_dir_acl' AND column_name='inherit' AND data_type='tinyint' AND is_nullable='NO') + AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_dir_acl' AND column_name='share_mode' AND data_type='tinyint' AND is_nullable='NO') + AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_dir_acl' AND column_name='revision' AND data_type='char' AND is_nullable='NO' AND character_maximum_length=36 AND collation_name='ascii_bin') + AND (SELECT COUNT(*) FROM information_schema.statistics WHERE table_schema=DATABASE() AND table_name='cf_dir_acl' AND index_name='PRIMARY')=6 + AND (SELECT COUNT(*) FROM information_schema.statistics WHERE table_schema=DATABASE() AND table_name='cf_dir_acl' AND index_name='PRIMARY' AND non_unique=0 AND sub_part IS NULL AND ((seq_in_index=1 AND column_name='repo_id') OR (seq_in_index=2 AND column_name='path_hash') OR (seq_in_index=3 AND column_name='subject_type') OR (seq_in_index=4 AND column_name='provider') OR (seq_in_index=5 AND column_name='namespace') OR (seq_in_index=6 AND column_name='external_id')))=6 + AND (SELECT COUNT(*) FROM information_schema.statistics WHERE table_schema=DATABASE() AND table_name='cf_dir_acl' AND index_name='acl_shared_subject')=6 + AND (SELECT COUNT(*) FROM information_schema.statistics WHERE table_schema=DATABASE() AND table_name='cf_dir_acl' AND index_name='acl_shared_subject' AND non_unique=1 AND sub_part IS NULL AND ((seq_in_index=1 AND column_name='provider') OR (seq_in_index=2 AND column_name='subject_type') OR (seq_in_index=3 AND column_name='namespace') OR (seq_in_index=4 AND column_name='external_id') OR (seq_in_index=5 AND column_name='share_mode') OR (seq_in_index=6 AND column_name='repo_id')))=6 + AND EXISTS(SELECT 1 FROM information_schema.table_constraints WHERE constraint_schema=DATABASE() AND table_name='cf_dir_acl' AND constraint_name='acl_subject_type' AND constraint_type='CHECK') + AND EXISTS(SELECT 1 FROM information_schema.table_constraints WHERE constraint_schema=DATABASE() AND table_name='cf_dir_acl' AND constraint_name='acl_permission' AND constraint_type='CHECK') + AND EXISTS(SELECT 1 FROM information_schema.table_constraints WHERE constraint_schema=DATABASE() AND table_name='cf_dir_acl' AND constraint_name='acl_inherit' AND constraint_type='CHECK') + AND EXISTS(SELECT 1 FROM information_schema.table_constraints WHERE constraint_schema=DATABASE() AND table_name='cf_dir_acl' AND constraint_name='acl_share' AND constraint_type='CHECK')) THEN SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='Target contract mismatch: cf_dir_acl'; END IF; + SELECT 'cf_dir_acl verified' AS result; + IF NOT ((SELECT COUNT(*) FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_edit_guard')=15 + AND EXISTS(SELECT 1 FROM information_schema.tables WHERE table_schema=DATABASE() AND table_name='cf_edit_guard' AND engine='InnoDB' AND table_collation='utf8mb4_bin') + AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_edit_guard' AND column_name='repo_id' AND data_type='char' AND is_nullable='NO' AND character_maximum_length=36 AND collation_name='ascii_bin') + AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_edit_guard' AND column_name='path_hash' AND data_type='binary' AND is_nullable='NO' AND character_maximum_length=32) + AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_edit_guard' AND column_name='path' AND data_type='varchar' AND is_nullable='NO' AND character_maximum_length=4096 AND collation_name='utf8mb4_bin') + AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_edit_guard' AND column_name='guard_id' AND data_type='char' AND is_nullable='YES' AND character_maximum_length=36 AND collation_name='ascii_bin') + AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_edit_guard' AND column_name='generation' AND data_type='bigint' AND is_nullable='NO' AND column_type LIKE '%unsigned%') + AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_edit_guard' AND column_name='owner_uid' AND data_type='varchar' AND is_nullable='YES' AND character_maximum_length=225 AND collation_name='utf8mb4_bin') + AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_edit_guard' AND column_name='owner_native_user' AND data_type='varchar' AND is_nullable='YES' AND character_maximum_length=255 AND collation_name='utf8mb4_bin') + AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_edit_guard' AND column_name='terminal_type' AND data_type='varchar' AND is_nullable='YES' AND character_maximum_length=16 AND collation_name='ascii_bin') + AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_edit_guard' AND column_name='terminal_name' AND data_type='varchar' AND is_nullable='YES' AND character_maximum_length=128 AND collation_name='utf8mb4_bin') + AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_edit_guard' AND column_name='checkout_ip' AND data_type='varchar' AND is_nullable='YES' AND character_maximum_length=45 AND collation_name='ascii_bin') + AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_edit_guard' AND column_name='proof_digest' AND data_type='char' AND is_nullable='YES' AND character_maximum_length=64 AND collation_name='ascii_bin') + AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_edit_guard' AND column_name='base_file_id' AND data_type='char' AND is_nullable='YES' AND character_maximum_length=40 AND collation_name='ascii_bin') + AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_edit_guard' AND column_name='lease_expires_at' AND data_type='datetime' AND is_nullable='YES' AND datetime_precision=6) + AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_edit_guard' AND column_name='created_at' AND data_type='datetime' AND is_nullable='NO' AND datetime_precision=6) + AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='cf_edit_guard' AND column_name='updated_at' AND data_type='datetime' AND is_nullable='NO' AND datetime_precision=6) + AND (SELECT COUNT(*) FROM information_schema.statistics WHERE table_schema=DATABASE() AND table_name='cf_edit_guard' AND index_name='PRIMARY')=2 + AND (SELECT COUNT(*) FROM information_schema.statistics WHERE table_schema=DATABASE() AND table_name='cf_edit_guard' AND index_name='PRIMARY' AND non_unique=0 AND sub_part IS NULL AND ((seq_in_index=1 AND column_name='repo_id') OR (seq_in_index=2 AND column_name='path_hash')))=2 + AND (SELECT COUNT(*) FROM information_schema.statistics WHERE table_schema=DATABASE() AND table_name='cf_edit_guard' AND index_name='cf_edit_guard_id')=1 + AND (SELECT COUNT(*) FROM information_schema.statistics WHERE table_schema=DATABASE() AND table_name='cf_edit_guard' AND index_name='cf_edit_guard_id' AND non_unique=0 AND sub_part IS NULL AND ((seq_in_index=1 AND column_name='guard_id')))=1 + AND (SELECT COUNT(*) FROM information_schema.statistics WHERE table_schema=DATABASE() AND table_name='cf_edit_guard' AND index_name='cf_edit_guard_active')=2 + AND (SELECT COUNT(*) FROM information_schema.statistics WHERE table_schema=DATABASE() AND table_name='cf_edit_guard' AND index_name='cf_edit_guard_active' AND non_unique=1 AND sub_part IS NULL AND ((seq_in_index=1 AND column_name='repo_id') OR (seq_in_index=2 AND column_name='guard_id')))=2 + AND (SELECT COUNT(*) FROM information_schema.statistics WHERE table_schema=DATABASE() AND table_name='cf_edit_guard' AND index_name='cf_edit_guard_owner')=2 + AND (SELECT COUNT(*) FROM information_schema.statistics WHERE table_schema=DATABASE() AND table_name='cf_edit_guard' AND index_name='cf_edit_guard_owner' AND non_unique=1 AND sub_part IS NULL AND ((seq_in_index=1 AND column_name='owner_uid') OR (seq_in_index=2 AND column_name='guard_id')))=2 + AND EXISTS(SELECT 1 FROM information_schema.table_constraints WHERE constraint_schema=DATABASE() AND table_name='cf_edit_guard' AND constraint_name='cf_edit_terminal_type' AND constraint_type='CHECK') + AND EXISTS(SELECT 1 FROM information_schema.table_constraints WHERE constraint_schema=DATABASE() AND table_name='cf_edit_guard' AND constraint_name='cf_edit_active_shape' AND constraint_type='CHECK')) THEN SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='Target contract mismatch: cf_edit_guard'; END IF; + SELECT 'cf_edit_guard verified' AS result; +END$$ +CALL cfv2_verify_20261021()$$ +DROP PROCEDURE cfv2_verify_20261021$$ +DELIMITER ; diff --git a/database/v2/20261021-four-table-baseline/README.md b/database/v2/20261021-four-table-baseline/README.md new file mode 100644 index 00000000..43d09ee1 --- /dev/null +++ b/database/v2/20261021-four-table-baseline/README.md @@ -0,0 +1,72 @@ +# 四表开发基线 SQL 包 + +适用:Server `06efc4dea43c5200eea5dd2341f581536ad0838c` 的四表 DDL、Hub `fc9e89c53636432e2a6da824a8758d1b0a374316` 的对应 V2 读写合同。Docker 启动只读校验的配套提交随本包交付。配套修复将原生锁屏障切换至 V2 Guard,Hub 根路由退出旧 Resource/Intent 控制面;SQL 包通过不表示这些已退役功能重新可用。 + +## 文件与执行顺序 + +| 文件 | 数据库 | 用途 | +| --- | --- | --- | +| `00-create.sql` | Seafile 原生库 | 仅空的新安装,创建已确认四表;不用 IF NOT EXISTS 掩盖同名旧表 | +| `01-preflight.sql` | Seafile 原生库 | 只读查看实际结构、索引、记录数、Guard 状态 | +| `05-archive-legacy.sql` | Seafile 原生库 | 只读列出历史表与归档 SQL,审核后人工移到独立备份库 | +| `10-upgrade-029-042.sql` | Seafile 原生库 | 仅 3 列存储表、10 列组映射、15 列统一 ACL、16 列资源 Guard 的既有组合 | +| `20-seahub-compat.sql` | Seahub 原生库 | 显式补齐当前 CE14 模型兼容字段,按所发布模型审核后执行;不新增 cf_* 业务表 | +| `50-verify.sql` | Seafile 原生库 | 检查目标字段类型、长度、NULL、字符集/排序规则、必要索引和 CHECK 名称;失败抛错 | +| `checksums.json` | 不连接数据库 | 固定交付 SQL 的 SHA256;发布前核对 | + +新安装使用 `00 → 50`;已有规定旧结构使用 `01 → 10 → 审核执行 05 生成的归档 SQL → 50`。**DEV 已在 2026-10-21 完成四表转换,不再执行 00 或 10,只做校验。** 20 对应移出 bootstrap 的原生表字段修复,不属于四表建表动作;只为明确依赖这些字段的代码执行,不能把遗留扩展功能当成新版需求。其他历史集成表不随本包创建。 + +升级前显式创建受控独立归档库,并设置 `@cf_upgrade_archive_database`。升级脚本将原表移入该独立库,名称为 `backup_20261021_cf_*`,先创建 `stage_20261021_cf_*`、复制并核对所有行及原生组 ID/ACL 字段,再原子切换。ACL 文本 SHA256 转换成 BINARY(32);组名称等不再属于新表的旧元数据留在原表/外部备份。存储状态及 repo_key、repo_id 均保留。 + +以下情况拒绝执行:未确认停写、外键依赖、源结构不符或已升级、候选/备份已存在、任意 Guard 记录、文件级 ACL、长度/哈希不符、数据复制不一致。非空生命周期 ACL 必须先确认路径语义转换,不能暗中放宽生命周期约束。没有通用的活跃锁转换 SQL;有实际锁或待办 Intent 时先单独设计迁移。 + +## 手动执行示例 + +先备份数据库和持久存储,停止所有 Hub/Server/后台写入者,核对数据库与实际源结构。下例的受控客户端配置文件由运维预先管理,密码不放命令参数、Git 或 SQL。不要使用 `--force`。 + +```sh +mariadb --defaults-extra-file=/secure/cloudfile-client.cnf --database=seafile_db --batch \ + < database/v2/20261021-four-table-baseline/01-preflight.sql + +mariadb --defaults-extra-file=/secure/cloudfile-client.cnf --database=seafile_db --batch <<'SQL' +SET @cf_upgrade_writers_stopped=1; +SET @cf_upgrade_archive_database='cloudfile_legacy_archive'; -- DBA must create/back up this separate DB first +-- Only after reviewing existing lifecycle ACL semantics, if applicable: +-- SET @cf_upgrade_accept_path_acl=1; +SOURCE database/v2/20261021-four-table-baseline/10-upgrade-029-042.sql; +SQL + +mariadb --defaults-extra-file=/secure/cloudfile-client.cnf --database=seafile_db --batch \ + < database/v2/20261021-four-table-baseline/50-verify.sql +``` + +`@cf_upgrade_writers_stopped` 是操作者声明,不会替你停止进程。10 和 50 使用短暂存储过程提供明确 SIGNAL,执行账号需有相应权限;它们不是业务表或新的迁移台账。短过程 DDL 的权限也应单独审核。无存储过程权限时须由 DBA 以等价、同样可中止的方式执行,不可直接删掉检查。 + +脚本中断不会自动恢复原表。查看 `stage_*`/`backup_*`、原表名称及实际结构,确认是否完成切换后再决定下一步。客户端必须遇错中止;中断后遗留的同名辅助过程会在下次显式执行时清理,但业务表与备份不会自动删除。 + +## 验证和 DEV → PROD + +已在隔离 MariaDB 11.4 容器验证:新建与验收、旧结构迁移及组身份/ACL 二进制保持、重复执行拒绝、缺索引拒绝、非空 Guard 的 generation 保留、生命周期转换确认、Seahub 兼容 SQL 重放,以及只读启动检查。MySQL 8 及其它数据库版本须在该目标版本另行验证,不以 MariaDB 成功代替。 + +复现(已有镜像,不下载、不连接现有 DEV 数据库): + +```sh +python3 tests/test_manual_sql_integration.py --hub /path/to/cloudfile-hub +# Or use a dedicated test host with an existing mariadb:11.4 image: +python3 tests/test_manual_sql_integration.py --hub /path/to/cloudfile-hub --ssh-host test-host +``` + +DEV/PROD 都应记录 SQL 校验和、三仓源码 SHA、目标数据库版本、备份位置、前后结构/数量、执行人及时间。验收至少包含普通上传→目录可见→下载哈希一致、0B/1B/中文文件名、已有锁保护及新旧实例重启不改表。50 和启动校验会拒绝混入历史 cf 表。现有 DEV 尚保留历史表及备份,部署前须审核执行 05 生成的归档 SQL;不得为了通过启动校验删除数据或恢复旧表。四表启动不再接入旧 PolicyHost,显式启用旧入口的配置会在启动时被拒绝并列出变量名。本次不修改现有 DEV 数据库或部署镜像。 + +目标结构验收还需核对完整 SHOW CREATE(包括默认值与 CHECK 内容),50 不能代替所有业务语义和原生消费者测试。 + + +## 运行依赖收敛(2026-10-21) + +- 普通上传、下载、目录 ACL 与 Etech 目录同步使用现有 V2 入口;关闭 Checkout 时,C/Go/Hub 均不查询 Guard。 +- C fileop 锁屏障与普通写入事务只读 path-keyed Guard,不再关联 `cf_resource` 或查询 029 台账。活动租约阻止无受控凭据的写入(包括租约所有者);SQL 故障拒绝写入。 +- Docker 不再为四表运行时安装旧 PolicyHost Gunicorn hooks,仅删除自己管理的旧 hook 块,保留操作者其它 hook。`CLOUDFILE_*` 的旧 OIDC/授权/编辑/传输/注释/审计/搜索入口若显式开启,启动拒绝;不会自动修改环境变量。原生 OAuth、`/oidc/` 和 V2 API 不属于该旧入口。 +- Hub 根路由只保留当前原生文件/库接口、原生库配置、V2 兼容只读别名及部署扩展;不再挂载旧 SchemaRunner 管理、同步、会话、Intent 和 capabilities 入口。管理使用 `/api/v2.1/cloudfile/authorization/v1/directory-rules/`,组同步使用 `/api/v2.1/admin/cloudfile/directory/entities/sync/`。旧只读别名采用现有 V2 的参数和回执,不保留历史字段。 +- 兼容读取:`extensions/authorization/v1/{effective-permission,shared-directories,directory-entries}/` 直接使用 V2 视图;旧 `directory-address`、`library-rules`、`directory-shares` 等历史路由不再提供。不能用保留历史表来恢复它们。 + +本轮:配置/只读启动/SQL 契约 46 项;Hub 28 项及 7 个子用例;原生 C 锁屏障及最终事务 12 项;隔离 MariaDB 9 组场景(含从生产 C 抽取的空/活动/过期 Guard 查询)通过。未全量构建、部署或调整 DEV 环境;当前 DEV 历史表归档仍是部署前人工 SQL 步骤,不是启动时升级。 diff --git a/database/v2/20261021-four-table-baseline/checksums.json b/database/v2/20261021-four-table-baseline/checksums.json new file mode 100644 index 00000000..325dddd1 --- /dev/null +++ b/database/v2/20261021-four-table-baseline/checksums.json @@ -0,0 +1,8 @@ +{ + "00-create.sql": "a3e1f6468707517a424224504e371c705e5c0952ac5f46a2adef9812fea75ef5", + "01-preflight.sql": "7fedf2aacc78829023d4eb440c973aa85142389fa4db294c1464c704b615bcf7", + "05-archive-legacy.sql": "1f76e0917712209c92bd86c0e9269f50587760cbc2007f0d8054c9455777101d", + "10-upgrade-029-042.sql": "3c1f3f956bd6e126461b4a0b37fec076ebc83c24601e181de40e2d3bb3670b71", + "20-seahub-compat.sql": "d63f5de53e130821e10d2e9bcaea59fd87444eac81e8a6e3f9bf5a47b55cab2a", + "50-verify.sql": "8a7cb1970e8c55f89af3d20cc0741676382979a2b76378709c27cc17b15d64a7" +} diff --git a/docs/README.md b/docs/README.md index 145e1455..3f70cf15 100644 --- a/docs/README.md +++ b/docs/README.md @@ -13,6 +13,7 @@ | [项目概览](overview.md) | 基线、产品边界和事实口径 | | [当前架构](architecture.md) | 三仓职责、数据边界和故障隔离 | | [部署](deployment.md) | Compose 部署、升级、备份和恢复 | +| [数据库 SQL 交付](../database/README.md) | 开发期四表建模、独立 SQL 升级及 DEV → PROD 迁移 | | [配置](configuration.md) | 环境变量、profile 和配置生成规则 | | [扩展能力矩阵](feature-matrix.md) | 每项能力的状态、来源、定位、证据和上游策略 | diff --git a/docs/deployment.md b/docs/deployment.md index 1cc1ae11..8451796c 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -73,14 +73,12 @@ docker compose ps docker compose logs -f cloudfile ``` -核心栈包括 `cloudfile`、`db`、`cache` 和 `proxy`。首次启动会初始化数据库与 Seafile -配置;`scripts/scripts_14.0/start.py` 在每次启动时重写带 `CF_BEGIN`/`CF_END` 标记的 -CloudFile 配置块,并幂等应用 `cloudfile.sql`。 - -新部署在启动 Server/Hub 前使用随镜像交付的版本化 schema runner 创建并校验当前 -DDL(包括资源表、统一编辑锁表及迁移记录),避免只创建旧集成表而遗漏写入屏障依赖。 -初始化意图保存在共享卷 `.cloudfile-schema-init-pending`;失败时停止启动并保留标记, -重启后按正式 runner 的断点继续。既有数据目录且无该标记时不自动升级。 +核心栈包括 `cloudfile`、`db`、`cache` 和 `proxy`。原生 Seafile 数据库/配置初始化继续遵循对应成熟 CE 版本;CloudFile 开发期扩展数据库使用 [独立 SQL 包](../database/README.md),不得由容器启动自动升级。 + +CloudFile 新版当前只明确新增四张表:`cf_library_storage`、`cf_dir_acl`、`cf_sso_group_map`、`cf_edit_guard`。新部署手动执行匹配版本的 00-create.sql,既有部署按实际源结构选择升级脚本,再执行 50-verify.sql。Server/Hub 启动前做只读结构校验,缺表或布局不符即停止并提示人工执行 SQL,不再自动调用旧 SchemaRunner 或 bootstrap 兼容 DDL。历史 pending 标记不授予自动迁移权限。 + +DEV 到 PROD 使用经过 DEV 验证的同一 SQL 校验和及对应源码组合,不复制 DEV 业务数据。仅成熟版本之间另行审核自动升级机制。已运行容器的启动行为不会因仓库提交而立即改变,须随匹配镜像显式发布。 + 文件锁与 Checkout 仍默认关闭,建表不代表启用能力。 ## 可选 profile diff --git a/docs/dev-v2-four-tables-repair-20261021.md b/docs/dev-v2-four-tables-repair-20261021.md new file mode 100644 index 00000000..50fbe16c --- /dev/null +++ b/docs/dev-v2-four-tables-repair-20261021.md @@ -0,0 +1,50 @@ +# DEV V2 四表修复执行记录(2026-10-21) + +## 授权、目标和源码 + +用户明确要求“按照新的数据库表设计,直接修复 dev 环境中 cloudfile dev 中 v2 新增的 4 张 cf_* 表”。本次连接 minio157 的 cloudfile-dev 容器和 seafile_db,核对容器为运行中且未共享宿主 PID。没有操作 PROD、改环境变量、构建镜像、自动回滚或删除无关表。 + +采用 GitHub 最新 dev:Server 06efc4dea43c5200eea5dd2341f581536ad0838c 的 cloudfile-v2.sql 与 cloudfile-v2-edit.sql;Hub fc9e89c53636432e2a6da824a8758d1b0a374316 的 FreshSchema 进行精确复核。Docker 源码基线 0b47da4a811a8c3e240545f3ec6203526792b4d3。目标 DDL SHA256:`07565ddf1bce244d800135d81e085211970dc8bb145a71f0eea4f1824874270d`。 + +## 执行结果 + +| 表 | 新版结构 | 保留记录数 | +| --- | --- | --- | +| cf_library_storage | repo_key 主键、repo_id 唯一键、state 状态约束 | 102(active 74、retired 28) | +| cf_sso_group_map | provider/subject_type/namespace/external_id 复合主键、group_id BIGINT 且唯一 | 1143 | +| cf_dir_acl | 复合主键、BINARY(32) path_hash、共享主体索引及权限/继承/分享约束 | 3 | +| cf_edit_guard | repo_id/path_hash 主键、15 字段、cf_edit_guard_active 等必要索引和活跃行约束 | 0 | + +先检查四表无外键依赖,组映射字段长度/类型可容纳、ACL 原始 SHA256 文本与路径匹配,Guard 为空。三个 ACL 均为目录规则,可映射新版;没有把文件 ACL 静默提升为目录授权。ACL 中一条含旧 lifecycle_ref;该旧元数据随原表及独立备份保留,新表按已授权新版的路径语义转换,不伪造生命周期或版本。 + +转换保留所有存储状态、原生组 ID、规则主体、路径、权限、继承、分享模式和 revision。path_hash 从十六进制文本转换为对应的 32 字节二进制。组名称等新版不再存储的旧字段完整留在受控备份及旧表;新表不再保留临时 DEFAULT 空 name 的旧结构补丁。 + +工具 tools/repair-v2-four-tables.py 是显式一次性 DEV 转换器,不是启动钩子:先导出四表 DDL/全部记录,再创建候选表;短暂停住容器内 DB 写入服务,重新核对源 DDL/数据摘要,复制并核验全部目标记录,最后一条 RENAME TABLE 原子切换四个名称。任何源变化或 Guard 变为非空均拒绝切换。原进程在 finally 恢复;不自动回滚数据库,不自动重放旧迁移。 + +## 备份与变更边界 + +- 独立备份:容器内 `/shared/schema-backups/four-tables-20261020T174028Z_636083.json`,权限 0600;同目录有 receipt,权限 0600。目录权限 0700。内容包含业务映射,禁止提交备份文件。 +- 原表保留:`cfv2_old_20261020T174028Z_636083_library_storage`、`cfv2_old_20261020T174028Z_636083_sso_group_map`、`cfv2_old_20261020T174028Z_636083_dir_acl`、`cfv2_old_20261020T174028Z_636083_edit_guard`。 +- 时间标识使用 UTC;20261020T174028Z 对应北京时间 2026-10-21 01:40:28。 +- cf_resource、cf_commit_intent、cf_edit_session、cf_managed_library、cf_library_storage_key、cf_schema_migration 等均未删除或篡改。保留旧表不表示继续让旧代码按原布局写入四张新表。 +- 数据切换后不应再用旧 029/042 校验或重新执行旧迁移恢复旧字段;历史 ledger 是旧迁移事实,不能作为新四表契约通过的证明。完整实例仍包含其它 cf_* 表,此次不声称全库已经变成只有四表的全新安装。 + +## 已完成验证 + +- 修复前后全部四表目标投影的行数和数据摘要一致;备份中的源摘要已保留。 +- 用最新 FreshSchema._verify_one 逐表验证 SHOW CREATE TABLE,字段类型、长度、NULL/默认值、字符集/排序规则、索引及 CHECK 约束全部与目标 DDL 一致。 +- 原生 path Guard 查询 EXPLAIN 通过,原 cf_edit_guard_active 缺失导致的 SQL 1176 不再存在。 +- 没有遗留暂停的文件服务或 Python/Gunicorn 写入进程;普通上传后 cf_edit_guard 仍为 0。 +- 实际浏览器直传链路创建上传会话、传输及上传完成提交成功;1600 字节唯一测试文件目录可见,通过 EAP 下载票据回读为 HTTP 200,内容逐字节相同,SHA256 `63cdc6faac70060f4bcd1061a93fb5567da981eee2d5f08121bac4d60fc03dd9`。 +- EAP 中转上传实测 0B、1B、中文文件名 1KB 三例均返回 code=0/data=true,目录实际条目与大小匹配,下载票据回读 HTTP 200 且内容逐字节一致,排除了仅返回成功但未落盘的假成功。 +- 旧测试脚本把原生签名 URL 改写到旧反代路径,最后下载曾得到 404;正确 EAP 票据路径回读通过。单独的原生 URL 反代路径问题不以本次四表变更替代修复。 +- 转换工具 6 项定向测试通过,现有跨仓 SQL 门禁通过;新增转换测试接入 tools/run-checks.sh。测试过程不调用 Checkout/Checkin;普通上传会话是文件传输会话,并非编辑签出会话。 + +## 后续兼容事项 + +四表实际修复不代表所有旧调用者已经迁移。Docker 的 bootstrap 每次启动仍含集成 DDL,旧版本化 SchemaRunner 仍识别 resource/统一 ACL 结构;已编译 C cf-lock.c/cloudfile-policy.c 的部分路径及旧目录同步写入也仍有旧字段依赖。启动 Schema Owner、旧入口退出及所有原生锁/ACL 写入消费者需按 [方案对比](upload-plan-comparison-20261021.md)继续统一,发布或重启前必须覆盖相关测试。不能靠重新加旧字段、补默认值或清除真实锁来掩盖新旧代码不一致。 + + +### 后续源码修复状态 + +启动已取消自动扩展 DDL;Docker 四表运行时不再加载旧 PolicyHost。原生 C 锁屏障/普通写终判已直接使用 V2 Guard,Hub 旧控制面路由已退出,V2 读取别名不再进入旧 NativeRead。现有 DEV 的历史表及同库备份仍须按独立 SQL 审核归档后再部署;本轮没有操作现有 DEV 表、环境变量或容器。详见 [运行依赖收敛](../database/v2/20261021-four-table-baseline/README.md)。 diff --git a/docs/directory-schema-upgrade.md b/docs/directory-schema-upgrade.md index 3654214c..164216c1 100644 --- a/docs/directory-schema-upgrade.md +++ b/docs/directory-schema-upgrade.md @@ -1,3 +1,7 @@ +# 历史目录 Schema 迁移说明 + +> 2026-10-21 起,当前新版只确认四张新增业务表,开发期统一使用 [独立 SQL 交付](../database/README.md)。本文的 008/009/042、cf_dir_admin 及 Python apply 命令仅供对应历史版本追溯,不是新版初始化或发布入口。 + # Directory authority schema `cf_dir_acl` and `cf_dir_admin` belong to the native Seafile diff --git a/docs/upload-plan-comparison-20261021.md b/docs/upload-plan-comparison-20261021.md new file mode 100644 index 00000000..c7eaab9a --- /dev/null +++ b/docs/upload-plan-comparison-20261021.md @@ -0,0 +1,45 @@ +# 上传修复方案对比与补充(2026-10-21) + +> 后续状态:用户已另行授权并完成 DEV 四表转换,详见 [实际执行记录](dev-v2-four-tables-repair-20261021.md)。本文盘点及方案差异描述对应切换前状态,最新数据库状态以执行记录为准。 + +依据用户提供的《CloudFile 上传挂起与数据库表结构检查》全文,与本分支上传修复、实际 DEV 盘点及对应源码比较。引用会话读取超时,比较依据是用户贴出的正文;其中“会修改并创建 PR”的文字只是计划,不构成另一个会话已经实施的证据。 + +## 一致结论 + +普通上传下载不需要 Checkout/Checkin 会话。表存在不能证明 Guard 契约兼容;SQL 1176 只是错误布局的首个症状,单补索引不能补齐字段与数据语义。确定性的权限、Guard、Schema 故障不能重试十次;只允许真实 Branch HEAD 并发冲突重试。不得关闭锁保护、自动删表或重置 generation。 + +## 实质差异与合并后的处理 + +| 项目 | 引用方案 | 本分支事实及补充 | +| --- | --- | --- | +| 恢复上传 | 同时修错误重试与统一 V2 Schema 入口 | 已修重试和按已安装完整结构选择锁查询;先恢复现有实例的正确提交,不以数据库转换作为上传前置条件 | +| Schema 目标 | 三表核心加可选一张 Guard 作为目标 | 这是独立 fresh 契约,当前运行 C/Hub/启动入口仍有版本化表依赖。统一为该目标需要同时调整所有调用者,不能只替换初始化器 | +| DEV 证据 | 未连库,表结构和 Guard 数未知 | 已读到 50 张 cf_* 表、42 项 applied 迁移,最新 runner 校验通过;盘点时 Guard 记录 0,三仓运行 SHA 已记录。迁移执行前必须重新盘点,不能复用旧的零记录结果 | +| 存储绑定 | 提醒 cf_library_storage 风险 | 实际同时有 cf_library_storage(repo_key,repo_id,state) 和 cf_library_storage_key(repo_key,repo_id),各有主键和 repo_id 唯一键;结构共存不证明绑定记录或存储路由一致 | +| C 原生写保护 | 提醒 common/cf-lock.c 的资源联表 | 确认被 server/Makefile.am 编译,读取 029 ledger、resource_uid、mode、owner_native_user、lifecycle_ref;server/cloudfile-policy.c 也依赖资源 Guard 与统一 ACL。迁移必须覆盖这些实际读写者 | +| 启动 DDL | 指出 SchemaRunner 与 bootstrap 并存 | 确认 start.py 先 write_cloudfile_config,再 initialize_fresh_schema。前者每次启动执行集成表 DDL;后者仅新安装标记执行 42 项迁移。现有测试只覆盖后者,不证明整个启动只读 | +| EXDEV | 保留独立排查项 | 最新 Server 已含 31a51f3 的同目录临时文件、原子 rename、fsync 修复;源码存在不等于运行镜像已覆盖目标挂载,仍需 Linux 挂载场景与部署后读回验收 | +| EAP 假成功 | 列为独立链路问题但未给此次修复 | 本分支已检查 HTTP 状态和完整同步回执;错误、异步结果、重定向不报成功,自动编号按实际名称记账 | +| 其它写入契约 | 主要比较 DDL | 补充 GroupCatalog 新建映射漏 name、ACL 内部写入二进制哈希及必填字段与现有统一表不匹配,不能把迁移 ledger 通过当成全部业务写入通过 | + +## 本次实际补充 + +Go 最终锁查询之前的结构识别收紧为完整 15 列 path 布局或完整 16 列 resource 布局,与 Hub 的完整集合校验一致。拒绝之前可被五个字段子集误识别的残缺表,也拒绝夹杂另一布局字段的表;增加相关回归测试。已有锁、未知结构或 SQL 错误继续拒绝写入,不创建 Checkout 会话。 + +SQL 门禁的边界须明确:当前校验 CREATE 与 Hub 读取合同,并固定 45 份 SQL/迁移文件摘要,能阻止这些输入未经审查变化;它不自动证明全部 C/Go/Python 读写 SQL 与运行数据库兼容,也不覆盖所有 bootstrap 内联 DDL。迁移测试和完整启动测试不能由摘要门禁代替。 + +## 后续实施顺序与验收条件 + +1. **上传修复交付**:固定 Server/Hub/Docker/EAP 提交及构建收据,核对运行镜像,部署后以普通 API 上传并下载回读;不调用 Checkout/Checkin。出现不兼容结构直接失败,检查日志不再十次重试。完整构建及 DEV 部署仍等待明确授权。 +2. **确定唯一 Schema 管理入口**:先登记每张表的实际消费者、初始化器、升级器和当前数据;把 bootstrap 的所有扩展 DDL 纳入清单。新安装只有一个明确 DDL 入口;已有卷启动只做契约验证,升级另行显式执行。修改启动入口前必须补齐它目前创建的被依赖表,不能简单删调用。 +3. **完成读写契约统一**:同时处理 Go 最终提交、已编译 cf-lock.c、cloudfile-policy.c、Hub 编辑/组目录/ACL、存储绑定读写与冷读/GC。核心表校验应包含类型、长度、字符集/排序规则、NULL/默认值、索引唯一性/列顺序/前缀、引擎及生命周期语义。明确未选目标布局的支持期限,避免无期限双契约。 +4. **离线迁移设计**:迁移前重新只读统计有效、过期、挂起 Guard 和 Intent/Session,检查孤立 resource、generation、凭据 epoch、基础版本及路径关联;盘点两张存储绑定表的键冲突、悬空库和对应对象路径。先形成每类记录保留/转换/拒绝迁移规则,再安排停止全部写入者、备份及恢复演练。DDL 非事务,不可承诺自动回滚;保留不可改写的历史校验和和可恢复的逐步执行记录。数据转换、环境变量和回滚另行明确审批。 +5. **验收矩阵**:新安装、已有当前布局、已识别的历史布局、残缺/混合布局、迁移中断重启分别测试。普通文件覆盖 0B/1B/1KB/64KB/1MB、中文/空格名、同名覆盖与保留编号、子目录、直传和 EAP 中转;每例检查目录实际名称、大小、下载哈希。批量中途失败不得整批假成功,说明之前成功的文件不自动回滚。锁定文件拒绝越权覆盖、同库无锁文件可写,下载无需 Checkout;并发取得锁与 Branch 提交不能绕过终判。 + +EXDEV 另做隔离 Linux 挂载测试,覆盖后端目录、库目录和分片目录 bind mount;确认失败不会发布半文件、重试不会制造错误完成回执。当前 macOS 定向测试不能代替这些 Linux 用例。部署验收保留镜像 Digest、提交 SHA、时间、状态码及字节哈希,日志不包含账号凭据或上传票据 URL。 + +## 完成状态 + +本次对比、文档补充和 Go 完整结构校验已经实施;统一启动 DDL、业务表数据转换、Linux 挂载集成测试及 DEV 交付尚未执行。没有根据引用方案直接替换建表器、迁移/删除表、重建容器或改变环境变量。引用方案承诺的 PR/合并状态需要真实 Git/PR 证据才能确认。 + +补充修复提交:Server `92616b43a58712caed0b3da25e75abe8a84d8a54`。Go Guard 专项测试通过;对象同目录原子落盘与并发发布两项定向测试通过(不含 Linux bind mount 场景)。 diff --git a/docs/upload-schema-audit-20261021.md b/docs/upload-schema-audit-20261021.md new file mode 100644 index 00000000..97ea0d3a --- /dev/null +++ b/docs/upload-schema-audit-20261021.md @@ -0,0 +1,62 @@ +# 普通上传故障与新增表 SQL 审查(2026-10-21) + +> 后续状态:用户已另行授权并完成 DEV 四表转换,详见 [实际执行记录](dev-v2-four-tables-repair-20261021.md)。本文盘点及方案差异描述对应切换前状态,最新数据库状态以执行记录为准。 + +## 原则与源码基线 + +普通文件上传、下载不创建、不续租、不提交 Checkout/Checkin 会话。已有锁定文件仍须在最终 Branch 事务中防止被无票据覆盖,不能通过删除锁表或忽略查询错误恢复上传。 + +本次四个独立工作区重新拉取 GitHub dev 并核对祖先关系:Server `06efc4dea43c5200eea5dd2341f581536ad0838c`、Hub `fc9e89c53636432e2a6da824a8758d1b0a374316`、Docker `0b47da4a811a8c3e240545f3ec6203526792b4d3`、EAP `876e7a7f8d4e6ef2ba588b11f27a58b214ff1c51`。DEV 容器实际 Server/Hub/Docker 分别为 `366f5db742dbd8b8bf08add300b14068d9cfe193`、`f8a396a46a2a14cbac5da9e6fed9319c52d57a20`、`38be08c71e6ff7847b75ebdb35040ba986f65612`;不能把本地最新分支当成已部署版本。 + +## 已确认上传根因及修复 + +DEV cf_edit_guard 是 16 字段的 resource_uid 主键布局,索引 PRIMARY、guard_identity;Go 最终提交却按另一套 path/lease_expires_at 布局强制读取 cf_edit_guard_active 索引。只检查表存在导致 SQL 1176,错误又被当作并发冲突重试 10 次。原生日志已有 stop updating repo after 10 retries,故不是文件体积或 Checkout 会话缺失。 + +Server 修复按实际字段区分已安装布局,并在同一 Branch 事务中查询。资源锁通过 cf_resource 解析路径;孤立资源、生命周期错配、未知/混合结构、SQL 或差异计算失败均拒绝,不跳过已有锁。空锁集合允许普通提交。仅 Branch 头变化返回并发重试,Guard/结构失败直接报错。原生修改 fileop.go 已在 upstream-patches/cloudfile-server.txt 登记;该原生最终提交点不可由上层预检替代。common/cf-v2-edit-fence.c 同样含旧路径查询,但本次源码中只有声明/定义,没有调用点,Makefile 未编入 C 源,记录为未启用历史实现,不能据此声称 C 终判已迁移。 + +Hub 原先仅查表存在即开放可选 Rust/path Checkout,已改为核对列集合和完整必要索引;已安装 resource 布局不会冒充 path 能力。普通 local-view/optimistic-edit 不因缺少 path Checkout 被强制签出。 + +EAP 修复只有 200/201 加完整原生单文件回执才能报成功,并核对名称、字节数、文件 ID;错误页、异步任务和重定向均拒绝。同名编号按实际文件名记账。失败终止后续批次,不记成功日志;批量上传不提供事务回滚。未增加 Checkout API 调用。 + +## SQL 审查与其他风险 + +检查 3 份 Server SQL 和 42 份 Hub 迁移,共 45 份输入、47 条 CREATE、44 个表名。发现三类同名不兼容布局: + +| 表 | 隔离 fresh SQL | 现有 DEV 迁移布局 | 处理 | +| --- | --- | --- | --- | +| cf_edit_guard | path_hash/path、lease_expires_at、15 字段 | resource_uid/lifecycle_ref、lease_until/hard_expire_at、16 字段 | 已修复 Go/Hub 对布局的误判;不修改现有表 | +| cf_dir_acl | 11 字段、复合主键、BINARY(32) path_hash | 008 建 13 字段,042 再加 share_mode/lifecycle_ref,总 15 字段;id/subject_hash/kind、CHAR(64) path_hash | 禁止把 fresh DDL 套入当前卷;当前原生 cloudfile-policy.c 使用统一迁移布局 | +| cf_sso_group_map | 5 字段、复合主键、group_id BIGINT | 006 的 10 字段,id 自增,name 非空无默认值、group_id INT | 新建映射必须使用与该布局匹配的写入契约 | + +额外静态风险:cloudfile_ext/v2/memberships.py 的 GroupCatalog 新建映射仅写 5 字段,在现有 006 表布局下未提供必填 name;创建原生组 RPC 先于映射 INSERT,可能留下无映射组。cloudfile_ext/v2/acl_store.py 的内部 mutate 使用 fresh 11 字段及二进制 path_hash,不能直接写现有 15 字段统一 ACL 表(缺少 id/kind/subject_hash 等)。已有映射查询不等于新增写入已验证。此次未在 DEV 创建业务组或新增授权以触发这些风险,故不称为线上复现;它们不是普通文件上传故障根因。 + +推荐后续方案:现有实例维持 42 项迁移布局为事实源,分别为组目录和 ACL 写入补齐完整类型/必填列/唯一键/读回契约,先确认具体运行调用路径,再决定统一写入适配;不要用补默认空 name、删列、重建表或整枝复制旧实现掩盖不一致。ACL 还必须保持原生 Branch 事务、权限及生命周期终判。已应用迁移不可改写;确需变更用新增迁移并做显式版本验证。当前发现的两项额外风险未纳入本次上传补丁,不应把 SQL 门禁通过等同于所有业务写入兼容。 + +release.yaml 的 database_schema: 3 是旧发布元数据,不等于当前迁移版本。已有持久卷不会仅因新镜像执行 initialize_fresh_schema 的整套版本迁移,但 write_cloudfile_config → apply_cloudfile_schema 仍在每次启动执行 cloudfile.sql 的 CREATE IF NOT EXISTS,并执行多项 compatibility 建表/改列逻辑;不能把新安装标记测试通过理解成已有卷启动完全不写 DDL。fresh schema 是独立新库合同,不是现有卷的升级脚本。 + +## 新增防回归门禁 + +Docker tools/check-sql-contracts.py 只读检查同名 CREATE 冲突以及 Hub Checkout 读取合同与 Server DDL 的字段/索引一致性;tools/reviewed-sql-contracts.json 固定全部 45 份已检查输入的摘要,涵盖字段类型、默认值、约束、ALTER/DROP 和索引 SQL 的变化。已知同名表并非允许任意变化的白名单。更新/新增/删除迁移或 DDL 必须重新审查并显式更新基线,不提供自动刷新基线命令;摘要门禁不能代替 SQL 语义和调用方审查。两项检查接入 tools/run-checks.sh。 + +## 验证证据及未完成事项 + +- DEV 只读抓取:50 张 cf_* 表,42 项迁移均 applied,无未完成步骤。 +- 使用最新 Hub 源码 SchemaRunner.require_current(),连接包装器仅允许 SELECT,42 项迁移校验和、实际 verify SQL、目录布局及索引契约全部通过。没有执行 DDL/DML。 +- 资源锁替代 SQL 已在 DEV EXPLAIN 和只读事务查询通过,当前锁记录为 0;事务立即结束。不能代替真实并发锁行为验收。 +- Go 上传/已有锁/托管库写保护定向测试通过;Hub 结构识别 4 项、Docker SQL 漂移 5 项通过;EAP JDK17 定向测试 95 项通过。脚本语法及差异空白检查通过。 +- 未全量构建、替换镜像、重启服务或修改 DEV 环境变量。尚不能声称上传已恢复。后续须确认构建/DEV 部署,再验收普通上传→目录可见→下载字节一致、已有锁阻止覆盖、无锁文件正常传输及失败不报成功。 + +## 本次修复提交 + +- etech-eap-upload-result: `14a1792eacae16546baddadc6c6a88b727953070` +- cloudfile-upload-guard: `578d9fd51c73b5a70d4a3ebdbe0f627e710ead76` +- cloudfile-upload-guard-hub: `b3fa0b2d256d5a7286a3ee85da54fd9a97a7e11c` + +Docker 门禁及本报告随同一审查提交交付;均尚未部署。 + +## 方案对比补充 + +参见 [跨方案对比与补充计划](upload-plan-comparison-20261021.md)。补充了实际参与编译的 common/cf-lock.c、存储绑定两表共存、bootstrap 启动 DDL 边界,以及 EXDEV 和离线迁移验收;与尚未接入的 common/cf-v2-edit-fence.c 区分。 + + +后续收敛:生产 C `common/cf-lock.c` 与 `cf_policy_check_unleased_write` 已改为 V2 path Guard,不再查询 Resource 或旧迁移台账。`python3 tests/cf-lock/run.py` 在 Server 仓编译完整生产锁屏障和 fileop 分发器及最终事务,12 项通过;Docker 隔离 MariaDB 另执行从生产 C 抽取的 SQL,空/活动/过期租约均符合预期。Go 的此前验证继续有效。未执行完整 Linux C 构建或镜像联验。 diff --git a/docs/upstream-patches/cloudfile-docker.txt b/docs/upstream-patches/cloudfile-docker.txt index 550de1e1..cce53434 100644 --- a/docs/upstream-patches/cloudfile-docker.txt +++ b/docs/upstream-patches/cloudfile-docker.txt @@ -8,5 +8,7 @@ .gitignore README.md build/README.md +# Development DDL is delivered as manual SQL; runtime config must not repair tables. scripts/scripts_14.0/bootstrap.py +# Startup performs a read-only four-table contract gate before writers start. scripts/scripts_14.0/start.py diff --git a/docs/upstream-patches/cloudfile-server.txt b/docs/upstream-patches/cloudfile-server.txt index 733b3a00..abb2a64e 100644 --- a/docs/upstream-patches/cloudfile-server.txt +++ b/docs/upstream-patches/cloudfile-server.txt @@ -22,12 +22,15 @@ common/obj-backend.h common/obj-store.c common/obj-store.h configure.ac +# Upload repair: preserve the final Guard fence; retry only Branch-head conflicts +# and surface Guard/schema errors immediately. Native publication owns this seam. fileserver/fileop.go fileserver/sync_api.go fileserver/go.mod fileserver/go.sum fileserver/objstore/objstore.go fileserver/objstore/objstore_test.go +# Existing checkout_enabled gates Guard discovery; table presence never enables Checkout. fileserver/option/option.go include/seafile-rpc.h fuse/Makefile.am @@ -60,3 +63,6 @@ common/user-mgr.h fileserver/fileserver.go server/gc/seafile-session.c + +# 2026-10-21:cf-lock.c/h 与 cloudfile-policy.c 是独立 CloudFile 文件。 +# C 普通写入/锁屏障使用 V2 单 Guard,Checkout 关闭不查询锁;不改原生调用点。 diff --git a/docs/v2/cf-edit-feature.md b/docs/v2/cf-edit-feature.md index cf1fb288..f616ebc5 100644 --- a/docs/v2/cf-edit-feature.md +++ b/docs/v2/cf-edit-feature.md @@ -62,3 +62,10 @@ Checkout 本身是独占机制,但普通编辑**不申请锁、不续租、不 只有 `cf_edit_guard` 为本功能扩展状态;**不重新引入服务器 Intent、设备账户、编辑 Session 或历史锁表。** + + +## Checkout 与普通传输的边界(2026-10-21) + +Checkout 尚未完成,不得阻塞现有普通上传、下载或乐观编辑。当前复用已有 `CF_ENABLE_CHECKOUT`(原生配置 `[cloudfile] checkout_enabled`),默认 false;关闭时不探测/查询 Guard,不开放独占编辑操作,启动只校验基础表。显式启用后才校验 Guard,查询失败仍拒绝受保护的提交。 + +库存储中配置“是否支持 Checkout、默认不支持”仅为后续规划;本次不新增字段,不修改 `cf_library_storage` 结构或数据。库级设计确认并进入实施时,再单独交付 SQL。 diff --git a/release.yaml b/release.yaml index f0c3db9b..4dff47c1 100644 --- a/release.yaml +++ b/release.yaml @@ -65,12 +65,9 @@ ce_anchor: seafile_server: 17e4b875c2118233830316392ed76a736b374c69 seahub: 3c408439bfca54dbad61e2f6cdb706d2dcdf9f7a -# Bump when cf_* tables change shape; the upgrade path keys off this. -# 1 -> 2 (2026-08-17): added cf_dir_admin (directory-level delegated manage). -# Additive, IF NOT EXISTS, auto-applied by the bootstrap on every start -- no -# manual migration step. -# 2 -> 3 (2026-10-08): reserve automatic library keys in cf_library_storage_key. -# Additive bootstrap table; retain reservations for cold GC and key uniqueness. +# Historical manifest counter retained for receipt compatibility only. +# It is not a schema owner and must not trigger bootstrap DDL. V2 uses the +# fixed, manually executed SQL package in database/v2/20261021-four-table-baseline/. database_schema: 3 # Bump when cloudfile_ext.registry hook signatures change. extension_api: 1 diff --git a/scripts/scripts_14.0/bootstrap.py b/scripts/scripts_14.0/bootstrap.py index 1df78c2d..d2373127 100755 --- a/scripts/scripts_14.0/bootstrap.py +++ b/scripts/scripts_14.0/bootstrap.py @@ -1384,12 +1384,10 @@ def write_cloudfile_config(): write_cloudfile_seafile_conf() write_seafevents_search_config() write_seafevents_audit_config() - apply_cloudfile_schema() - apply_sso_hierarchy_schema_compatibility() - apply_metadata_schema_compatibility() - apply_tag_schema_compatibility() - apply_starred_obj_id_schema_compatibility() - apply_audit_schema() + # Development upgrades (including compatibility columns) are explicit SQL + # releases. Restarting/config generation must not mutate a database or + # bring back retired fields after the four-table V2 conversion. + # Historical helpers remain for source traceability, not startup execution. def apply_sso_hierarchy_schema_compatibility(): diff --git a/scripts/scripts_14.0/cloudfile.py b/scripts/scripts_14.0/cloudfile.py index deed941a..13d0c0b6 100644 --- a/scripts/scripts_14.0/cloudfile.py +++ b/scripts/scripts_14.0/cloudfile.py @@ -252,9 +252,9 @@ def render_settings(environment=None): webdav_enabled = _boolean(environment, "CLOUDFILE_WEBDAV_ENABLED", False) authorization_enabled = _boolean(environment, "CLOUDFILE_AUTHORIZATION_ENABLED", False) local_edit_enabled = _boolean(environment, "CLOUDFILE_LOCAL_EDIT_ENABLED", False) - # Native administrator policy uses existing SQL/account authority, not OIDC. - # The session route remains an explicit alternative owning the same paths. - admin_policy_enabled = _boolean(environment, "CLOUDFILE_ADMIN_POLICY_ENABLED", not authorization_enabled) + # The old administrator control plane requires the retired multi-table + # schema. Current V2 DirectoryRules owns management and stays available. + admin_policy_enabled = _boolean(environment, "CLOUDFILE_ADMIN_POLICY_ENABLED", False) if admin_policy_enabled and authorization_enabled: raise ValueError("administrator policy requires distinct route ownership") editing_enabled = _boolean(environment, "CLOUDFILE_EDITING_ENABLED", False) @@ -422,10 +422,27 @@ def write_settings(path, environment=None): return True -def write_policy_worker_hooks(path, environment=None): +def write_policy_worker_hooks(path, environment=None, *, v2=False): """Manage only our block; preserve and compose existing Gunicorn hooks.""" environment = os.environ if environment is None else environment enabled = _boolean(environment, "CLOUDFILE_POLICY_WORKER_HOOKS", False) + if v2: + # The old PolicyHost validates the 42-migration Resource/Intent schema. + # V2 native OAuth, ACL and transfers use their direct adapters instead. + # Never silently enable retired routes or start that host on four tables. + retired = ( + "CLOUDFILE_OIDC_ENABLED", "CLOUDFILE_ADMIN_POLICY_ENABLED", "CLOUDFILE_AUTHORIZATION_ENABLED", + "CLOUDFILE_TRANSFER_ENABLED", "CLOUDFILE_EDITING_ENABLED", + "CLOUDFILE_LOCAL_EDIT_ENABLED", "CLOUDFILE_ANNOTATIONS_ENABLED", + "CLOUDFILE_AUDIT_QUERY_ENABLED", "CLOUDFILE_AUDIT_EXPORT_ENABLED", + "CLOUDFILE_RESOURCE_SEARCH_ENABLED", + ) + requested = [name for name in retired if _boolean(environment, name, False)] + if requested: + raise ValueError("V2 does not support retired schema-dependent routes: " + ", ".join(requested)) + # Removing our generated block preserves operator hooks. It does not + # change environment variables, schema, or native V2 permissions. + enabled = False begin_marker = "# BEGIN CLOUDFILE POLICY WORKER HOOKS" end_marker = "# END CLOUDFILE POLICY WORKER HOOKS" try: diff --git a/scripts/scripts_14.0/cloudfile_schema.py b/scripts/scripts_14.0/cloudfile_schema.py index 255c197d..9bc139ba 100644 --- a/scripts/scripts_14.0/cloudfile_schema.py +++ b/scripts/scripts_14.0/cloudfile_schema.py @@ -1,32 +1,78 @@ -"""Initialize the shipped schema only for a new, persistent installation.""" - +"""Read-only startup gate; development DDL is applied with reviewed SQL files.""" import os -from pathlib import Path -import sys +# Keep these names aligned with database/v2/20261021-four-table-baseline/00-create.sql. +# Table existence alone previously enabled an incompatible Guard on every upload. +COLUMNS = { + 'cf_library_storage': 'repo_key repo_id state', + 'cf_sso_group_map': 'provider subject_type namespace external_id group_id', + 'cf_dir_acl': 'repo_id path_hash path subject_type provider namespace external_id permission inherit share_mode revision', + 'cf_edit_guard': 'repo_id path_hash path guard_id generation owner_uid owner_native_user terminal_type terminal_name checkout_ip proof_digest base_file_id lease_expires_at created_at updated_at', +} +INDEXES = { + 'cf_library_storage': {'PRIMARY': (0, ('repo_key',)), 'library_storage_repo': (0, ('repo_id',))}, + 'cf_sso_group_map': {'PRIMARY': (0, ('provider','subject_type','namespace','external_id')), 'sso_native_group': (0, ('group_id',))}, + 'cf_dir_acl': {'PRIMARY': (0, ('repo_id','path_hash','subject_type','provider','namespace','external_id')), + 'acl_shared_subject': (1, ('provider','subject_type','namespace','external_id','share_mode','repo_id'))}, + 'cf_edit_guard': {'PRIMARY': (0, ('repo_id','path_hash')), 'cf_edit_guard_id': (0, ('guard_id',)), + 'cf_edit_guard_active': (1, ('repo_id','guard_id')), 'cf_edit_guard_owner': (1, ('owner_uid','guard_id'))}, +} + + +def verify_v2_schema(connection, checkout_enabled=False): + """Check the installed core contract without CREATE, ALTER, DML or repair. -def prepare_fresh_schema(shared_dir): - shared_dir = Path(shared_dir) - marker = shared_dir / '.cloudfile-schema-init-pending' - # Persist intent before CE creates seafile-data: a failed DDL attempt must - # retry after restart, while existing installations never opt into upgrades. - if not (shared_dir / 'seafile-data').exists(): - marker.touch(mode=0o600, exist_ok=True) - return marker + This is a startup gate, not a replacement for release SQL/data validation. + The four-table baseline includes the optional edit table, but disabled + Checkout does not qualify or require that table for ordinary file transfers. + """ + with connection.cursor() as cursor: + # Legacy tables are migration inputs/archives, never runtime fallbacks. + # Reject mixed databases even when the four new tables themselves match. + cursor.execute("SELECT TABLE_NAME FROM information_schema.TABLES " + "WHERE TABLE_SCHEMA=DATABASE() AND LOWER(LEFT(TABLE_NAME,2))='cf'") + installed = {row[0] for row in cursor.fetchall()} + required = set(COLUMNS) if checkout_enabled else set(COLUMNS) - {'cf_edit_guard'} + if not required <= installed or installed - set(COLUMNS): + raise ValueError('runtime database contains missing core or unapproved CloudFile tables') + for table, names in COLUMNS.items(): + # An unfinished, disabled Checkout must never prevent file service + # startup because its Guard is missing or has an incompatible shape. + if table == 'cf_edit_guard' and not checkout_enabled: + continue + cursor.execute('SELECT ENGINE,TABLE_COLLATION FROM information_schema.TABLES ' + 'WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=%s', (table,)) + if cursor.fetchall() != (('InnoDB', 'utf8mb4_bin'),): + raise ValueError('core table engine/collation mismatch') + cursor.execute('SELECT COLUMN_NAME,DATA_TYPE,CHARACTER_MAXIMUM_LENGTH ' + 'FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=%s', (table,)) + columns = {name: (kind, length) for name, kind, length in cursor.fetchall()} + if set(columns) != set(names.split()): + raise ValueError('core column layout mismatch') + if table in ('cf_dir_acl', 'cf_edit_guard') and columns['path_hash'] != ('binary', 32): + raise ValueError('core path hash must be BINARY(32)') + if table == 'cf_sso_group_map' and columns['group_id'][0] != 'bigint': + raise ValueError('native group identity must be BIGINT') + cursor.execute('SELECT INDEX_NAME,NON_UNIQUE,SEQ_IN_INDEX,COLUMN_NAME,SUB_PART ' + 'FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=%s ' + 'ORDER BY INDEX_NAME,SEQ_IN_INDEX', (table,)) + indexes = {} + for name, non_unique, sequence, column, prefix in cursor.fetchall(): + unique, parts = indexes.setdefault(name, (non_unique, [])) + if prefix is not None or unique != non_unique or sequence != len(parts) + 1: + raise ValueError('core index contract mismatch') + parts.append(column) + actual = {name: (unique, tuple(parts)) for name, (unique, parts) in indexes.items()} + if any(actual.get(name) != value for name, value in INDEXES[table].items()): + raise ValueError('required core index missing or changed') -def initialize_fresh_schema(marker, installdir, environment=None): - marker = Path(marker) - if not marker.exists(): - return False +def verify_development_schema(installdir=None, environment=None): + """Fail startup with an operator action, never migrate a development DB.""" environment = os.environ if environment is None else environment - # Import the canonical runner without django.setup(): schema must exist - # before enabled applications or native write barriers can access it. - sys.path.insert(0, str(Path(installdir) / 'seahub')) connection = None try: import pymysql - from cloudfile_extensions.schema.runner import SchemaRunner connection = pymysql.connect( host=environment['SEAFILE_MYSQL_DB_HOST'], port=int(environment.get('SEAFILE_MYSQL_DB_PORT', '3306')), @@ -35,15 +81,14 @@ def initialize_fresh_schema(marker, installdir, environment=None): database=environment.get('SEAFILE_MYSQL_DB_SEAFILE_DB_NAME', 'seafile_db'), charset='utf8mb4', autocommit=True, connect_timeout=5, ) - # apply() verifies DDL and checksums under the runner's database lock. - # Keep the marker on failure so partial MySQL DDL can resume safely. - SchemaRunner(connection).apply() - marker.unlink() - return True + requested = str(environment.get('CF_ENABLE_CHECKOUT', 'false')).lower() + if requested not in ('true', 'false'): + raise ValueError('invalid Checkout setting') + verify_v2_schema(connection, checkout_enabled=requested == 'true') except Exception: - # Driver errors can contain credentials; expose only an actionable cause. - raise RuntimeError('Fresh CloudFile schema initialization failed; inspect the schema ledger before retry') from None + # Driver details may contain credentials. Never invoke a legacy runner + # or restore old fields when this fails: use the fixed release SQL pack. + raise RuntimeError('CloudFile development schema check failed; stop writers and apply the reviewed database/v2 SQL package before starting services') from None finally: if connection is not None: connection.close() - sys.path.pop(0) diff --git a/scripts/scripts_14.0/start.py b/scripts/scripts_14.0/start.py index 38f1cf44..ec2d604d 100755 --- a/scripts/scripts_14.0/start.py +++ b/scripts/scripts_14.0/start.py @@ -21,7 +21,7 @@ from bootstrap import init_seafile_server, write_cloudfile_config from cloudfile import write_settings as write_cloudfile_settings from cloudfile import write_policy_worker_hooks -from cloudfile_schema import prepare_fresh_schema, initialize_fresh_schema +from cloudfile_schema import verify_development_schema shared_seafiledir = '/shared/seafile' @@ -59,12 +59,9 @@ def main(): print(e) wait_for_mysql() - # Record fresh-install intent before CE creates its data directory. This - # initializes current DDL without upgrading existing deployments on restart. - schema_marker = prepare_fresh_schema(shared_seafiledir) init_seafile_server() + write_policy_worker_hooks(join(topdir, 'conf', 'gunicorn.conf.py'), v2=True) write_cloudfile_settings(join(topdir, 'conf', 'seahub_settings.py')) - write_policy_worker_hooks(join(topdir, 'conf', 'gunicorn.conf.py')) check_upgrade() @@ -72,7 +69,9 @@ def main(): # rearranged conf/, so flipping a CF_ENABLE_* switch in .env takes effect # on restart rather than only at first bootstrap. write_cloudfile_config() - initialize_fresh_schema(schema_marker, installdir) + # Development schemas are upgraded by an operator using fixed SQL files. + # Refuse incompatible tables instead of silently replaying legacy DDL. + verify_development_schema(installdir) # Legacy defaults must load before the native extension overrides. write_cloudfile_settings(join(topdir, 'conf', 'seahub_settings.py')) diff --git a/tests/test_cloudfile_settings.py b/tests/test_cloudfile_settings.py index b1552216..bed98040 100644 --- a/tests/test_cloudfile_settings.py +++ b/tests/test_cloudfile_settings.py @@ -44,6 +44,31 @@ def test_policy_worker_hooks_preserve_compose_and_remove_only_owned_block(self): self.assertEqual(path.read_text(encoding="utf-8"), original) self.assertFalse(CLOUDFILE.write_policy_worker_hooks(path, {})) + def test_v2_removes_only_legacy_worker_hooks_without_starting_policy_host(self): + # A stale policy config/hook must not load a historical SchemaRunner. + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "gunicorn.conf.py" + original = "def post_worker_init(worker):\n worker.append('native')\n" + path.write_text(original) + env = {"CLOUDFILE_POLICY_WORKER_HOOKS": "true"} + CLOUDFILE.write_policy_worker_hooks(path, env) + CLOUDFILE.write_policy_worker_hooks(path, env, v2=True) + self.assertEqual(path.read_text(), original) + namespace = {}; exec(path.read_text(), namespace) + events = []; namespace['post_worker_init'](events) + self.assertEqual(events, ['native']) + self.assertEqual(env, {"CLOUDFILE_POLICY_WORKER_HOOKS": "true"}) + + def test_v2_refuses_retired_routes_before_mutating_hooks(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "gunicorn.conf.py" + path.write_text("# operator config\n") + for key in ('CLOUDFILE_EDITING_ENABLED', 'CLOUDFILE_AUTHORIZATION_ENABLED', + 'CLOUDFILE_OIDC_ENABLED', 'CLOUDFILE_RESOURCE_SEARCH_ENABLED'): + with self.assertRaisesRegex(ValueError, key): + CLOUDFILE.write_policy_worker_hooks(path, {key: 'true'}, v2=True) + self.assertEqual(path.read_text(), "# operator config\n") + def test_policy_exit_cleanup_runs_after_existing_hook_failure(self): with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "gunicorn.conf.py" @@ -104,10 +129,10 @@ def test_existing_block_is_replaced(self): self.assertIn("before = True", rendered) self.assertIn("after = True", rendered) - def test_administrator_policy_defaults_on_without_oidc_or_extra_secrets(self): + def test_retired_administrator_policy_defaults_off_without_loading_its_schema(self): values = {} exec(CLOUDFILE.render_settings({}), values) - self.assertIs(values['CLOUDFILE_ADMIN_POLICY_ENABLED'], True) + self.assertIs(values['CLOUDFILE_ADMIN_POLICY_ENABLED'], False) self.assertIs(values['CLOUDFILE_OIDC_ENABLED'], False) self.assertIsNone(values['CLOUDFILE_POLICY_CONFIG']) with self.assertRaisesRegex(ValueError, 'administrator policy'): diff --git a/tests/test_fresh_schema.py b/tests/test_fresh_schema.py index 97b5eceb..5bb63a6b 100644 --- a/tests/test_fresh_schema.py +++ b/tests/test_fresh_schema.py @@ -1,71 +1,90 @@ -"""Fresh-install DDL must resume failures without upgrading existing volumes.""" - +"""Development startup checks contracts and never executes an upgrade.""" +import ast import importlib.util from pathlib import Path import sys -import tempfile from types import ModuleType import unittest from unittest.mock import Mock, patch - -SCRIPT = Path(__file__).resolve().parents[1] / 'scripts/scripts_14.0/cloudfile_schema.py' -SPEC = importlib.util.spec_from_file_location('fresh_schema', SCRIPT) +ROOT = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location('development_schema', ROOT / 'scripts/scripts_14.0/cloudfile_schema.py') SCHEMA = importlib.util.module_from_spec(SPEC) SPEC.loader.exec_module(SCHEMA) -class FreshSchemaTest(unittest.TestCase): - def test_existing_volume_does_not_opt_into_ddl(self): - with tempfile.TemporaryDirectory() as directory: - (Path(directory) / 'seafile-data').mkdir() - marker = SCHEMA.prepare_fresh_schema(directory) - self.assertFalse(marker.exists()) - self.assertFalse(SCHEMA.initialize_fresh_schema(marker, '/missing', {})) +class ReadOnlyCursor: + def __init__(self, defect=None): self.defect, self.rows, self.calls = defect, (), [] + def __enter__(self): return self + def __exit__(self, *args): pass + def execute(self, sql, args=()): + if not sql.startswith('SELECT '): raise AssertionError('Startup attempted a database write') + self.calls.append(sql) + if not args: + self.rows = tuple((name,) for name in SCHEMA.COLUMNS) + if self.defect == 'legacy_table': self.rows += (('cf_resource',),) + return + table = args[0] + if 'information_schema.TABLES' in sql: + self.rows = (('InnoDB', 'utf8mb4_bin'),) + elif 'information_schema.COLUMNS' in sql: + self.rows = tuple((name, 'binary' if name == 'path_hash' else 'bigint' if name == 'group_id' else 'varchar', + 32 if name == 'path_hash' else 255) for name in SCHEMA.COLUMNS[table].split()) + if self.defect == 'old_guard' and table == 'cf_edit_guard': self.rows += (('resource_uid', 'char', 36),) + if self.defect == 'text_hash' and table == 'cf_dir_acl': + self.rows = tuple((name, 'char', 64) if name == 'path_hash' else (name, kind, length) for name, kind, length in self.rows) + else: + self.rows = tuple((name, unique, i+1, column, None) + for name, (unique, columns) in SCHEMA.INDEXES[table].items() for i, column in enumerate(columns)) + if self.defect == 'missing_index' and table == 'cf_edit_guard': + self.rows = tuple(row for row in self.rows if row[0] != 'cf_edit_guard_active') + def fetchall(self): return self.rows + + +class DevelopmentSchemaTest(unittest.TestCase): + def test_correct_contract_issues_only_selects(self): + cursor = ReadOnlyCursor() + connection = Mock(); connection.cursor.return_value = cursor + SCHEMA.verify_v2_schema(connection) + self.assertEqual(len(cursor.calls), 10) + connection.commit.assert_not_called() + connection.rollback.assert_not_called() - def test_failed_first_install_retains_intent_after_ce_initializes(self): - with tempfile.TemporaryDirectory() as directory: - marker = SCHEMA.prepare_fresh_schema(directory) - self.assertTrue(marker.exists()) - (Path(directory) / 'seafile-data').mkdir() - self.assertEqual(SCHEMA.prepare_fresh_schema(directory), marker) - self.assertTrue(marker.exists()) + def test_old_layout_hash_and_missing_index_fail_without_repair(self): + for defect in ('old_guard', 'text_hash', 'missing_index', 'legacy_table'): + connection = Mock(); connection.cursor.return_value = ReadOnlyCursor(defect) + with self.assertRaises(ValueError): SCHEMA.verify_v2_schema(connection, checkout_enabled=True) + connection.commit.assert_not_called() - def _run(self, fail=False): - driver = ModuleType('pymysql') - driver.connect = Mock() - runner_module = ModuleType('cloudfile_extensions.schema.runner') - runner_module.SchemaRunner = Mock() - if fail: - runner_module.SchemaRunner.return_value.apply.side_effect = ValueError('private-database-value') - modules = {'pymysql': driver, 'cloudfile_extensions.schema.runner': runner_module} - environment = dict(SEAFILE_MYSQL_DB_HOST='db', SEAFILE_MYSQL_DB_USER='native', - SEAFILE_MYSQL_DB_PASSWORD='fixture', SEAFILE_MYSQL_DB_SEAFILE_DB_NAME='native_db') - with tempfile.TemporaryDirectory() as directory, patch.dict(sys.modules, modules): - marker = SCHEMA.prepare_fresh_schema(directory) - previous_path = list(sys.path) - if fail: - with self.assertRaisesRegex(RuntimeError, '^Fresh CloudFile schema initialization failed;') as error: - SCHEMA.initialize_fresh_schema(marker, '/installed', environment) - self.assertNotIn('private-database-value', str(error.exception)) - self.assertTrue(marker.exists()) - else: - self.assertTrue(SCHEMA.initialize_fresh_schema(marker, '/installed', environment)) - self.assertFalse(marker.exists()) - # A completed installation must not issue DDL again on restart. - self.assertFalse(SCHEMA.initialize_fresh_schema(marker, '/installed', environment)) - self.assertEqual(sys.path, previous_path) - runner_module.SchemaRunner.return_value.apply.assert_called_once_with() - driver.connect.return_value.close.assert_called_once_with() - self.assertEqual(driver.connect.call_args.kwargs['database'], 'native_db') - self.assertTrue(driver.connect.call_args.kwargs['autocommit']) + def test_disabled_checkout_ignores_guard_layout_and_index(self): + for defect in ('old_guard', 'missing_index'): + cursor = ReadOnlyCursor(defect) + connection = Mock(); connection.cursor.return_value = cursor + SCHEMA.verify_v2_schema(connection) + self.assertEqual(len(cursor.calls), 10) - def test_success_verifies_and_clears_intent(self): - self._run() + def test_connection_failure_is_sanitized_and_requires_sql(self): + driver = ModuleType('pymysql'); driver.connect = Mock(side_effect=RuntimeError('private-fixture-value')) + env = dict(SEAFILE_MYSQL_DB_HOST='db', SEAFILE_MYSQL_DB_PASSWORD='fixture') + with patch.dict(sys.modules, {'pymysql': driver}): + with self.assertRaises(RuntimeError) as error: SCHEMA.verify_development_schema(environment=env) + self.assertIn('SQL', str(error.exception)); self.assertNotIn('private-fixture-value', str(error.exception)) - def test_failure_stops_startup_and_allows_retry(self): - self._run(fail=True) + def test_startup_and_config_do_not_call_ddl_helpers(self): + bootstrap = ast.parse((ROOT/'scripts/scripts_14.0/bootstrap.py').read_text()) + function = next(n for n in bootstrap.body if isinstance(n, ast.FunctionDef) and n.name == 'write_cloudfile_config') + names = {n.func.id for n in ast.walk(function) if isinstance(n, ast.Call) and isinstance(n.func, ast.Name)} + self.assertFalse(any(name.startswith('apply_') for name in names)) + start = (ROOT/'scripts/scripts_14.0/start.py').read_text() + self.assertIn('verify_development_schema(installdir)', start) + startup = ast.parse(start) + hook = next(n for n in ast.walk(startup) if isinstance(n, ast.Call) + and isinstance(n.func, ast.Name) and n.func.id == 'write_policy_worker_hooks') + self.assertTrue(any(k.arg == 'v2' and isinstance(k.value, ast.Constant) + and k.value.value is True for k in hook.keywords)) + self.assertNotIn('initialize_fresh_schema', start) + self.assertNotIn('prepare_fresh_schema', start) + self.assertNotIn('SchemaRunner', (ROOT/'scripts/scripts_14.0/cloudfile_schema.py').read_text()) -if __name__ == '__main__': - unittest.main() +if __name__ == '__main__': unittest.main() diff --git a/tests/test_manual_sql_integration.py b/tests/test_manual_sql_integration.py new file mode 100644 index 00000000..c21babcb --- /dev/null +++ b/tests/test_manual_sql_integration.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""Disposable database integration test. Publishes no ports, reads no DEV secrets. + +Requires an existing mariadb:11.4 image on the selected test host. All fixtures +are synthetic; the uniquely named container and tmpfs are removed in finally. +""" +from pathlib import Path +import json,subprocess,base64 +import argparse +root=Path(__file__).resolve().parents[1] +parser=argparse.ArgumentParser(description='Verify manual SQL on a disposable no-network MariaDB container; never target an existing DB') +parser.add_argument('--hub',type=Path,default=root.parent/'cloudfile-hub') +parser.add_argument('--server',type=Path,help='Server source for the native Guard SQL contract') +parser.add_argument('--ssh-host',help='Optional test host with an existing mariadb:11.4 image') +options=parser.parse_args() +folder=root/'database/v2/20261021-four-table-baseline';hub=options.hub +pack={p.name:p.read_text() for p in folder.glob('*.sql')} +statements=[] +for pat in ('006*.json','008*.json','029*.json'): + for p in (hub/'cloudfile_extensions/schema/migrations').glob(pat): + for step in json.loads(p.read_text())['steps']: + sql=step['sql'] + if any(sql.startswith('CREATE TABLE '+t+' ') for t in ('cf_sso_group_map','cf_dir_acl','cf_edit_guard')):statements.append(sql+';') +core=pack['00-create.sql'];import re +storage=re.search(r'CREATE TABLE cf_library_storage \([\s\S]*?\) ENGINE[^;]+;',core)[0] +fixture='\n'.join([storage,*statements])+''' +ALTER TABLE cf_dir_acl ADD COLUMN share_mode TINYINT NOT NULL DEFAULT 0, + ADD COLUMN lifecycle_ref VARCHAR(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT ''; +INSERT INTO cf_library_storage VALUES ('fixture:key','01234567-89ab-cdef-0123-456789abcdef','active'); +INSERT INTO cf_sso_group_map(provider,subject_type,namespace,external_id,group_id,name) VALUES ('fixture','org','directory','dept1',7,'Fixture department'); +INSERT INTO cf_dir_acl(id,repo_id,path,path_hash,kind,subject_type,provider,namespace,external_id,subject_hash,permission,inherit,revision) +VALUES ('11111111-1111-1111-1111-111111111111','01234567-89ab-cdef-0123-456789abcdef','/fixture',SHA2('/fixture',256),'dir','org','fixture','directory','dept1',SHA2('fixture-subject',256),'rw',1,'22222222-2222-2222-2222-222222222222'); +''' +pack['fixture']=fixture +pack['gate_source']=(root/'scripts/scripts_14.0/cloudfile_schema.py').read_text() +# Execute SQL extracted from the changed production C entry points, rather +# than a handwritten equivalent which could hide a field/encoding mismatch. +import ast +native = options.server or root.parent/'cloudfile-server' +queries = [] +for path, name in [('common/cf-lock.c','load_exact ('), + ('common/cf-lock.c','load_descendant ('), + ('server/cloudfile-policy.c','cf_policy_check_unleased_write (')]: + source=(native/path).read_text().split(name,1)[1] + match=re.search(r'("SELECT path FROM cf_edit_guard FORCE INDEX[\s\S]+?)",\n',source) + if not match:raise RuntimeError('native Guard SQL not found') + queries.append(''.join(ast.literal_eval(part) for part in re.findall(r'"(?:\\.|[^"\\])*"',match[1]+'"'))) +pack['native_queries']=queries +script='pack='+repr(pack)+'\n'+r''' +import subprocess,uuid,time,json +name='cf-v2-sql-test-'+uuid.uuid4().hex[:10] +def sql(value,db=None,ok=True): + cmd=['docker','exec','-i',name,'mariadb','-uroot','--skip-ssl','--batch','--skip-column-names'] + if db:cmd.append(db) + r=subprocess.run(cmd,input=value,text=True,capture_output=True,timeout=30) + if ok and r.returncode: + # Fixture data and SQL definitions only; connection has no secrets and no ports. + print('SQL_TEST_ERROR',r.stderr[-1500:]);raise RuntimeError('isolated SQL failed') + if not ok and not r.returncode:raise RuntimeError('unsafe SQL unexpectedly succeeded') + return r.stdout.strip() +try: + r=subprocess.run(['docker','run','--detach','--pull=never','--name',name,'--network','none','--tmpfs','/var/lib/mysql:rw,size=768m','-e','MARIADB_ALLOW_EMPTY_ROOT_PASSWORD=1','mariadb:11.4'],capture_output=True,text=True,timeout=30) + if r.returncode:raise RuntimeError('isolated MariaDB could not start') + for _ in range(90): + ready=subprocess.run(['docker','exec',name,'mariadb-admin','-uroot','ping'],capture_output=True,timeout=5) + if ready.returncode==0 and subprocess.run(['docker','exec',name,'cat','/proc/1/comm'],capture_output=True,text=True,timeout=5).stdout.strip()=='mariadbd':break + time.sleep(.5) + else:raise RuntimeError('isolated MariaDB readiness timeout') + sql('CREATE DATABASE fresh CHARACTER SET utf8mb4; CREATE DATABASE old_layout CHARACTER SET utf8mb4; CREATE DATABASE guarded CHARACTER SET utf8mb4; CREATE DATABASE lifecycle CHARACTER SET utf8mb4; CREATE DATABASE seahub_fixture CHARACTER SET utf8mb4; CREATE DATABASE old_layout_archive; CREATE DATABASE lifecycle_archive; CREATE DATABASE guarded_archive;') + sql(pack['00-create.sql'],'fresh');sql(pack['50-verify.sql'],'fresh');print('PASS fresh initialization and verification',flush=True) + sql(pack['fixture'],'old_layout') + sql("SET @cf_upgrade_archive_database='old_layout_archive'; SET @cf_upgrade_writers_stopped=1;\n"+pack['10-upgrade-029-042.sql'],'old_layout');sql(pack['50-verify.sql'],'old_layout') + result=sql("SELECT (SELECT COUNT(*) FROM cf_library_storage),(SELECT COUNT(*) FROM cf_sso_group_map),(SELECT COUNT(*) FROM cf_dir_acl),LENGTH(path_hash),permission,inherit,group_id FROM cf_dir_acl JOIN cf_sso_group_map USING(provider,subject_type,namespace,external_id);",'old_layout') + if result!='1\t1\t1\t32\trw\t1\t7':raise RuntimeError('migration identity or data mismatch') + print('PASS old layout migration, native group identity and ACL bytes',flush=True) + # The C barrier and final publication SQL must work with exactly four tables. + repo='01234567-89ab-cdef-0123-456789abcdef'; path='/parent/file' + def guard_query(query,args): + for arg in args:query=query.replace('?',"'"+arg.replace("'","''")+"'",1) + return sql(query,'old_layout') + import hashlib + args=[(repo,hashlib.sha256(path.encode()).hexdigest(),path),(repo,'/parent/%'), + (repo,hashlib.sha256(path.encode()).hexdigest(),path)] + for query,params in zip(pack['native_queries'],args): + if guard_query(query,params):raise RuntimeError('empty Guard unexpectedly conflicts') + sql("INSERT INTO cf_edit_guard(repo_id,path_hash,path,guard_id,owner_uid,owner_native_user,proof_digest,base_file_id,lease_expires_at) VALUES ('"+repo+"',UNHEX(SHA2('"+path+"',256)),'"+path+"','33333333-3333-3333-3333-333333333333','fixture','owner',REPEAT('a',64),REPEAT('b',40),UTC_TIMESTAMP(6)+INTERVAL 1 MINUTE)",'old_layout') + for query,params in zip(pack['native_queries'],args): + if guard_query(query,params)!=path:raise RuntimeError('active Guard not found by native SQL') + sql('UPDATE cf_edit_guard SET lease_expires_at=UTC_TIMESTAMP(6)-INTERVAL 1 SECOND','old_layout') + for query,params in zip(pack['native_queries'],args): + if guard_query(query,params):raise RuntimeError('expired Guard blocks a native write') + sql('DELETE FROM cf_edit_guard','old_layout') + print('PASS production C Guard SQL: empty, active, expired against four-table database',flush=True) + sql("SET @cf_upgrade_archive_database='old_layout_archive'; SET @cf_upgrade_writers_stopped=1;\n"+pack['10-upgrade-029-042.sql'],'old_layout',ok=False) + sql(pack['50-verify.sql'],'old_layout');print('PASS repeat rejected without destroying target or backups',flush=True) + sql('ALTER TABLE cf_edit_guard DROP INDEX cf_edit_guard_active;','fresh');sql(pack['50-verify.sql'],'fresh',ok=False);print('PASS missing index rejected',flush=True) + sql(pack['fixture'],'guarded') + sql("INSERT INTO cf_edit_guard(resource_uid,repo_id,lifecycle_ref,generation,credential_epoch) VALUES ('33333333-3333-3333-3333-333333333333','01234567-89ab-cdef-0123-456789abcdef','fixture-life',7,1);",'guarded') + sql("SET @cf_upgrade_archive_database='guarded_archive'; SET @cf_upgrade_writers_stopped=1;\n"+pack['10-upgrade-029-042.sql'],'guarded',ok=False) + if sql('SELECT generation FROM cf_edit_guard;','guarded')!='7':raise RuntimeError('guard identity lost') + print('PASS nonempty Guard rejected and generation preserved',flush=True) + sql(pack['fixture'],'lifecycle');sql("UPDATE cf_dir_acl SET lifecycle_ref='fixture-life';",'lifecycle') + sql("SET @cf_upgrade_archive_database='lifecycle_archive'; SET @cf_upgrade_writers_stopped=1;\n"+pack['10-upgrade-029-042.sql'],'lifecycle',ok=False) + sql("SET @cf_upgrade_archive_database='lifecycle_archive'; SET @cf_upgrade_writers_stopped=1; SET @cf_upgrade_accept_path_acl=1;\n"+pack['10-upgrade-029-042.sql'],'lifecycle');sql(pack['50-verify.sql'],'lifecycle');print('PASS lifecycle conversion requires explicit acknowledgement',flush=True) + sql('CREATE TABLE repo_metadata(id INT); CREATE TABLE repo_tags_repotags(id INT); CREATE TABLE base_userstarredfiles(id INT);','seahub_fixture') + sql('SET @cf_upgrade_writers_stopped=1;\n'+pack['20-seahub-compat.sql'],'seahub_fixture');sql('SET @cf_upgrade_writers_stopped=1;\n'+pack['20-seahub-compat.sql'],'seahub_fixture');print('PASS explicit Seahub compatibility SQL and replay',flush=True) + # Exercise the real startup checker over actual DB output; no driver dependency. + import types + gate=types.ModuleType('gate');exec(pack['gate_source'],gate.__dict__) + class Cursor: + def __enter__(self):return self + def __exit__(self,*args):pass + def execute(self,query,args=()): + if not query.startswith('SELECT '):raise RuntimeError('startup wrote SQL') + if args:query=query.replace('%s',"'"+args[0]+"'") + self.rows=[] + for line in sql(query,'old_layout').splitlines(): + values=[] + for part in line.split('\t'): + values.append(None if part=='NULL' else int(part) if part.isdigit() else part) + self.rows.append(tuple(values)) + def fetchall(self):return tuple(self.rows) + class Connection: + def cursor(self):return Cursor() + gate.verify_v2_schema(Connection());print('PASS read-only startup gate against migrated MariaDB',flush=True) +finally: + subprocess.run(['docker','rm','--force',name],capture_output=True,timeout=30) + print('isolated temporary database removed',flush=True) +''' +r=subprocess.run((['ssh','-o','BatchMode=yes',options.ssh_host,'python3'] if options.ssh_host else ['python3']),input=script,text=True,capture_output=True,timeout=150) +print(r.stdout);print('integration_exit',r.returncode) +if r.returncode:print(r.stderr[-1500:]) +raise SystemExit(r.returncode) diff --git a/tests/test_manual_sql_package.py b/tests/test_manual_sql_package.py new file mode 100644 index 00000000..c96ada72 --- /dev/null +++ b/tests/test_manual_sql_package.py @@ -0,0 +1,47 @@ +"""Release SQL and read-only startup must not drift into different contracts.""" +import hashlib +import importlib.util +import json +from pathlib import Path +import unittest + +ROOT=Path(__file__).resolve().parents[1] +def module(name,path): + spec=importlib.util.spec_from_file_location(name,path) + value=importlib.util.module_from_spec(spec);spec.loader.exec_module(value);return value +SQL=module('sql_contracts',ROOT/'tools/check-sql-contracts.py') +GATE=module('readonly_schema',ROOT/'scripts/scripts_14.0/cloudfile_schema.py') +FOLDER=ROOT/'database/v2/20261021-four-table-baseline' + + +class ManualSQLPackageTest(unittest.TestCase): + def test_fixed_release_checksums_cover_every_sql_input(self): + expected=json.loads((FOLDER/'checksums.json').read_text()) + actual={p.name:hashlib.sha256(p.read_bytes()).hexdigest() for p in FOLDER.glob('*.sql')} + self.assertEqual(actual,expected) + + def test_no_fifth_business_table_is_delivered(self): + names={name for path in FOLDER.glob('*.sql') for name,_,_ in SQL.creates(path.read_text())} + self.assertEqual(names,{'cf_library_storage','cf_dir_acl','cf_sso_group_map','cf_edit_guard'}) + + def test_startup_contract_matches_delivered_create_sql(self): + tables={name:(columns,indexes) for name,columns,indexes in SQL.creates((FOLDER/'00-create.sql').read_text())} + self.assertEqual(set(tables),set(GATE.COLUMNS)) + for name,(columns,indexes) in tables.items(): + self.assertEqual(columns,set(GATE.COLUMNS[name].split())) + self.assertEqual(indexes,GATE.INDEXES[name]) + + def test_upgrade_requires_offline_and_refuses_guard_or_partial_state(self): + upgrade=(FOLDER/'10-upgrade-029-042.sql').read_text() + self.assertIn('@cf_upgrade_writers_stopped',upgrade) + self.assertIn('@cf_upgrade_accept_path_acl',upgrade) + self.assertIn('IF EXISTS(SELECT 1 FROM cf_edit_guard)',upgrade) + self.assertIn('UNHEX(path_hash)',upgrade) + self.assertEqual(upgrade.count('RENAME TABLE '),1) + self.assertIn('@cf_upgrade_archive_database',upgrade) + self.assertNotIn('DROP TABLE ',upgrade) + self.assertIn('Candidate data mismatch',upgrade) + self.assertNotIn('cf_schema_migration SET',upgrade) + + +if __name__=='__main__': unittest.main() diff --git a/tests/test_repair_v2_four_tables.py b/tests/test_repair_v2_four_tables.py new file mode 100644 index 00000000..f27786fa --- /dev/null +++ b/tests/test_repair_v2_four_tables.py @@ -0,0 +1,57 @@ +"""Lossless DEV projections must reject data the four-table contract cannot hold.""" +import hashlib +import importlib.util +from pathlib import Path +import unittest + +SPEC = importlib.util.spec_from_file_location('four_table_repair', Path(__file__).resolve().parents[1] / 'tools/repair-v2-four-tables.py') +REPAIR = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(REPAIR) + + +class FourTableRepairTest(unittest.TestCase): + def acl(self): + return dict(repo_id='01234567-89ab-cdef-0123-456789abcdef', path='/目录/a', + path_hash=hashlib.sha256('/目录/a'.encode()).hexdigest(), + subject_type='user', provider='fixture', namespace='user', + external_id='1', permission='rw', inherit=1, share_mode=0, + revision='01234567-89ab-cdef-0123-456789abcdef', kind='dir', + lifecycle_ref='retained-in-private-backup') + + def test_acl_preserves_rules_and_converts_hash_to_binary(self): + source = self.acl() + target = REPAIR.project('cf_dir_acl', [source])[0] + self.assertEqual(len(target['path_hash']), 32) + self.assertEqual(target['path_hash'].hex(), source['path_hash']) + self.assertEqual(set(target), set(REPAIR.TARGET_COLUMNS['cf_dir_acl'])) + for column in target: + if column != 'path_hash': self.assertEqual(target[column], source[column]) + + def test_hash_corruption_and_file_scope_have_no_silent_conversion(self): + for changed in (dict(path_hash='00'*32), dict(kind='file')): + source = self.acl(); source.update(changed) + with self.assertRaises(RuntimeError): REPAIR.project('cf_dir_acl', [source]) + + def test_group_mapping_keeps_native_group_identity(self): + source = dict(provider='fixture', subject_type='org', namespace='directory', + external_id='12', group_id=34, name='native-display-name', id=56) + result = REPAIR.project('cf_sso_group_map', [source])[0] + self.assertEqual(result, {key:source[key] for key in REPAIR.TARGET_COLUMNS['cf_sso_group_map']}) + + def test_any_old_guard_row_requires_a_separate_conversion(self): + self.assertEqual(REPAIR.project('cf_edit_guard', []), []) + with self.assertRaises(RuntimeError): REPAIR.project('cf_edit_guard', [dict(resource_uid='guard')]) + + def test_storage_states_and_keys_remain_unchanged(self): + source = dict(repo_key='native/key', repo_id='repo', state='retired') + self.assertEqual(REPAIR.project('cf_library_storage', [source]), [source]) + + def test_digest_is_order_independent_and_binary_safe(self): + rows = [dict(value=b'\x00\xff'), dict(value='text')] + self.assertEqual(REPAIR.digest(rows), REPAIR.digest(rows[::-1])) + self.assertNotEqual(REPAIR.digest(rows), REPAIR.digest(rows[:1])) + self.assertEqual(set(REPAIR.definitions()), set(REPAIR.TABLES)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_sql_contracts.py b/tests/test_sql_contracts.py new file mode 100644 index 00000000..33f113f4 --- /dev/null +++ b/tests/test_sql_contracts.py @@ -0,0 +1,81 @@ +"""Run SQL contract regressions on copies of the real reviewed DDL inputs.""" +import argparse +import importlib.util +from pathlib import Path +import shutil +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location('sql_contracts', ROOT / 'tools/check-sql-contracts.py') +CHECK = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(CHECK) +SERVER = ROOT.parent / 'cloudfile-server' +HUB = ROOT.parent / 'cloudfile-hub' + + +class SqlContractTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.server = Path(self.temporary.name) / 'server' + self.hub = Path(self.temporary.name) / 'hub' + sql = self.server / 'scripts/sql/mysql' + sql.mkdir(parents=True) + for name in ('cloudfile.sql', 'cloudfile-v2.sql', 'cloudfile-v2-edit.sql'): + shutil.copyfile(SERVER / 'scripts/sql/mysql' / name, sql / name) + shutil.copytree(HUB / 'cloudfile_extensions/schema/migrations', self.hub / 'cloudfile_extensions/schema/migrations') + helper = self.hub / 'cloudfile_ext/v2' + helper.mkdir(parents=True) + shutil.copyfile(HUB / 'cloudfile_ext/v2/checkout_schema.py', helper / 'checkout_schema.py') + + def test_actual_reviewed_contracts_agree_and_report_layout_boundaries(self): + result = CHECK.audit(self.server, self.hub) + self.assertEqual(result['checkout_reader_contracts'], 'matched') + self.assertEqual(result['isolated_layout_collisions'], ['cf_dir_acl', 'cf_edit_guard', 'cf_sso_group_map']) + + def test_new_missing_index_or_changed_column_is_rejected(self): + path = self.server / 'scripts/sql/mysql/cloudfile-v2-edit.sql' + original = path.read_text() + for changed in (original.replace('cf_edit_guard_active', 'wrong_index'), + original.replace('lease_expires_at', 'lease_expiry')): + path.write_text(changed) + with self.assertRaisesRegex(ValueError, 'reader and CREATE DDL'): + CHECK.audit(self.server, self.hub) + + def test_new_unreviewed_same_name_layout_is_rejected(self): + core = self.server / 'scripts/sql/mysql/cloudfile-v2.sql' + core.write_text(core.read_text() + '\nCREATE TABLE cf_probe(alpha INT) ENGINE=InnoDB;') + migration = self.hub / 'cloudfile_extensions/schema/migrations/999_probe.json' + migration.write_text('{"steps":[{"sql":"CREATE TABLE cf_probe(beta INT) ENGINE=InnoDB"}]}') + with self.assertRaisesRegex(ValueError, 'new incompatible table-name collision'): + CHECK.audit(self.server, self.hub) + + def test_type_default_and_migration_changes_require_review(self): + path = self.server / 'scripts/sql/mysql/cloudfile-v2-edit.sql' + original = path.read_text() + # Same columns/indexes still cannot hide changed types or SQL semantics. + path.write_text(original.replace('CHAR(36)', 'CHAR(35)')) + self.assertNotEqual(path.read_text(), original) + with self.assertRaisesRegex(ValueError, 'SQL review baseline changed'): + CHECK.audit(self.server, self.hub) + path.write_text(original) + migration = self.hub / 'cloudfile_extensions/schema/migrations/999_drift.json' + migration.write_text('{"steps":[{"sql":"ALTER TABLE cf_edit_guard ADD unsafe INT DEFAULT 1"}]}') + with self.assertRaisesRegex(ValueError, 'SQL review baseline changed'): + CHECK.audit(self.server, self.hub) + + def test_nested_constraints_do_not_become_fake_columns(self): + result = list(CHECK.creates("CREATE TABLE cf_test(path VARCHAR(99), guard_id CHAR(36), CONSTRAINT c CHECK (path IN ('a,b','c')), KEY lookup(path,guard_id)) ENGINE=InnoDB")) + self.assertEqual(result[0][1], {'path', 'guard_id'}) + self.assertEqual(result[0][2]['lookup'], (1, ('path', 'guard_id'))) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument('--server', type=Path, default=SERVER) + parser.add_argument('--hub', type=Path, default=HUB) + options, remaining = parser.parse_known_args() + SERVER, HUB = options.server, options.hub + unittest.main(argv=[sys.argv[0], *remaining]) diff --git a/tools/check-sql-contracts.py b/tools/check-sql-contracts.py new file mode 100644 index 00000000..abe3fee9 --- /dev/null +++ b/tools/check-sql-contracts.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""Check reviewed cross-repository CREATE contracts without touching a database. + +The complete migration runner remains responsible for ALTER/DROP and ledger +checks. This gate catches new table-name collisions and optional Checkout +columns/indexes drifting away from their actual reader before a build ships. +""" +import argparse +import hashlib +import importlib.util +import json +from pathlib import Path +import re + + +KNOWN_ISOLATED_LAYOUTS = {'cf_dir_acl', 'cf_sso_group_map', 'cf_edit_guard'} + + +def sql_steps(value): + if isinstance(value, dict): + for name, child in value.items(): + if name == 'sql' and isinstance(child, str): + yield child + else: + yield from sql_steps(child) + elif isinstance(value, list): + for child in value: + yield from sql_steps(child) + + +def split_definitions(body): + depth, quote, start = 0, None, 0 + for index, char in enumerate(body): + if char in "'\"`" and (index == 0 or body[index - 1] != '\\'): + if quote == char: + quote = None + elif quote is None: + quote = char + if quote is None: + if char == '(': + depth += 1 + elif char == ')': + depth -= 1 + elif char == ',' and depth == 0: + yield body[start:index].strip() + start = index + 1 + yield body[start:].strip() + + +def creates(sql): + sql = re.sub(r'--[^\n]*', '', sql) + for match in re.finditer(r'CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?`?(cf_\w+)`?\s*\((.*?)\)\s*ENGINE', sql, re.I | re.S): + name, body = match.groups() + columns, indexes = set(), {} + for definition in split_definitions(body): + first = definition.split()[0].strip('`').upper() + if first not in {'PRIMARY', 'KEY', 'UNIQUE', 'CONSTRAINT', 'FOREIGN', 'CHECK', 'INDEX'}: + columns.add(definition.split()[0].strip('`')) + primary = re.match(r'PRIMARY\s+KEY\s*\(([^)]+)\)', definition, re.I) + key = re.match(r'(UNIQUE\s+)?KEY\s+`?(\w+)`?\s*\(([^)]+)\)', definition, re.I) + if primary: + indexes['PRIMARY'] = (0, tuple(x.strip(' `') for x in primary[1].split(','))) + elif key: + indexes[key[2]] = (0 if key[1] else 1, tuple(x.strip(' `') for x in key[3].split(','))) + yield name, frozenset(columns), indexes + + +def ddl_sources(server, hub): + for name in ('cloudfile.sql', 'cloudfile-v2.sql', 'cloudfile-v2-edit.sql'): + yield 'server/' + name, server / 'scripts/sql/mysql' / name + for path in sorted((hub / 'cloudfile_extensions/schema/migrations').glob('*.json')): + yield 'hub/' + path.name, path + + +def reviewed_ddl_digest(path): + # Include ALTER/DROP, types, defaults and checks, not just CREATE column names. + # Canonical JSON ignores formatting while preserving every SQL statement. + value = path.read_text() + if path.suffix == '.json': + value = json.dumps(json.loads(value), sort_keys=True, ensure_ascii=False) + return hashlib.sha256(value.encode()).hexdigest() + + +def audit(server, hub): + contracts = {} + definitions = 0 + for name in ('cloudfile.sql', 'cloudfile-v2.sql', 'cloudfile-v2-edit.sql'): + path = server / 'scripts/sql/mysql' / name + for table, columns, indexes in creates(path.read_text()): + contracts.setdefault(table, []).append((name, columns, indexes)) + definitions += 1 + for path in sorted((hub / 'cloudfile_extensions/schema/migrations').glob('*.json')): + for statement in sql_steps(json.loads(path.read_text())): + for table, columns, indexes in creates(statement): + contracts.setdefault(table, []).append((path.name, columns, indexes)) + definitions += 1 + collisions = {name for name, layouts in contracts.items() + if len({layout[1] for layout in layouts}) > 1} + unexpected = collisions - KNOWN_ISOLATED_LAYOUTS + if unexpected: + raise ValueError('new incompatible table-name collision: ' + ', '.join(sorted(unexpected))) + # Load only the dependency-free contract checker, never Django/settings/DDL. + path = hub / 'cloudfile_ext/v2/checkout_schema.py' + spec = importlib.util.spec_from_file_location('checkout_schema_contract', path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + guards = contracts.get('cf_edit_guard', []) + path_guard = next((x for x in guards if x[0] == 'cloudfile-v2-edit.sql'), None) + resource_guard = next((x for x in guards if x[0] == '029_editing_core.json'), None) + if path_guard is None or path_guard[1] != module.PATH_COLUMNS or path_guard[2] != module.PATH_INDEXES: + raise ValueError('path Checkout reader and CREATE DDL columns/indexes differ') + if resource_guard is None or resource_guard[1] != module.RESOURCE_COLUMNS: + raise ValueError('resource Checkout detector and migration DDL differ') + # A source change requires a deliberate SQL review and baseline update; + # merely retaining a known table name must never bless a new layout. + baseline = json.loads((Path(__file__).with_name('reviewed-sql-contracts.json')).read_text()) + actual = {name: reviewed_ddl_digest(path) for name, path in ddl_sources(server, hub)} + drift = sorted(name for name in set(actual) | set(baseline) if actual.get(name) != baseline.get(name)) + if drift: + raise ValueError('SQL review baseline changed: ' + ', '.join(drift)) + return {'reviewed_ddl_sources': len(actual), 'create_definitions': definitions, 'table_names': len(contracts), + 'isolated_layout_collisions': sorted(collisions), + 'checkout_reader_contracts': 'matched'} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + workspace = Path(__file__).resolve().parents[2] + parser.add_argument('--server', type=Path, default=workspace / 'cloudfile-server') + parser.add_argument('--hub', type=Path, default=workspace / 'cloudfile-hub') + args = parser.parse_args() + try: + print(json.dumps(audit(args.server, args.hub), ensure_ascii=False, sort_keys=True)) + except (OSError, ValueError, ImportError) as error: + parser.exit(1, 'SQL contract check failed: ' + str(error) + '\n') + + +if __name__ == '__main__': + main() diff --git a/tools/repair-v2-four-tables.py b/tools/repair-v2-four-tables.py new file mode 100644 index 00000000..56020260 --- /dev/null +++ b/tools/repair-v2-four-tables.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""Explicit DEV-only four-table conversion; never a startup migration. + +Keep original tables and a private persistent backup. Ordinary upload/download +must not acquire Checkout sessions. This command does not update the legacy +ledger, delete unrelated tables, set environment variables, or auto-roll back. +Run inside the already identified cloudfile-dev container using its controlled +DB credentials; never copy credentials to command arguments or backup files. +""" +import argparse +import base64 +import datetime +import hashlib +import json +import os +from pathlib import Path +import re +import signal +import sys +import uuid + +DDL = "-- CloudFile V2 fresh schema. Native CE tables remain the authoritative facts.\n-- This file is independent of cloudfile.sql and the legacy migration chain.\n\nCREATE TABLE cf_library_storage (\n repo_key VARCHAR(80) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,\n repo_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,\n state VARCHAR(8) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,\n PRIMARY KEY (repo_key),\n UNIQUE KEY library_storage_repo (repo_id),\n CONSTRAINT library_storage_state CHECK (state IN ('reserved','active','retired'))\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;\n\nCREATE TABLE cf_sso_group_map (\n provider VARCHAR(32) COLLATE utf8mb4_bin NOT NULL,\n subject_type VARCHAR(4) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,\n namespace VARCHAR(128) COLLATE utf8mb4_bin NOT NULL,\n external_id VARCHAR(225) COLLATE utf8mb4_bin NOT NULL,\n group_id BIGINT NOT NULL,\n PRIMARY KEY (provider,subject_type,namespace,external_id),\n UNIQUE KEY sso_native_group (group_id),\n CONSTRAINT sso_group_type CHECK (subject_type IN ('org','role')),\n CONSTRAINT sso_group_id CHECK (group_id > 0)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;\n\nCREATE TABLE cf_dir_acl (\n repo_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,\n path_hash BINARY(32) NOT NULL,\n path VARCHAR(4096) COLLATE utf8mb4_bin NOT NULL,\n subject_type VARCHAR(4) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,\n provider VARCHAR(32) COLLATE utf8mb4_bin NOT NULL,\n namespace VARCHAR(128) COLLATE utf8mb4_bin NOT NULL,\n external_id VARCHAR(225) COLLATE utf8mb4_bin NOT NULL,\n permission VARCHAR(9) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,\n inherit TINYINT NOT NULL,\n share_mode TINYINT NOT NULL,\n revision CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,\n PRIMARY KEY (repo_id,path_hash,subject_type,provider,namespace,external_id),\n KEY acl_shared_subject (provider,subject_type,namespace,external_id,share_mode,repo_id),\n CONSTRAINT acl_subject_type CHECK (subject_type IN ('user','org','role')),\n CONSTRAINT acl_permission CHECK (permission IN ('r','rw','invisible')),\n CONSTRAINT acl_inherit CHECK (inherit IN (0,1)),\n CONSTRAINT acl_share CHECK (share_mode IN (0,1) AND\n (share_mode=0 OR (permission IN ('r','rw') AND path <> '/')))\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;\n\n-- CloudFile V2 optional Rust local editing. Fresh isolated CF-EDIT installations ONLY.\n-- The three-table V2 MVP remains unchanged unless editing is explicitly enabled.\n-- There is NO server-side Intent or editing Session table. Seafile keeps native\n-- File/Block/Commit/Branch versions; Agent Rust keeps its local pending Journal.\n-- One row per physical file path preserves generation across Checkout cycles.\nCREATE TABLE cf_edit_guard (\n repo_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,\n path_hash BINARY(32) NOT NULL,\n path VARCHAR(4096) COLLATE utf8mb4_bin NOT NULL,\n guard_id CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NULL,\n generation BIGINT UNSIGNED NOT NULL DEFAULT 0,\n owner_uid VARCHAR(225) COLLATE utf8mb4_bin NULL,\n owner_native_user VARCHAR(255) COLLATE utf8mb4_bin NULL,\n terminal_type VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NULL,\n terminal_name VARCHAR(128) COLLATE utf8mb4_bin NULL,\n checkout_ip VARCHAR(45) CHARACTER SET ascii COLLATE ascii_bin NULL,\n proof_digest CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NULL,\n base_file_id CHAR(40) CHARACTER SET ascii COLLATE ascii_bin NULL,\n lease_expires_at DATETIME(6) NULL,\n created_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),\n updated_at DATETIME(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),\n PRIMARY KEY (repo_id, path_hash),\n UNIQUE KEY cf_edit_guard_id (guard_id),\n KEY cf_edit_guard_active (repo_id, guard_id),\n KEY cf_edit_guard_owner (owner_uid, guard_id),\n CONSTRAINT cf_edit_terminal_type CHECK (\n terminal_type IS NULL OR terminal_type IN ('windows','macos','linux','other')\n ),\n CONSTRAINT cf_edit_active_shape CHECK (\n (guard_id IS NULL AND owner_uid IS NULL AND owner_native_user IS NULL\n AND proof_digest IS NULL AND base_file_id IS NULL AND lease_expires_at IS NULL)\n OR\n (guard_id IS NOT NULL AND owner_uid IS NOT NULL AND owner_native_user IS NOT NULL\n AND proof_digest IS NOT NULL AND base_file_id IS NOT NULL AND lease_expires_at IS NOT NULL)\n )\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;\n" +TABLES = ('cf_library_storage', 'cf_sso_group_map', 'cf_dir_acl', 'cf_edit_guard') +TARGET_COLUMNS = { + 'cf_library_storage': ('repo_key','repo_id','state'), + 'cf_sso_group_map': ('provider','subject_type','namespace','external_id','group_id'), + 'cf_dir_acl': ('repo_id','path_hash','path','subject_type','provider','namespace','external_id','permission','inherit','share_mode','revision'), + 'cf_edit_guard': ('repo_id','path_hash','path','guard_id','generation','owner_uid','owner_native_user','terminal_type','terminal_name','checkout_ip','proof_digest','base_file_id','lease_expires_at','created_at','updated_at'), +} + +def encoded(value): + if isinstance(value, bytes): return {'base64':base64.b64encode(value).decode()} + if isinstance(value,(datetime.date,datetime.datetime)): return {'iso':value.isoformat()} + if isinstance(value,dict): return {k:encoded(v) for k,v in value.items()} + if isinstance(value,(list,tuple)): return [encoded(x) for x in value] + return value + +def canonical(value): + return json.dumps(encoded(value),ensure_ascii=False,sort_keys=True,separators=(',',':')) + +def digest(rows): + return hashlib.sha256('\n'.join(sorted(canonical(row) for row in rows)).encode()).hexdigest() + +def identifier(name): + if not re.fullmatch(r'[a-zA-Z0-9_]{1,64}',name):raise RuntimeError('invalid SQL identifier') + return '`'+name+'`' + +def definitions(): + text=re.sub(r'--[^\n]*','',DDL) + result={} + for statement in text.split(';'): + if not statement.strip():continue + m=re.match(r'\s*CREATE TABLE (cf_[a-z_]+)\s*\(',statement) + if not m or m[1] not in TABLES or m[1] in result:raise RuntimeError('unexpected target DDL') + result[m[1]]=statement.strip() + if set(result)!=set(TABLES):raise RuntimeError('incomplete target DDL') + return result + +def project(table,rows): + result=[] + for row in rows: + value={name:row[name] for name in TARGET_COLUMNS[table] if name in row} + if table=='cf_dir_acl': + if row.get('kind','dir')!='dir':raise RuntimeError('file ACL has no lossless target mapping') + value['path_hash']=hashlib.sha256(row['path'].encode()).digest() + previous=row['path_hash'] + if isinstance(previous,str):previous=bytes.fromhex(previous) + if previous!=value['path_hash']:raise RuntimeError('source ACL hash is inconsistent') + if str(uuid.UUID(row['revision']))!=row['revision']:raise RuntimeError('invalid ACL revision') + if table=='cf_edit_guard': + # Empty installed Guard was verified. Do not invent a conversion for old + # generation, intent or credential state when a row appears concurrently. + raise RuntimeError('Guard is no longer empty; explicit lease conversion required') + if set(value)!=set(TARGET_COLUMNS[table]):raise RuntimeError('source columns cannot be mapped') + result.append(value) + return result + +def freeze_writers(): + stopped=[] + # Freeze DB-using services only, after requests have had time to drain. + # The migration process itself is excluded. Resume in finally, never rollback. + try: + for proc in Path('/proc').iterdir(): + if not proc.name.isdigit() or int(proc.name)==os.getpid():continue + try:name=(proc/'comm').read_text().strip() + except (OSError,ProcessLookupError):continue + if name in ('seaf-server','fileserver','seafevents','gunicorn','python','python3','seafile-control') or name.startswith('gunicorn'): + pid=int(proc.name) + try:os.kill(pid,signal.SIGSTOP);stopped.append(pid) + except ProcessLookupError:pass + return stopped + except BaseException: + resume_writers(stopped);raise + +def resume_writers(stopped): + for pid in stopped: + try:os.kill(pid,signal.SIGCONT) + except ProcessLookupError:pass + +def main(): + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--apply',action='store_true') + parser.add_argument('--expected-database',required=True) + parser.add_argument('--backup-root',default='/shared/schema-backups') + args=parser.parse_args() + import pymysql + conn=pymysql.connect(host=os.environ['SEAFILE_MYSQL_DB_HOST'],port=int(os.environ.get('SEAFILE_MYSQL_DB_PORT','3306')),user=os.environ['SEAFILE_MYSQL_DB_USER'],password=os.environ['SEAFILE_MYSQL_DB_PASSWORD'],database=os.environ['SEAFILE_MYSQL_DB_SEAFILE_DB_NAME'],charset='utf8mb4',autocommit=True,connect_timeout=5,read_timeout=30,write_timeout=30,cursorclass=pymysql.cursors.DictCursor) + stopped=[];switched=False;receipt={} + try: + with conn.cursor() as q: + q.execute('SELECT DATABASE() AS db') + if q.fetchone()['db']!=args.expected_database:raise RuntimeError('database identity mismatch') + q.execute("SELECT GET_LOCK('cloudfile.dev.v2.four-table-repair',0) AS acquired") + if q.fetchone()['acquired']!=1:raise RuntimeError('another repair is running') + q.execute('SET SESSION lock_wait_timeout=10') + q.execute("SET SESSION sql_mode=CONCAT(@@sql_mode,',STRICT_ALL_TABLES')") + source={};create={} + for table in TABLES: + q.execute('SHOW CREATE TABLE '+identifier(table));create[table]=q.fetchone()['Create Table'] + q.execute('SELECT * FROM '+identifier(table));source[table]=q.fetchall() + q.execute("SELECT COUNT(*) AS n FROM information_schema.KEY_COLUMN_USAGE WHERE REFERENCED_TABLE_SCHEMA=DATABASE() AND REFERENCED_TABLE_NAME IN (%s,%s,%s,%s)",TABLES) + if q.fetchone()['n']:raise RuntimeError('foreign key dependency needs an explicit migration') + target={t:project(t,rows) for t,rows in source.items()} + receipt={'source_counts':{t:len(rows) for t,rows in source.items()},'source_digests':{t:digest(rows) for t,rows in source.items()},'target_digests':{t:digest(rows) for t,rows in target.items()},'ddl_sha256':hashlib.sha256(DDL.encode()).hexdigest(),'database':args.expected_database,'apply':args.apply,'switched':False} + print(json.dumps(receipt,sort_keys=True),flush=True) + if not args.apply:return + stamp=datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%dT%H%M%SZ')+'_'+uuid.uuid4().hex[:6] + backup_root=Path(args.backup_root);backup_root.mkdir(parents=True,exist_ok=True,mode=0o700) + backup=backup_root/('four-tables-'+stamp+'.json') + # Export under private permissions before any table-changing statement. + fd=os.open(str(backup),os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600) + with os.fdopen(fd,'w') as out: + out.write(canonical({'receipt':receipt,'create':create,'rows':source}));out.flush();os.fsync(out.fileno()) + stages={t:'cfv2_new_'+stamp+'_'+t[3:] for t in TABLES} + backups={t:'cfv2_old_'+stamp+'_'+t[3:] for t in TABLES} + for table,statement in definitions().items(): + q.execute(statement.replace('CREATE TABLE '+table,'CREATE TABLE '+identifier(stages[table]),1)) + stopped=freeze_writers() + # Re-read after freezing, because discovery and backup happened while live. + # Any concurrent change aborts before copying or renaming an original table. + for table in TABLES: + q.execute('SHOW CREATE TABLE '+identifier(table)) + if q.fetchone()['Create Table']!=create[table]:raise RuntimeError('source DDL changed during preparation') + q.execute('SELECT * FROM '+identifier(table)) + if digest(q.fetchall())!=receipt['source_digests'][table]:raise RuntimeError('source data changed during preparation') + for table,rows in target.items(): + cols=TARGET_COLUMNS[table] + if rows:q.executemany('INSERT INTO '+identifier(stages[table])+' ('+','.join(identifier(c) for c in cols)+') VALUES ('+','.join(['%s']*len(cols))+')',[tuple(row[c] for c in cols) for row in rows]) + q.execute('SELECT '+','.join(identifier(c) for c in cols)+' FROM '+identifier(stages[table])) + if digest(q.fetchall())!=receipt['target_digests'][table]:raise RuntimeError('staged data verification failed') + # One atomic RENAME switches all four names together. No DROP, no loss of + # old data, and no automatic rollback that could erase subsequent writes. + parts=[] + for table in TABLES:parts.extend((identifier(table)+' TO '+identifier(backups[table]),identifier(stages[table])+' TO '+identifier(table))) + q.execute('RENAME TABLE '+','.join(parts));switched=True + receipt.update(switched=True,backup_file=str(backup),backup_tables=backups) + for table in TABLES: + cols=TARGET_COLUMNS[table] + q.execute('SELECT '+','.join(identifier(c) for c in cols)+' FROM '+identifier(table)) + if digest(q.fetchall())!=receipt['target_digests'][table]:raise RuntimeError('post-switch data verification failed') + receipt_path=backup_root/('four-tables-'+stamp+'-receipt.json') + fd=os.open(str(receipt_path),os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600) + with os.fdopen(fd,'w') as out:json.dump(receipt,out,sort_keys=True);out.flush();os.fsync(out.fileno()) + print(json.dumps(receipt,sort_keys=True),flush=True) + finally: + resume_writers(stopped) + conn.close() + +if __name__=='__main__': + try:main() + except Exception as error: + # SQL errors may embed row values or connection secrets. Emit only a class + # and numeric driver code; private data remains in the controlled backup. + code=error.args[0] if error.args and isinstance(error.args[0],int) else None + print(json.dumps({'repair_failed':type(error).__name__,'sql_code':code,'reason':str(error) if type(error) is RuntimeError else 'private driver details withheld'}),flush=True) + raise SystemExit(1) diff --git a/tools/reviewed-sql-contracts.json b/tools/reviewed-sql-contracts.json new file mode 100644 index 00000000..fadd12a8 --- /dev/null +++ b/tools/reviewed-sql-contracts.json @@ -0,0 +1,47 @@ +{ + "hub/001_resources.json": "f593e975b816a6fe40552f399250c59546afadd3bcb4c767326afbdbe936fd8c", + "hub/002_jobs.json": "c264c1b971dbbd2e46cdf10125b421b1246dffe7bfe1e328bf69cee20cd32dc4", + "hub/003_outbox.json": "e7e8f3a1572a6a8fe5bb4cb1a8aafe132ab00238003abbaf4eb656f1482686ba", + "hub/004_audit.json": "c5d93cfa4273bb60877693fe2bed7aebee713fd31f1161eb5b5ef6b6dcf370f4", + "hub/005_audit_query.json": "1c0afcbaf290bb76dbcbd3cc0655f0c72123404afb69f8c14784242cc0deb919", + "hub/006_group_maps.json": "52abe80df2a8e024040bb7fe39a623706fae9e4086978c165ec5c36a1f22007a", + "hub/007_group_map_contract.json": "33da4efb158d663d0ddfff1f3aeda42b1cc053194fbb8fbfa95a43307e1373a5", + "hub/008_directory_acl.json": "3bb7fa18732a5098ccf6514079c8fdd84e6f7d686ad22fb0b634db18cffe11e4", + "hub/009_directory_admins.json": "fd9de33e8c9e13b6ac593c89e9c4bd4447d957664587ab05a63a3933328248d9", + "hub/010_policy_requests.json": "3abec80d2652b1428eab00a8f75013b1c8bc797d36e7310141b7787b974572e7", + "hub/011_tags.json": "1a86f7d28f9bafad3cf68f06104b22a954625fd5c0e360af0210181833556fa0", + "hub/012_oidc_sessions.json": "06eac60da0d7aa725ee5562e104a27c11ed29df3a7ae5c8f253aa6115b670d2c", + "hub/013_oidc_logout_fences.json": "0bbfdf2a4d6bbeec9d96da31c1ddaaaf5254ef19b06d148d257073a1b76a6f41", + "hub/014_oidc_scope_expiry.json": "45bfbafa8dd618afbed014accd8a1326dd078b7a71ed9cabc890c3b23c740974", + "hub/015_search_tasks.json": "a3a267fa6241d14a317c14b7e44cb97c5d87f3073f71a3f30778cc9f18bcd71c", + "hub/016_search_plans.json": "7184fec95eb8bd607a19e94d241a57a107c26deb7088e4dfd3e5dc4ff2de6e0e", + "hub/017_search_fanout.json": "bd1169626b968ecec4181cf4d60aa10a2290e89903831745faecd658b66810ba", + "hub/018_search_generations.json": "5e700d57cf21d6fdfedccd2d605730810ad5f6880f875ee635e16a62cc11bb88", + "hub/019_search_initialization.json": "c06e869b6f836b7eb95db1afa4ddf8b2fc27f516632bcba738a7edd97a5a5e43", + "hub/020_search_rebuild.json": "84b0a09802bb502d6e7a2bccb126b98ea9c0889f2df807787c7a9e84b33e8a1e", + "hub/021_search_pending_index.json": "76c7325d4eccadd4f538faeaf22ab4e8322581edd7affce78476865400466468", + "hub/022_search_plan_generation_indexes.json": "0bbc32d15d042ede1ced59f98df10a42161a325e5dc06b43b0f5c25f77ff43bb", + "hub/023_outbox_stream_sequence.json": "28d4315bed679fa7b17e187fa7aa3073a5f6f37cb824836c6f6ae33e516cfcfd", + "hub/024_search_fanout_pages.json": "7b1f6d861b20c09ee51c8b2d721fa589e177ab1380ccbe869311757f6067afad", + "hub/025_search_catchup.json": "eb8a45c5544f20f4096322b3188659600e3489fb76e2e7fa75330ae3a69ca528", + "hub/026_search_global_fanout_scope.json": "6e12456da505ed15acbb792f151b67eb98be62c9b6214a8395fea35c25c6f15f", + "hub/027_search_global_catchup.json": "d0eddca075fac956b03e0faf3dd33668e5c8c84fdf696a506241d34b7ffe90e3", + "hub/028_search_publication.json": "fa03bbcff7726f00ecbcdc4dfebcdfffa10879a6518273e1d6a129dbdc8be256", + "hub/029_editing_core.json": "2314f55fbd29f1cbb2d9b1fd3b90feac3a4ba95b97fe934abe4bcc447262568f", + "hub/030_local_devices.json": "f88021b19837e91ccd82d536dd9a32295636d1628ef5ba4e35b4f0c7fae5037f", + "hub/031_edit_sessions.json": "324338f9cc082ee6cab53bbe8dde5b28efba75197fa636779dfd567b1efd5fe9", + "hub/032_managed_libraries.json": "c2d4705f332602b67fe9a6623692b0a052a811b5f13e57fb7eb24dcf5995ba3e", + "hub/033_audit_admin_page.json": "84ee659fba98cea6d1365875cc52085f7230bb3cf86e6af31f74964f4e9d17c4", + "hub/034_audit_client_ip.json": "9bb046750307b332a40dcc958f439df8ebd18ee17d4575cf5919924341c92a8e", + "hub/035_library_shares.json": "fef034f1e49c02dc7a149bfdc49285e8328a73c40a6133aef2fe836e2ef22fa9", + "hub/036_library_acl_page.json": "c6704a220883a8d8648a8d63ff6cd876a470fa4157040cc178ed435361cba4c2", + "hub/037_device_checkout_lease.json": "0ca3e86a4222e4fb597a16a27599c9c554f3d0589f839d65b4a1a0b55b4aba5e", + "hub/038_device_block_plan.json": "ad14ef6bbac3ab1b9ebd04d5a2fd78345a8a07bfb04eb88434a2ce3c6bb01917", + "hub/039_optimistic_upload_intent.json": "b07bb3a40000656ce4705859b3a243c73a63867d25b2ec59baaf89b7d7546dff", + "hub/040_directory_shares.json": "5a02dbcfa00c0a047600aa01814cc5107aee4b89c4095da993ff285674f849e6", + "hub/041_directory_admin_page.json": "9be58c5e8c2830d031725edfea6ba38ea797df5d6bf1892297b529302bf02ff9", + "hub/042_unified_directory_acl.json": "b94945e988e1549addb0d334a60a3e72b06692570082fb813b3c6b5c06872865", + "server/cloudfile-v2-edit.sql": "671ac9d117355b43117ba66e1d4961d0cb6216126ff84dcc208d5a50a17238e5", + "server/cloudfile-v2.sql": "8950ef43d4811992109e9a3526023faa65cd5e51a53a94ed0f21dedc1ade5196", + "server/cloudfile.sql": "0af48853f485deaf64864df0262edbbbb5d86b82f01683f0bb21d4993119a517" +} diff --git a/tools/run-checks.sh b/tools/run-checks.sh index b2d80f77..9fb837fb 100755 --- a/tools/run-checks.sh +++ b/tools/run-checks.sh @@ -83,8 +83,15 @@ run "前端产物完整性契约" "$docker_repo/tests/tools/test-production-arti run "最终发布包完整性契约" "$docker_repo/tests/tools/test-release-artifact.sh" # Validate the exact dependency patch payload, not an unrelated local checkout. run "自动本地存储 Python 策略" python3 -m pytest "$docker_repo/tests/test_automatic_local_storage.py" -q +# Table names can belong to incompatible reviewed layouts. Check actual DDL +# and optional Checkout readers before any build; this never connects to a DB. +run "跨仓 SQL 表结构契约" python3 "$docker_repo/tools/check-sql-contracts.py" --server "$server" --hub "$hub" +run "SQL 结构漂移回归" python3 "$docker_repo/tests/test_sql_contracts.py" --server "$server" --hub "$hub" +# Explicit DEV conversion must preserve rule/group identity and reject live Guards. +run "V2 四表修复数据映射" python3 "$docker_repo/tests/test_repair_v2_four_tables.py" # Fresh DDL must remain resumable and must never turn restart into an upgrade. -run "新部署 DDL 初始化" python3 -m unittest discover -s "$docker_repo/tests" -p test_fresh_schema.py -q +run "开发期只读 Schema 启动校验" python3 -m unittest discover -s "$docker_repo/tests" -p test_fresh_schema.py -q +run "独立 SQL 交付契约" python3 -m unittest discover -s "$docker_repo/tests" -p test_manual_sql_package.py -q run "v0.6 新安装每库独立目录 Compose 契约" python3 -m unittest discover -s "$docker_repo/tests" -p test_v06_default_library_storage.py -q run "RustFS/MinIO 存储类与 Compose 契约" python3 -m unittest discover -s "$docker_repo/tests" -p test_storage_class_config.py -q # CloudFile v0.5 W1-W3 standalone checks, no live Docker/DB dependencies.