From 30e1f5a7e7b411210bcbc1eea27ed6b61e1f1895 Mon Sep 17 00:00:00 2001 From: dev9-bb Date: Tue, 20 Oct 2026 17:37:40 +0800 Subject: [PATCH 1/4] fix(storage): preflight explicit auto mode and preserve legacy config --- deploy/compose/.env.example | 6 +- .../docker-compose.auto-library-storage.yml | 38 ++-- deploy/compose/docker-compose.yml | 2 + docs/features/automatic-local-storage.md | 93 +++++--- scripts/scripts_14.0/bootstrap.py | 43 ++-- scripts/scripts_14.0/library_storage.py | 180 +++++++++++++++ tests/test_library_storage_config.py | 211 ++++++++++++++++++ tests/test_settings_block_order.py | 59 +++++ tests/test_v06_default_library_storage.py | 19 +- tools/test-bootstrap-settings.py | 1 + 10 files changed, 563 insertions(+), 89 deletions(-) create mode 100644 scripts/scripts_14.0/library_storage.py create mode 100644 tests/test_library_storage_config.py create mode 100644 tests/test_settings_block_order.py diff --git a/deploy/compose/.env.example b/deploy/compose/.env.example index 690ebd84..5d4c0a2a 100644 --- a/deploy/compose/.env.example +++ b/deploy/compose/.env.example @@ -17,9 +17,11 @@ # Fresh installs use ONE parent mount; every new library gets its own folder. # Existing .env and existing storage mapping remain untouched. Do NOT paste # this into a running legacy/S3 installation without a migration review. -# To keep the classic local CE layout on a fresh installation, use -# COMPOSE_FILE=docker-compose.yml instead of this default. +# Existing installations leave CF_LIBRARY_STORAGE unset until explicitly +# switching after inventory validation. New installations default to auto. +# Create CF_LIBRARY_STORAGE_HOST_DIR before starting; missing bind sources fail. COMPOSE_FILE=docker-compose.yml:docker-compose.auto-library-storage.yml +CF_LIBRARY_STORAGE=auto CF_LIBRARY_STORAGE_HOST_DIR=./data/libraries # --- image ----------------------------------------------------------------- diff --git a/deploy/compose/docker-compose.auto-library-storage.yml b/deploy/compose/docker-compose.auto-library-storage.yml index 2ccb9acd..79c4f869 100644 --- a/deploy/compose/docker-compose.auto-library-storage.yml +++ b/deploy/compose/docker-compose.auto-library-storage.yml @@ -1,30 +1,22 @@ -# Optional for existing deployments; included by default for NEW deployments -# created from .env.example. Never add this to an existing .env without checking -# its current storage classes, RepoStorageId bindings, and backup inventory. -# -# One parent mount for all new libraries; server/worker share the same storage. -# No per-library bind mount, new storage class file or container restart. -# An auto-local template is NOT the fallback storage class: legacy-default -# continues to serve old libraries without an explicit RepoStorageId mapping. +# New-install V2 local storage. Existing installations retain their persisted +# storage configuration unless CF_LIBRARY_STORAGE=auto is explicitly selected. +# Prepare this host directory first; Docker must not silently create an empty one. services: cloudfile: volumes: - - ${CF_LIBRARY_STORAGE_HOST_DIR:-./data/libraries}:/shared/seafile/library-storage + - type: bind + source: ${CF_LIBRARY_STORAGE_HOST_DIR:-./data/libraries} + target: /shared/seafile/library-storage + bind: + create_host_path: false environment: - # Existing multi-storage bootstrap switch also enables local classes. - # This does NOT configure S3, despite the historical switch name. - CF_ENABLE_S3_STORAGE: "true" - SEAF_SERVER_STORAGE_TYPE: multiple - CF_STORAGE_CLASSES_JSON: >- - [ - {"storage_id":"legacy-default","storage_name":"Existing local libraries","is_default":true,"commits":{"backend":"fs","dir":"/shared/seafile/seafile-data"},"fs":{"backend":"fs","dir":"/shared/seafile/seafile-data"},"blocks":{"backend":"fs","dir":"/shared/seafile/seafile-data"}}, - {"storage_id":"auto-local","storage_name":"New library directories","is_auto":true,"is_default":false,"commits":{"backend":"fs","dir":"/shared/seafile/library-storage"},"fs":{"backend":"fs","dir":"/shared/seafile/library-storage"},"blocks":{"backend":"fs","dir":"/shared/seafile/library-storage"}} - ] + CF_LIBRARY_STORAGE: ${CF_LIBRARY_STORAGE:-} cf-worker: volumes: - - ${CF_LIBRARY_STORAGE_HOST_DIR:-./data/libraries}:/shared/seafile/library-storage + - type: bind + source: ${CF_LIBRARY_STORAGE_HOST_DIR:-./data/libraries} + target: /shared/seafile/library-storage + bind: + create_host_path: false environment: - # Match the main server mode. The worker reads the same shared - # seafile.conf and storage class file generated by cloudfile bootstrap. - CF_ENABLE_S3_STORAGE: "true" - SEAF_SERVER_STORAGE_TYPE: multiple + CF_LIBRARY_STORAGE: ${CF_LIBRARY_STORAGE:-} diff --git a/deploy/compose/docker-compose.yml b/deploy/compose/docker-compose.yml index 984952bd..696123fb 100644 --- a/deploy/compose/docker-compose.yml +++ b/deploy/compose/docker-compose.yml @@ -224,6 +224,7 @@ services: CLOUDFILE_EDITING_ENABLED: ${CLOUDFILE_EDITING_ENABLED:-false} CLOUDFILE_RESOURCE_SECRET: ${CLOUDFILE_RESOURCE_SECRET:-} CF_ENABLE_LOCAL_APP: ${CF_ENABLE_LOCAL_APP:-false} + CF_LIBRARY_STORAGE: ${CF_LIBRARY_STORAGE:-} CF_ENABLE_S3_STORAGE: ${CF_ENABLE_S3_STORAGE:-false} SEAF_SERVER_STORAGE_TYPE: ${SEAF_SERVER_STORAGE_TYPE:-} S3_COMMIT_BUCKET: ${S3_COMMIT_BUCKET:-} @@ -483,6 +484,7 @@ services: CLOUDFILE_EDITING_ENABLED: ${CLOUDFILE_EDITING_ENABLED:-false} CLOUDFILE_RESOURCE_SECRET: ${CLOUDFILE_RESOURCE_SECRET:-} CF_ENABLE_LOCAL_APP: ${CF_ENABLE_LOCAL_APP:-false} + CF_LIBRARY_STORAGE: ${CF_LIBRARY_STORAGE:-} CF_ENABLE_S3_STORAGE: ${CF_ENABLE_S3_STORAGE:-false} CF_ENABLE_EXTERNAL_SOURCES: ${CF_ENABLE_EXTERNAL_SOURCES:-false} CF_EXTERNAL_SOURCES_ROOTS: ${CF_EXTERNAL_SOURCES_ROOTS:-/shared/external} diff --git a/docs/features/automatic-local-storage.md b/docs/features/automatic-local-storage.md index 85e83d15..47aacfcd 100644 --- a/docs/features/automatic-local-storage.md +++ b/docs/features/automatic-local-storage.md @@ -1,45 +1,74 @@ -# 默认按库自动分配本地存储 +# 按库自动分配本地存储 -更新:2026-10-11。**v0.6 复用已实现的 C/Go/Python 路由**,本次只补新安装默认 Compose 配置、契约验证和运维说明,不重做用户已在一套 CloudFile 环境验证的真实库。2026-10-08 原目标仍成立:逐库登记存储类和 bind mount 会使每次建库依赖运维修改部署并重启。一个父目录挂载,后端首次使用时按稳定 key 创建子后端。 +更新:2026-10-20。当前 `dev` 使用 V2 本地路由,物理对象布局保持 +`//storage//`。不搬数据、不改 Repo ID 或对象格式。 +Seafile 原生 `RepoStorageId` 是库路由事实来源;不读取、转换或清理旧表 +`cf_library_storage_key`,该表原样保留。 -## 新安装的开箱即用配置(不改变已有部署) +## 新安装 -1. **仅新安装**:在 `deploy/compose/` 内执行 `cp .env.example .env`,新模板默认设置 `COMPOSE_FILE=docker-compose.yml:docker-compose.auto-library-storage.yml`。之后仍使用 `docker compose up -d`;不需要逐个创建库时修改 Compose 或静态存储类。 -2. 覆盖文件 `deploy/compose/docker-compose.auto-library-storage.yml` 把 `CF_LIBRARY_STORAGE_HOST_DIR=./data/libraries` **一次性**挂载至主服务及 `cf-worker` 的 `/shared/seafile/library-storage`,该目录必须持久化并包含在运维备份清单内。若调整父目录,只改新装环境这一项,并在部署前确认源目录与权限;**不得把旧库绑定目录当作新的空目录**。 -3. 新模板沿用当前多存储引导兼容开关 `CF_ENABLE_S3_STORAGE=true` 与 `SEAF_SERVER_STORAGE_TYPE=multiple`,这是已有代码的历史开关名称,**实际配置为本地文件系统,没有 S3 后端或凭据**。存储类 `legacy-default` 是已有 Seafile 本地对象路径,`auto-local` 为非默认存储类模板,供原生建库绑定到每库子目录。 -4. 现有 CloudFile / DEV / PROD 的 `.env` 和持久化 `seafile.conf` **完全不自动变更**。已经启用独立父目录的环境保持原挂载、配置和 `RepoStorageId`。新版本更新文件不会自动覆盖原有 `.env`,但**手工套用新 overlay 就属于存储变更**,必须先核对旧库位置、实例备份与回退,禁止无迁移方案直接切换已有 S3/legacy 实例。 -5. 不采用每库独立目录的**全新安装**可把 `.env` 的 `COMPOSE_FILE` 改回 `docker-compose.yml`;特殊 S3/存储类部署沿用已有显式配置及受控存储迁移工具,不与新默认模板叠加。 +1. 从 `deploy/compose/.env.example` 创建 `.env`,新模板明确设置 + `CF_LIBRARY_STORAGE=auto`,并选择 `docker-compose.auto-library-storage.yml`。 +2. 运维先创建宿主机 `CF_LIBRARY_STORAGE_HOST_DIR` 指定的真实目录,并设置服务读写权限; + 默认是 `deploy/compose/data/libraries`。Compose 使用 `create_host_path: false`, + 不会悄悄创建缺失的挂载源。不要把旧对象目录替换成空目录。 +3. 主服务和 worker 一次性挂载同一个父目录,容器内统一为 + `/shared/seafile/library-storage`。启动时还会拒绝根或任一级祖先为符号链接、 + 根不存在或不是目录;不会自动创建存储根。 +4. bootstrap 写 `[cloudfile_v2] auto_local_root=/shared/seafile/library-storage`, + `[storage] enable_storage_classes=false`,三个对象后端均使用 `name=fs`。 + 新模式不生成 multi 类,也不依赖 `CF_ENABLE_S3_STORAGE`。现有 V2 Server 在首个 + commit 前预留库目录,并使用已有 active/retired 生命周期;此配置不重写路由实现。 -**仅做配置解析核对(不启动容器、不写库)**: +直接运行 bootstrap 的新装也默认 auto,但显式选择旧 S3/multiple 变量的安装保留旧模式。 +只有新装判断能使用默认值;不能用“环境变量缺失”判定现有实例需要切换。 -```bash -cd deploy/compose -docker compose --env-file .env.example config --quiet -# 检查组合配置中的 cloudfile / cf-worker 父目录挂载及两个 storage class -python3 -m unittest discover -s ../../tests -p test_v06_default_library_storage.py -v -``` - - -## 契约 +## 已有安装与显式切换 -- 宿主机父目录可按环境设置(新装默认为 `./data/libraries`,已有实际部署可采用其他绝对路径),统一挂载为容器内 `/shared/seafile/library-storage`;主服务和 worker 使用同一读写挂载。**每库子目录由存储引擎自行处理,禁止逐库增加 Compose mount。** -- 存储类 JSON 保留既有 `legacy-default`,新增唯一模板 `auto-local`,`is_auto: true`、`is_default: false`,三个 `fs` 后端的 `dir` 均为上述容器父目录。模板不是一个可直接写入的共享后端。 -- 新库未显式选择后端时,原生建库在初始 commit 前绑定 `auto-local:`。CloudFile 建库 API 可提交 `repo_key`,此时绑定 `auto-local:`。默认导入提交已有配置中的 key,不新增部署配置或重启。 -- key 为 1–80 个 ASCII 小写字母、数字或连字符,首尾必须为字母或数字。目录为父目录/key;拒绝路径穿越和 key 目录符号链接。 -- `cf_library_storage_key` 使用 key 主键、repo UUID 唯一约束,原子保留绑定。永久删除保留绑定记录作为 GC 定位和 key 占用记录,避免旧对象混入同名新库。显示名变更不改变 key。 -- 既有未绑定库仍走 `legacy-default`;既有显式本地/S3 存储类继续使用原配置。自定义磁盘通过预登记的例外存储类选择,客户端不能指定任意宿主机路径。 -- C、Go fileserver、Python 对象读取器遵守同一 ID 和目录规则,未知存储类仍拒绝回落。虚拟库继承原库绑定。GC 通过保留的 key 记录清理正确子目录,不能依赖进程已缓存该库。 +已有 `.env`、Seafile 对象目录和存储类继续使用原配置。**升级镜像、更新 Compose 文件、 +或者没有设置 `CF_LIBRARY_STORAGE` 都不会自行切换旧实例。** 旧组合 +`CF_ENABLE_S3_STORAGE=true`、`SEAF_SERVER_STORAGE_TYPE=multiple` 和 auto-local 模板 +继续按旧方式运行。旧 overlay 的环境默认值被移除后,bootstrap 仍保留已落盘的旧后端段。 +已有 V2 `auto_local_root` 的实例在重启时保持 V2,并仍执行预检。 -## 验证要求 +切换需要运维显式设置 `CF_LIBRARY_STORAGE=auto`,并保留正确的父目录挂载。在改写 +`seafile.conf` 前,bootstrap 用临时候选配置执行 Hub 当前的统一 CLI;候选配置保留原 +`[storage]`、原存储类 JSON 文件以及原数据库,只有 V2 root 补成上述固定位置。 +不会使用独立 `cfv2_*` 数据库。连接信息通过环境变量传递,不放入命令参数。 -跨层 key 用例位于 `automatic-local-storage-cases.json`。除单位测试外,须以同一运行镜像验证:不修改 Compose/存储 JSON、不重启容器,连续创建两个不同 key 的库,上传下载及目录/文件搜索,重启后读取,重复 key 拒绝,原有库读取,以及永久删除后的物理对象清理。 +```text +python -m cloudfile_ext.v2 storage-reconcile \ + --seafile-conf --seafile-data --dry-run +``` -普通建库入口未填写 key 时使用 UUID,可保证无需配置即可独立存储;组织命名 key 由受控建库 API 或通用导入配置指定。网页上的专用 key 输入界面不在此变更范围。 +- CLI 输出完整 JSON 报告,bootstrap 原样输出;`BLOCKED`、执行失败或报告格式异常均拒绝启动。 +- `NATIVE_CLASS_PINNED` 也会阻止切换。它表示原生默认类仍被 `RepoStorageId` 引用, + 不能直接关闭存储类。运维须检查报告,再针对选定 ID 显式执行 + `storage-reconcile --apply --normalize-native-class `,bootstrap 不自动选择类或删除路由。 +- 预检通过后,bootstrap 再执行同一 CLI 的 `--apply`,仅补齐受支持原生库和自动库的 + `cf_library_storage` 登记,之后才原子替换为 filesystem 配置。 +- dry-run/apply 任一步失败,原 `seafile.conf` 保持原样。apply 以每库事务执行,跨库部分 + 成功可能已经登记,重试幂等;不会以配置回滚掩盖未完成的预检。 +- 不支持的 S3/其他后端、非法 key、缺失/符号链接目录及路由冲突均需处理后重新预检。 + `ORPHAN_DIRECTORY` 只报告,不登记、不删除。 -## 构建接线 +CLI 使用 `CF_V2_DB_HOST/PORT/USER/PASSWORD/NAME`;bootstrap 从现有 +`SEAFILE_MYSQL_DB_*` 变量传入,保持实例原数据库。首次缺表时 CLI 只幂等建立当前合同的 +`cf_library_storage`,不读取旧表。以上不是自动数据迁移;切换执行及 DEV/生产发布需按环境审批。 -Python 依赖保持锁定上游 SHA,`patches/seafobj/automatic-local-storage.patch` 提供惰性字典及虚拟库原库路由,`cloudfile-build.sh` 对 seafobj 应用补丁,失败即终止构建。C/Go 代码在 server fork 内;本次生产候选基于原生产 `d93b51b` 加存储改动单独构建,避免带入较新开发分支的编辑/RPC 改动。 +## 验证与边界 -## 提交门禁 +```bash +python3 -m unittest discover -s tests -p test_library_storage_config.py -v +python3 -m unittest discover -s tests -p test_settings_block_order.py -v +python3 -m unittest discover -s tests -p test_v06_default_library_storage.py -v +python3 tools/test-bootstrap-settings.py +cd deploy/compose +# 只解析配置,不启动容器或创建目录。 +docker compose --env-file .env.example config --quiet +``` -新增 key 保留表对应 `release.yaml` 的 `database_schema: 3`,启动仍以 `IF NOT EXISTS` 增量建表。统一 `tools/run-checks.sh` 执行补丁内 Python 策略、C 共享 key 用例和 Go 对象存储路由测试。C 用例由 Python 读取共享 JSON 并调用真实策略,仅依赖 glib,避免为纯策略测试引入原生 JSON 库。 +专项检查覆盖新装、旧环境保留、显式切换、根校验、原配置库存预检、BLOCKED/PINNED 拒绝、 +apply 失败配置不变及重启幂等。生成配置块保留原位置,避免本地别名或覆盖变量在重启时失效。 +真实旧库、跨挂载上传下载、历史版本、恢复、GC 和 fsck 仍需匹配 Server/Hub 镜像及数据副本验证; +配置测试不代表 DEV/生产验收完成。 diff --git a/scripts/scripts_14.0/bootstrap.py b/scripts/scripts_14.0/bootstrap.py index 512a212b..eccade55 100755 --- a/scripts/scripts_14.0/bootstrap.py +++ b/scripts/scripts_14.0/bootstrap.py @@ -124,27 +124,19 @@ def _replace_block(path, begin, end, body): with open(path, 'r') as fp: lines = fp.readlines() - out, skipping = [], False - for line in lines: - if line.strip() == begin: - skipping = True - continue - if skipping: - if line.strip() == end: - skipping = False - continue - out.append(line) - - while out and out[-1].strip() == '': - out.pop() - + # Preserve local aliases and overrides after the generated block. + starts = [i for i, line in enumerate(lines) if line.strip() == begin] + ends = [i for i, line in enumerate(lines) if line.strip() == end] + block = begin + '\n' + body + end + '\n' + if starts or ends: + if len(starts) != 1 or len(ends) != 1 or ends[0] < starts[0]: + raise ValueError('invalid generated settings markers in %s' % path) + updated = ''.join(lines[:starts[0]]) + block + ''.join(lines[ends[0] + 1:]) + else: + updated = ''.join(lines).rstrip() + ('\n\n' if lines else '') + block with open(path, 'w') as fp: - fp.writelines(out) - if out: - fp.write('\n\n') - fp.write(begin + '\n') - fp.write(body) - fp.write(end + '\n') + fp.write(updated) + def write_cloudfile_settings(): @@ -846,6 +838,12 @@ def write_cloudfile_seafile_conf(): dir_acl_enabled -- so a new capability needs no change here. """ lines = _seafile_conf_cloudfile_lines() + from library_storage import auto_storage_enabled, configure_auto_storage, retained_legacy_storage + conf_path = join(topdir, 'conf', 'seafile.conf') + fresh = globals().get('_cloudfile_fresh_install', False) + if auto_storage_enabled(conf_path, os.environ, fresh=fresh): + configure_auto_storage(conf_path, ''.join(lines), installdir) + return # Keep the S3 configuration in the same restart-safe generated block as # the feature switches. The server processes need these values in @@ -921,6 +919,8 @@ def write_cloudfile_seafile_conf(): if name != 'bucket': lines.append('%s = %s\n' % (name, value)) + if not cf_enabled('CF_ENABLE_S3_STORAGE'): + lines.append(retained_legacy_storage(conf_path)) _replace_block(join(topdir, 'conf', 'seafile.conf'), CF_BEGIN, CF_END, ''.join(lines)) @@ -1457,6 +1457,9 @@ def apply_sso_hierarchy_schema_compatibility(): def init_seafile_server(): + global _cloudfile_fresh_install + _cloudfile_fresh_install = (not exists(join(shared_seafiledir, 'seafile-data')) or + exists(join(shared_seafiledir, '.cloudfile-schema-init-pending'))) version_stamp_file = get_version_stamp_file() if exists(join(shared_seafiledir, 'seafile-data')): if not exists(version_stamp_file): diff --git a/scripts/scripts_14.0/library_storage.py b/scripts/scripts_14.0/library_storage.py new file mode 100644 index 00000000..c6a0a6d8 --- /dev/null +++ b/scripts/scripts_14.0/library_storage.py @@ -0,0 +1,180 @@ +"""Explicit V2 local-storage transition, using the Hub's native inventory CLI.""" +from configparser import ConfigParser +from io import StringIO +import json +import os +from pathlib import Path +import re +import stat +import subprocess +import sys +import tempfile + +LIBRARY_ROOT = '/shared/seafile/library-storage' +BEGIN = '# --- CloudFile (generated, do not edit) ---' +END = '# --- end CloudFile ---' +BACKEND_SECTIONS = frozenset(('storage', 'commit_object_backend', 'fs_object_backend', 'block_backend')) + + +def _read_config(path): + config = ConfigParser(interpolation=None) + if path.exists(): + with path.open() as stream: + config.read_file(stream) + return config + + +def auto_storage_enabled(path, environment, *, fresh=False): + mode = environment.get('CF_LIBRARY_STORAGE', '').strip() + if mode not in ('', 'auto'): + raise ValueError('CF_LIBRARY_STORAGE must be auto or unset') + if mode == 'auto': + return True + if fresh and environment.get('CF_ENABLE_S3_STORAGE', 'false').lower() != 'true' and not environment.get('SEAF_SERVER_STORAGE_TYPE', ''): + return True + # Retain an already selected V2 mode across restarts without opting an + # existing legacy installation into a transition when its variable is absent. + return bool(_read_config(Path(path)).get('cloudfile_v2', 'auto_local_root', fallback='')) + + +def retained_legacy_storage(path): + """Preserve persisted generated multi/S3 settings when legacy mode is unset. + + The old Compose overlay supplied values independently of .env; removing + that overlay's defaults must not erase those persisted backend settings. + """ + path = Path(path) + if not path.exists(): + return '' + current = _read_config(path) + outside = ConfigParser(interpolation=None) + outside.read_string(_without_generated(path.read_text())) + retained = ConfigParser(interpolation=None) + for section in sorted(BACKEND_SECTIONS): + if current.has_section(section) and not outside.has_section(section): + retained[section] = dict(current.items(section)) + output = StringIO() + retained.write(output) + return output.getvalue() + + +def _require_root(): + root = Path(LIBRARY_ROOT) + for path in (root, *root.parents): + if path.is_symlink(): + raise ValueError('Storage root and its ancestors must not be symbolic links: ' + str(path)) + if not root.is_dir(): + raise ValueError('Storage root must already exist as a directory: ' + str(root)) + + +def _without_generated(text): + lines = text.splitlines(keepends=True) + starts = [i for i, line in enumerate(lines) if line.strip() == BEGIN] + ends = [i for i, line in enumerate(lines) if line.strip() == END] + if not starts and not ends: + return text + if len(starts) != 1 or len(ends) != 1 or ends[0] < starts[0]: + raise ValueError('Invalid generated configuration markers') + return ''.join(lines[:starts[0]] + lines[ends[0] + 1:]) + + +def _render_auto(original, generated_body): + # Explicit transition owns the storage backend sections. Preserve other + # operator sections verbatim; retire legacy classes only after preflight. + out, skipping = [], False + for line in _without_generated(original).splitlines(keepends=True): + match = re.match(r'^\s*\[([^]]+)\]', line) + if match: + skipping = match.group(1).strip() in BACKEND_SECTIONS or match.group(1).strip() == 'cloudfile_v2' + if not skipping: + out.append(line) + body = generated_body + '\n[cloudfile_v2]\nauto_local_root = %s\n' % LIBRARY_ROOT + body += '\n[storage]\nenable_storage_classes = false\n' + for section in ('commit_object_backend', 'fs_object_backend', 'block_backend'): + body += '\n[%s]\nname = fs\n' % section + return ''.join(out).rstrip() + '\n\n' + BEGIN + '\n' + body + END + '\n' + + +def configure_auto_storage(conf, generated_body, installdir, environment=None): + """Preflight old routing, register supported libraries, then replace config. + + Neither a failed preflight nor a failed apply changes seafile.conf. Apply + is per-library transactional and may be retried after partial success. + """ + environment = dict(os.environ if environment is None else environment) + conf, installdir = Path(conf), Path(installdir) + _require_root() + original = conf.read_text() if conf.exists() else '' + current = _read_config(conf) + current_root = current.get('cloudfile_v2', 'auto_local_root', fallback='') + if current_root and current_root != LIBRARY_ROOT: + raise ValueError('Existing auto_local_root differs from the required storage root') + # Validate the final output before any SQL write, while inventory continues + # to see original storage classes/backend paths in its temporary candidate. + final = _render_auto(original, generated_body) + check = ConfigParser(interpolation=None) + check.read_string(final) + if not current.has_section('cloudfile_v2'): + current.add_section('cloudfile_v2') + current.set('cloudfile_v2', 'auto_local_root', LIBRARY_ROOT) + hub = installdir / 'seahub' + native = conf.parent.parent / 'seafile-data' + cli_env = dict(environment) + # The persisted Seafile configuration, rather than a future Compose value, + # describes the current routing that must pass the transition preflight. + cli_env.pop('CF_STORAGE_CLASSES_JSON', None) + for suffix, source, default in ( + ('HOST', 'SEAFILE_MYSQL_DB_HOST', 'db'), + ('PORT', 'SEAFILE_MYSQL_DB_PORT', '3306'), + ('USER', 'SEAFILE_MYSQL_DB_USER', 'seafile'), + ('PASSWORD', 'SEAFILE_MYSQL_DB_PASSWORD', ''), + ('NAME', 'SEAFILE_MYSQL_DB_SEAFILE_DB_NAME', 'seafile_db')): + cli_env['CF_V2_DB_' + suffix] = environment.get(source, default) + cli_env['PYTHONPATH'] = str(hub) + os.pathsep + environment.get('PYTHONPATH', '') + candidate_path = None + try: + with tempfile.NamedTemporaryFile(mode='w', prefix='.storage-preflight-', suffix='.conf', + dir=conf.parent, delete=False) as stream: + candidate_path = stream.name + current.write(stream) + command = [sys.executable, '-m', 'cloudfile_ext.v2', 'storage-reconcile', + '--seafile-conf', candidate_path, '--seafile-data', str(native)] + for mode in ('--dry-run', '--apply'): + result = subprocess.run(command + [mode], cwd=hub, env=cli_env, + capture_output=True, text=True, timeout=120) + # Preserve the CLI's complete single JSON report in startup output. + print(result.stdout, end='' if result.stdout.endswith('\n') else '\n', flush=True) + try: + report = json.loads(result.stdout) + if (report.get('mode') != ('apply' if mode == '--apply' else 'dry-run') or + not isinstance(report.get('items'), list) or + not isinstance(report.get('orphan_directories'), list)): + raise ValueError() + summary = report['summary'] + if not isinstance(summary, dict) or any(type(value) is not int or value < 0 for value in summary.values()): + raise ValueError() + blocked = summary.get('BLOCKED', 0) or summary.get('NATIVE_CLASS_PINNED', 0) + except (ValueError, KeyError, TypeError): + raise RuntimeError('Invalid storage preflight report; seafile.conf is unchanged') from None + if result.returncode or blocked: + raise RuntimeError('Storage transition refused; inspect the JSON report and explicitly normalize pinned native classes before retrying') + _atomic_write(conf, final) + finally: + if candidate_path is not None: + Path(candidate_path).unlink(missing_ok=True) + + +def _atomic_write(path, body): + temporary = None + try: + with tempfile.NamedTemporaryFile(mode='w', prefix='.storage-config-', dir=path.parent, delete=False) as stream: + temporary = stream.name + if path.exists(): + os.fchmod(stream.fileno(), stat.S_IMODE(path.stat().st_mode)) + stream.write(body) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + finally: + if temporary is not None: + Path(temporary).unlink(missing_ok=True) diff --git a/tests/test_library_storage_config.py b/tests/test_library_storage_config.py new file mode 100644 index 00000000..e5ab0f16 --- /dev/null +++ b/tests/test_library_storage_config.py @@ -0,0 +1,211 @@ +"""V2 startup configuration; real rendering with an explicit Hub CLI double.""" +import ast +import contextlib +import os +import sys +from configparser import ConfigParser +import importlib.util +import io +import json +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import Mock, patch + +SOURCE = Path(__file__).resolve().parents[1] / 'scripts/scripts_14.0/library_storage.py' +SPEC = importlib.util.spec_from_file_location('library_storage_config_test', SOURCE) +storage = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(storage) + + +class LibraryStorageConfigTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory(dir='/private/tmp' if Path('/private/tmp').exists() else None) + self.addCleanup(self.directory.cleanup) + self.top = Path(self.directory.name) + (self.top / 'conf').mkdir() + self.conf = self.top / 'conf/seafile.conf' + self.root = self.top / 'libraries' + self.root.mkdir() + self.installdir = self.top / 'server' + (self.installdir / 'seahub').mkdir(parents=True) + self.conf.write_text('[database]\nhost = persisted-db\n') + self.original = self.conf.read_text() + self.calls = [] + self.env = dict(CF_LIBRARY_STORAGE='auto', SEAFILE_MYSQL_DB_HOST='existing-db', + SEAFILE_MYSQL_DB_PASSWORD='secret', SEAFILE_MYSQL_DB_SEAFILE_DB_NAME='original_db', + CF_STORAGE_CLASSES_JSON='new-env-must-not-override-persisted') + self.root_patch = patch.object(storage, 'LIBRARY_ROOT', str(self.root)) + self.root_patch.start() + self.addCleanup(self.root_patch.stop) + + def cli(self, command, **kwargs): + candidate = Path(command[command.index('--seafile-conf') + 1]) + config = ConfigParser(interpolation=None) + config.read(candidate) + self.assertEqual(str(self.root), config['cloudfile_v2']['auto_local_root']) + self.assertEqual('persisted-db', config['database']['host']) + self.assertEqual('existing-db', kwargs['env']['CF_V2_DB_HOST']) + self.assertEqual('original_db', kwargs['env']['CF_V2_DB_NAME']) + self.assertNotIn('secret', command) + self.assertNotIn('CF_STORAGE_CLASSES_JSON', kwargs['env']) + self.assertEqual(str(self.top / 'seafile-data'), command[command.index('--seafile-data') + 1]) + self.calls.append((command, config)) + return self.report(command) + + def report(self, command, *, blocked=0, pinned=0, code=0): + return subprocess.CompletedProcess(command, code, json.dumps(dict( + mode='apply' if command[-1] == '--apply' else 'dry-run', + summary={'BLOCKED': blocked, 'NATIVE_CLASS_PINNED': pinned}, items=[], orphan_directories=[])) + '\n', '') + + def configure(self, cli=None): + with patch.object(storage.subprocess, 'run', side_effect=cli or self.cli), contextlib.redirect_stdout(io.StringIO()) as output: + storage.configure_auto_storage(self.conf, '[cloudfile]\nexample = true\n', self.installdir, self.env) + return output.getvalue() + + def bootstrap_writer(self, environment): + source = SOURCE.with_name('bootstrap.py') + names = {'write_cloudfile_seafile_conf', '_replace_block'} + nodes = [node for node in ast.parse(source.read_text()).body + if isinstance(node, ast.FunctionDef) and node.name in names] + namespace = dict(os=os, json=json, join=os.path.join, exists=os.path.exists, + topdir=str(self.top), installdir=str(self.installdir), + CF_BEGIN=storage.BEGIN, CF_END=storage.END, + get_conf=lambda key, default='': environment.get(key, default), + cf_enabled=lambda key: environment.get(key, 'false') == 'true', + logwarning=lambda message: None, + _seafile_conf_cloudfile_lines=lambda: ['[cloudfile]\nexample = true\n']) + exec(compile(ast.Module(body=nodes, type_ignores=[]), str(source), 'exec'), namespace) + return namespace['write_cloudfile_seafile_conf'] + + def test_interrupted_fresh_initialization_retains_auto_intent(self): + source = SOURCE.with_name('bootstrap.py') + node = next(node for node in ast.parse(source.read_text()).body + if isinstance(node, ast.FunctionDef) and node.name == 'init_seafile_server') + (self.top / 'seafile-data').mkdir() + marker = self.top / '.cloudfile-schema-init-pending' + marker.touch() + namespace = dict(exists=os.path.exists, join=os.path.join, shared_seafiledir=str(self.top), + get_version_stamp_file=Mock(side_effect=StopIteration)) + exec(compile(ast.Module(body=[node], type_ignores=[]), str(source), 'exec'), namespace) + with self.assertRaises(StopIteration): + namespace['init_seafile_server']() + self.assertTrue(namespace['_cloudfile_fresh_install']) + marker.unlink() + with self.assertRaises(StopIteration): + namespace['init_seafile_server']() + self.assertFalse(namespace['_cloudfile_fresh_install']) + + def test_bootstrap_legacy_multi_survives_environment_overlay_removal(self): + self.conf.write_text(self.original + storage.BEGIN + '\n[cloudfile]\nexample = true\n' + + '[storage]\nenable_storage_classes = true\nstorage_classes_file = existing.json\n' + + '[block_backend]\nname = multiple\n' + storage.END + '\n') + with patch.dict(sys.modules, {'library_storage': storage}), patch.dict(os.environ, {}, clear=True): + self.bootstrap_writer({})() + config = ConfigParser(interpolation=None) + config.read(self.conf) + self.assertEqual('existing.json', config['storage']['storage_classes_file']) + self.assertEqual('multiple', config['block_backend']['name']) + self.assertFalse(config.has_section('cloudfile_v2')) + + def test_bootstrap_explicit_auto_ignores_legacy_s3_variable(self): + environment = dict(self.env, CF_ENABLE_S3_STORAGE='true', SEAF_SERVER_STORAGE_TYPE='multiple') + with patch.dict(sys.modules, {'library_storage': storage}), patch.dict(os.environ, environment, clear=True), \ + patch.object(storage.subprocess, 'run', side_effect=self.cli), contextlib.redirect_stdout(io.StringIO()): + self.bootstrap_writer(environment)() + self.assertIn('enable_storage_classes = false', self.conf.read_text()) + self.assertNotIn('name = multiple', self.conf.read_text()) + + def test_new_install_defaults_auto_but_existing_unset_is_legacy(self): + self.assertTrue(storage.auto_storage_enabled(self.conf, {}, fresh=True)) + self.assertFalse(storage.auto_storage_enabled(self.conf, {})) + self.assertFalse(storage.auto_storage_enabled(self.conf, {'CF_ENABLE_S3_STORAGE': 'true', 'SEAF_SERVER_STORAGE_TYPE': 'multiple'}, fresh=True)) + self.assertTrue(storage.auto_storage_enabled(self.conf, self.env)) + with self.assertRaises(ValueError): + storage.auto_storage_enabled(self.conf, {'CF_LIBRARY_STORAGE': 'legacy'}) + + def test_new_auto_uses_filesystem_and_reconciles_before_write(self): + output = self.configure() + config = ConfigParser(interpolation=None) + config.read(self.conf) + self.assertEqual(['--dry-run', '--apply'], [call[0][-1] for call in self.calls]) + self.assertEqual('false', config['storage']['enable_storage_classes']) + self.assertEqual(str(self.root), config['cloudfile_v2']['auto_local_root']) + for section in ('commit_object_backend', 'fs_object_backend', 'block_backend'): + self.assertEqual('fs', config[section]['name']) + self.assertEqual('persisted-db', config['database']['host']) + self.assertEqual(2, len(output.splitlines())) + self.assertTrue(storage.auto_storage_enabled(self.conf, {})) + self.assertEqual([], list(self.conf.parent.glob('.storage-*'))) + + def test_explicit_transition_inventories_original_classes(self): + self.conf.write_text(self.original + '[storage]\nenable_storage_classes = true\nstorage_classes_file = existing.json\n[block_backend]\nname = multiple\n') + self.configure() + for command, config in self.calls: + self.assertEqual('existing.json', config['storage']['storage_classes_file']) + self.assertEqual('multiple', config['block_backend']['name']) + self.assertNotIn('existing.json', self.conf.read_text()) + + def test_preflight_blocked_or_pinned_does_not_write_or_apply(self): + for blocked, pinned in ((1, 0), (0, 1)): + with self.subTest(blocked=blocked, pinned=pinned): + calls = [] + def cli(command, **kwargs): + calls.append(command) + return self.report(command, blocked=blocked, pinned=pinned) + with self.assertRaises(RuntimeError): + self.configure(cli) + self.assertEqual(self.original, self.conf.read_text()) + self.assertEqual(1, len(calls)) + + def test_apply_failure_leaves_original_configuration(self): + def cli(command, **kwargs): + return self.report(command, blocked=int(command[-1] == '--apply'), code=2 if command[-1] == '--apply' else 0) + with self.assertRaises(RuntimeError): + self.configure(cli) + self.assertEqual(self.original, self.conf.read_text()) + self.assertEqual([], list(self.conf.parent.glob('.storage-*'))) + + def test_cli_exit_failure_invalid_json_or_missing_summary_refuses(self): + for output, code in (('not JSON', 0), ('{}', 0), ('{"summary": {"BLOCKED": "0"}}', 0), ('{"summary": {}}', 1)): + with self.subTest(output=output), self.assertRaises(RuntimeError): + self.configure(lambda command, **kwargs: subprocess.CompletedProcess(command, code, output, 'private driver error')) + self.assertEqual(self.original, self.conf.read_text()) + + def test_root_missing_file_symlink_or_symlink_ancestor_refuses(self): + self.root.rmdir() + for kind in ('missing', 'file', 'symlink', 'ancestor'): + with self.subTest(kind=kind): + if kind == 'file': + self.root.write_text('not directory') + elif kind == 'symlink': + self.root.unlink() + self.root.symlink_to(self.top, target_is_directory=True) + elif kind == 'ancestor': + self.root.unlink() + link = self.top / 'linked' + link.symlink_to(self.top, target_is_directory=True) + storage.LIBRARY_ROOT = str(link / 'conf') + with patch.object(storage.subprocess, 'run') as run, self.assertRaises(ValueError): + storage.configure_auto_storage(self.conf, '', self.installdir, self.env) + run.assert_not_called() + self.assertEqual(self.original, self.conf.read_text()) + self.assertFalse(self.root.exists()) + + def test_restart_is_idempotent_without_changing_other_configuration(self): + self.configure() + rendered = self.conf.read_text() + self.configure() + self.assertEqual(rendered, self.conf.read_text()) + + def test_malformed_markers_refuse_before_inventory_and_write(self): + self.conf.write_text(self.original + storage.BEGIN + '\n') + with patch.object(storage.subprocess, 'run') as run, self.assertRaises(ValueError): + storage.configure_auto_storage(self.conf, '', self.installdir, self.env) + run.assert_not_called() + self.assertEqual(self.original + storage.BEGIN + '\n', self.conf.read_text()) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_settings_block_order.py b/tests/test_settings_block_order.py new file mode 100644 index 00000000..bd97485a --- /dev/null +++ b/tests/test_settings_block_order.py @@ -0,0 +1,59 @@ +"""Generated configuration must remain executable across repeated restarts.""" +import ast +import os +import tempfile +import unittest +from pathlib import Path + + +SOURCE = Path(__file__).resolve().parents[1] / 'scripts/scripts_14.0/bootstrap.py' +TREE = ast.parse(SOURCE.read_text()) +FUNCTION = next(n for n in TREE.body if isinstance(n, ast.FunctionDef) and n.name == '_replace_block') +NAMESPACE = {'exists': os.path.exists} +exec(compile(ast.Module(body=[FUNCTION], type_ignores=[]), str(SOURCE), 'exec'), NAMESPACE) +REPLACE = NAMESPACE['_replace_block'] +BEGIN = '# --- CloudFile (generated, do not edit) ---' +END = '# --- end CloudFile ---' + + +class SettingsBlockOrderTest(unittest.TestCase): + def test_restart_preserves_dependent_aliases_and_local_overrides(self): + # DEV kept aliases after generated SSO settings. Appending the block + # made those aliases raise NameError and prevented Web/worker startup. + head = "SITE_NAME = 'fixture'\n" + tail = "CF_DIRECTORY_URL = CF_SERVICE_SSO_DIRECTORY_URL\nCF_SERVICE_SSO_DIRECTORY_URL = 'local-override'\n" + with tempfile.TemporaryDirectory() as directory: + p = Path(directory) / 'seahub_settings.py' + p.write_text(head + BEGIN + "\nCF_SERVICE_SSO_DIRECTORY_URL = 'old'\n" + END + '\n' + tail) + for value in ['new', 'new', 'updated']: + REPLACE(p, BEGIN, END, 'CF_SERVICE_SSO_DIRECTORY_URL = %r\n' % value) + rendered = p.read_text() + namespace = {} + exec(compile(rendered, str(p), 'exec'), namespace) + self.assertEqual(namespace['CF_DIRECTORY_URL'], value) + self.assertEqual(namespace['CF_SERVICE_SSO_DIRECTORY_URL'], 'local-override') + self.assertTrue(rendered.startswith(head)) + self.assertTrue(rendered.endswith(tail)) + + def test_invalid_markers_never_truncate_operator_settings(self): + for markers in [BEGIN, END, BEGIN + '\n' + BEGIN + '\n' + END, END + '\n' + BEGIN]: + with self.subTest(markers=markers), tempfile.TemporaryDirectory() as directory: + p = Path(directory) / 'seahub_settings.py' + original = markers + "\nLOCAL_SETTING = 'keep'\n" + p.write_text(original) + with self.assertRaises(ValueError): + REPLACE(p, BEGIN, END, 'GENERATED = True\n') + self.assertEqual(p.read_text(), original) + + def test_first_append_stays_idempotent(self): + with tempfile.TemporaryDirectory() as directory: + p = Path(directory) / 'seafile.conf' + p.write_text('[fileserver]\nport = 8082\n') + REPLACE(p, BEGIN, END, '[storage]\nname = fs\n') + rendered = p.read_text() + REPLACE(p, BEGIN, END, '[storage]\nname = fs\n') + self.assertEqual(p.read_text(), rendered) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_v06_default_library_storage.py b/tests/test_v06_default_library_storage.py index 0e8e6eaf..0a09afb8 100644 --- a/tests/test_v06_default_library_storage.py +++ b/tests/test_v06_default_library_storage.py @@ -1,4 +1,4 @@ -"""v0.6: fresh-install default uses one mount and auto-local per library. +"""Current V2 fresh-install default uses one mount and native per-library routing. Read-only tests. Never starts containers, creates libraries or modifies data. Existing .env files are deliberately not consulted or changed. @@ -18,6 +18,7 @@ def test_fresh_install_template_selects_overlay(self): env = (ROOT / ".env.example").read_text() self.assertIn("COMPOSE_FILE=docker-compose.yml:docker-compose.auto-library-storage.yml", env) self.assertIn("CF_LIBRARY_STORAGE_HOST_DIR=./data/libraries", env) + self.assertIn("CF_LIBRARY_STORAGE=auto", env) self.assertTrue((ROOT / "docker-compose.auto-library-storage.yml").is_file()) # This is the only template switch. Existing persisted .env values are # not rewritten just because a new version of .env.example ships. @@ -32,22 +33,16 @@ def test_merged_new_install_compose_contract(self): server = data["services"]["cloudfile"] worker = data["services"]["cf-worker"] for process in (server, worker): - self.assertEqual("multiple", process["environment"]["SEAF_SERVER_STORAGE_TYPE"]) - self.assertEqual("true", str(process["environment"]["CF_ENABLE_S3_STORAGE"]).lower()) - classes = json.loads(server["environment"]["CF_STORAGE_CLASSES_JSON"]) - self.assertEqual(["legacy-default", "auto-local"], [c["storage_id"] for c in classes]) - self.assertTrue(classes[0]["is_default"]) - self.assertFalse(classes[1]["is_default"]) - self.assertTrue(classes[1]["is_auto"]) + self.assertEqual("auto", process["environment"]["CF_LIBRARY_STORAGE"]) + self.assertEqual("false", str(process["environment"]["CF_ENABLE_S3_STORAGE"]).lower()) + self.assertEqual("", server["environment"]["SEAF_SERVER_STORAGE_TYPE"]) + self.assertEqual("", server["environment"]["CF_STORAGE_CLASSES_JSON"]) parent = "/shared/seafile/library-storage" - legacy = "/shared/seafile/seafile-data" - for kind in ("fs", "commits", "blocks"): - self.assertEqual({"backend": "fs", "dir": legacy}, classes[0][kind]) - self.assertEqual({"backend": "fs", "dir": parent}, classes[1][kind]) for process in (server, worker): mounts = [v for v in process["volumes"] if v["target"] == parent] self.assertEqual(1, len(mounts), "one parent mount per process") self.assertEqual("rw", mounts[0].get("mode", "rw")) + self.assertFalse(mounts[0]["bind"]["create_host_path"]) self.assertEqual(False, str(server["environment"]["CF_ENABLE_CHECKOUT"]).lower() == "true") diff --git a/tools/test-bootstrap-settings.py b/tools/test-bootstrap-settings.py index d89afc82..a9d902b9 100755 --- a/tools/test-bootstrap-settings.py +++ b/tools/test-bootstrap-settings.py @@ -28,6 +28,7 @@ HERE = os.path.dirname(os.path.abspath(__file__)) BOOTSTRAP = os.path.join(HERE, '..', 'scripts', 'scripts_14.0', 'bootstrap.py') +sys.path.insert(0, os.path.dirname(BOOTSTRAP)) failures = [] From f14a99801bb51f97ba775d907b13c59db8d5a993 Mon Sep 17 00:00:00 2001 From: dev9-bb Date: Tue, 20 Oct 2026 17:42:19 +0800 Subject: [PATCH 2/4] fix(storage): match native backend names and retain overlay routing --- docs/features/automatic-local-storage.md | 12 +++++- scripts/scripts_14.0/bootstrap.py | 3 +- scripts/scripts_14.0/library_storage.py | 6 ++- tests/check_storage_config_native.py | 50 ++++++++++++++++++++++++ tests/test_library_storage_config.py | 39 ++++++++++++++++-- 5 files changed, 103 insertions(+), 7 deletions(-) create mode 100644 tests/check_storage_config_native.py diff --git a/docs/features/automatic-local-storage.md b/docs/features/automatic-local-storage.md index 47aacfcd..56aa650c 100644 --- a/docs/features/automatic-local-storage.md +++ b/docs/features/automatic-local-storage.md @@ -16,7 +16,7 @@ Seafile 原生 `RepoStorageId` 是库路由事实来源;不读取、转换或 `/shared/seafile/library-storage`。启动时还会拒绝根或任一级祖先为符号链接、 根不存在或不是目录;不会自动创建存储根。 4. bootstrap 写 `[cloudfile_v2] auto_local_root=/shared/seafile/library-storage`, - `[storage] enable_storage_classes=false`,三个对象后端均使用 `name=fs`。 + `[storage] enable_storage_classes=false`,三个对象后端均使用 `name=filesystem`。 新模式不生成 multi 类,也不依赖 `CF_ENABLE_S3_STORAGE`。现有 V2 Server 在首个 commit 前预留库目录,并使用已有 active/retired 生命周期;此配置不重写路由实现。 @@ -70,5 +70,15 @@ docker compose --env-file .env.example config --quiet 专项检查覆盖新装、旧环境保留、显式切换、根校验、原配置库存预检、BLOCKED/PINNED 拒绝、 apply 失败配置不变及重启幂等。生成配置块保留原位置,避免本地别名或覆盖变量在重启时失效。 +原生 CE 的 `seafile-data` 链接会只读解析为真实目录后传给 CLI;托管库 root 仍禁止符号链接。 真实旧库、跨挂载上传下载、历史版本、恢复、GC 和 fsck 仍需匹配 Server/Hub 镜像及数据副本验证; 配置测试不代表 DEV/生产验收完成。 + +原生配置合同还可用当前 Server 的真实 C guard 直接核验: + +```bash +python3 tests/check_storage_config_native.py /path/to/cloudfile-server +``` + +该检查编译并执行生产 guard,接受生成的 `filesystem`,确认错误的 `fs` 被拒绝; +存储类 JSON 中已有的 `backend: fs` 格式不变。 diff --git a/scripts/scripts_14.0/bootstrap.py b/scripts/scripts_14.0/bootstrap.py index eccade55..8ab03d79 100755 --- a/scripts/scripts_14.0/bootstrap.py +++ b/scripts/scripts_14.0/bootstrap.py @@ -919,7 +919,8 @@ def write_cloudfile_seafile_conf(): if name != 'bucket': lines.append('%s = %s\n' % (name, value)) - if not cf_enabled('CF_ENABLE_S3_STORAGE'): + if (not cf_enabled('CF_ENABLE_S3_STORAGE') or + not get_conf('SEAF_SERVER_STORAGE_TYPE', '').strip()): lines.append(retained_legacy_storage(conf_path)) _replace_block(join(topdir, 'conf', 'seafile.conf'), CF_BEGIN, CF_END, ''.join(lines)) diff --git a/scripts/scripts_14.0/library_storage.py b/scripts/scripts_14.0/library_storage.py index c6a0a6d8..ed049993 100644 --- a/scripts/scripts_14.0/library_storage.py +++ b/scripts/scripts_14.0/library_storage.py @@ -91,7 +91,7 @@ def _render_auto(original, generated_body): body = generated_body + '\n[cloudfile_v2]\nauto_local_root = %s\n' % LIBRARY_ROOT body += '\n[storage]\nenable_storage_classes = false\n' for section in ('commit_object_backend', 'fs_object_backend', 'block_backend'): - body += '\n[%s]\nname = fs\n' % section + body += '\n[%s]\nname = filesystem\n' % section return ''.join(out).rstrip() + '\n\n' + BEGIN + '\n' + body + END + '\n' @@ -118,7 +118,9 @@ def configure_auto_storage(conf, generated_body, installdir, environment=None): current.add_section('cloudfile_v2') current.set('cloudfile_v2', 'auto_local_root', LIBRARY_ROOT) hub = installdir / 'seahub' - native = conf.parent.parent / 'seafile-data' + # CE commonly links /opt/seafile/seafile-data to the persisted native tree. + # Resolve that upstream layout without relaxing managed-root validation. + native = (conf.parent.parent / 'seafile-data').resolve(strict=True) cli_env = dict(environment) # The persisted Seafile configuration, rather than a future Compose value, # describes the current routing that must pass the transition preflight. diff --git a/tests/check_storage_config_native.py b/tests/check_storage_config_native.py new file mode 100644 index 00000000..b290a4a2 --- /dev/null +++ b/tests/check_storage_config_native.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Compile the current Server guard and check actual Docker-generated config. + +Usage: python3 tests/check_storage_config_native.py /path/to/cloudfile-server +Requires the Server local-storage include seams, a C compiler, glib and jansson dev. +""" +import importlib.util +from pathlib import Path +import shlex +import subprocess +import sys +import tempfile + + +def main(): + server = Path(sys.argv[1]).resolve() + module_path = Path(__file__).resolve().parents[1] / 'scripts/scripts_14.0/library_storage.py' + spec = importlib.util.spec_from_file_location('library_storage', module_path) + storage = importlib.util.module_from_spec(spec) + spec.loader.exec_module(storage) + flags = shlex.split(subprocess.check_output( + ['pkg-config', '--cflags', '--libs', 'glib-2.0', 'gobject-2.0', 'jansson'], text=True)) + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source = root / 'guard.c' + source.write_text('#include "common.h"\n#include "cf-v2-storage.c"\n' + 'SeafileSession *seaf;\n' + 'int main(int argc, char **argv) {\n' + ' GKeyFile *config = g_key_file_new();\n' + ' if (argc != 2 || !g_key_file_load_from_file(config, argv[1], G_KEY_FILE_NONE, NULL)) return 2;\n' + ' int result = cf_v2_storage_validate_config(config);\n' + ' g_key_file_free(config); return result == 0 ? 0 : 1;\n}\n') + command = ['cc', '-D_GNU_SOURCE', '-ffunction-sections', '-fdata-sections'] + for include in ('tests/local_storage/include', 'common', 'lib'): + command += ['-I' + str(server / include)] + command += [str(source), '-Wl,-dead_strip' if sys.platform == 'darwin' else '-Wl,--gc-sections'] + subprocess.run(command + flags + ['-o', str(root / 'guard')], check=True) + config = root / 'seafile.conf' + rendered = storage._render_auto('[database]\nhost = db\n', '[cloudfile]\n') + config.write_text(rendered) + subprocess.run([str(root / 'guard'), str(config)], check=True) + config.write_text(rendered.replace('name = filesystem', 'name = fs')) + rejected = subprocess.run([str(root / 'guard'), str(config)], check=False) + if rejected.returncode != 1: + raise AssertionError('Native guard did not reject invalid fs backend spelling') + print('Current Server guard accepts generated filesystem backends and rejects fs') + + +if __name__ == '__main__': + main() diff --git a/tests/test_library_storage_config.py b/tests/test_library_storage_config.py index e5ab0f16..0982bd97 100644 --- a/tests/test_library_storage_config.py +++ b/tests/test_library_storage_config.py @@ -25,6 +25,7 @@ def setUp(self): self.addCleanup(self.directory.cleanup) self.top = Path(self.directory.name) (self.top / 'conf').mkdir() + (self.top / 'seafile-data').mkdir() self.conf = self.top / 'conf/seafile.conf' self.root = self.top / 'libraries' self.root.mkdir() @@ -50,7 +51,7 @@ def cli(self, command, **kwargs): self.assertEqual('original_db', kwargs['env']['CF_V2_DB_NAME']) self.assertNotIn('secret', command) self.assertNotIn('CF_STORAGE_CLASSES_JSON', kwargs['env']) - self.assertEqual(str(self.top / 'seafile-data'), command[command.index('--seafile-data') + 1]) + self.assertEqual(str((self.top / 'seafile-data').resolve()), command[command.index('--seafile-data') + 1]) self.calls.append((command, config)) return self.report(command) @@ -83,7 +84,6 @@ def test_interrupted_fresh_initialization_retains_auto_intent(self): source = SOURCE.with_name('bootstrap.py') node = next(node for node in ast.parse(source.read_text()).body if isinstance(node, ast.FunctionDef) and node.name == 'init_seafile_server') - (self.top / 'seafile-data').mkdir() marker = self.top / '.cloudfile-schema-init-pending' marker.touch() namespace = dict(exists=os.path.exists, join=os.path.join, shared_seafiledir=str(self.top), @@ -109,6 +109,24 @@ def test_bootstrap_legacy_multi_survives_environment_overlay_removal(self): self.assertEqual('multiple', config['block_backend']['name']) self.assertFalse(config.has_section('cloudfile_v2')) + def test_bootstrap_s3_switch_without_removed_overlay_type_keeps_multi(self): + original = self.original + storage.BEGIN + '\n[cloudfile]\nexample = true\n' + ( + '[storage]\nenable_storage_classes = true\nstorage_classes_file = existing.json\n' + + ''.join('[%s]\nname = multiple\n' % section for section in + ('commit_object_backend', 'fs_object_backend', 'block_backend'))) + storage.END + '\n' + self.conf.write_text(original) + environment = {'CF_ENABLE_S3_STORAGE': 'true', 'SEAF_SERVER_STORAGE_TYPE': ''} + with patch.dict(sys.modules, {'library_storage': storage}), patch.dict(os.environ, environment, clear=True): + self.bootstrap_writer(environment)() + config = ConfigParser(interpolation=None) + config.read(self.conf) + self.assertEqual('existing.json', config['storage']['storage_classes_file']) + self.assertEqual('true', config['storage']['enable_storage_classes']) + for section in ('commit_object_backend', 'fs_object_backend', 'block_backend'): + self.assertEqual('multiple', config[section]['name']) + self.assertFalse(config.has_section('cloudfile_v2')) + self.assertNotIn('auto_local_root', self.conf.read_text()) + def test_bootstrap_explicit_auto_ignores_legacy_s3_variable(self): environment = dict(self.env, CF_ENABLE_S3_STORAGE='true', SEAF_SERVER_STORAGE_TYPE='multiple') with patch.dict(sys.modules, {'library_storage': storage}), patch.dict(os.environ, environment, clear=True), \ @@ -133,12 +151,27 @@ def test_new_auto_uses_filesystem_and_reconciles_before_write(self): self.assertEqual('false', config['storage']['enable_storage_classes']) self.assertEqual(str(self.root), config['cloudfile_v2']['auto_local_root']) for section in ('commit_object_backend', 'fs_object_backend', 'block_backend'): - self.assertEqual('fs', config[section]['name']) + self.assertEqual('filesystem', config[section]['name']) self.assertEqual('persisted-db', config['database']['host']) self.assertEqual(2, len(output.splitlines())) self.assertTrue(storage.auto_storage_enabled(self.conf, {})) self.assertEqual([], list(self.conf.parent.glob('.storage-*'))) + def test_native_layout_symlinks_are_resolved_but_managed_root_is_not(self): + shared = self.top / 'shared-seafile' + shared.mkdir() + (shared / 'seafile-data').mkdir() + (self.top / 'seafile-data').rmdir() + (self.top / 'seafile-data').symlink_to(shared / 'seafile-data', target_is_directory=True) + (self.top / 'conf').rename(shared / 'conf') + (self.top / 'conf').symlink_to(shared / 'conf', target_is_directory=True) + self.configure() + for command, _ in self.calls: + self.assertEqual(str(shared / 'seafile-data'), command[command.index('--seafile-data') + 1]) + config = ConfigParser(interpolation=None) + config.read(self.conf) + self.assertEqual(str(self.root), config['cloudfile_v2']['auto_local_root']) + def test_explicit_transition_inventories_original_classes(self): self.conf.write_text(self.original + '[storage]\nenable_storage_classes = true\nstorage_classes_file = existing.json\n[block_backend]\nname = multiple\n') self.configure() From 4785bce21dd6f1ccf5f7f140220a1080f55bcad4 Mon Sep 17 00:00:00 2001 From: dev9-bb Date: Tue, 20 Oct 2026 17:43:50 +0800 Subject: [PATCH 3/4] fix(storage): bind native environment and accept read-only preflight notices --- docs/features/automatic-local-storage.md | 7 +++++-- scripts/scripts_14.0/library_storage.py | 2 ++ tests/test_library_storage_config.py | 19 ++++++++++++++++++- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/docs/features/automatic-local-storage.md b/docs/features/automatic-local-storage.md index 56aa650c..d9215135 100644 --- a/docs/features/automatic-local-storage.md +++ b/docs/features/automatic-local-storage.md @@ -53,8 +53,11 @@ python -m cloudfile_ext.v2 storage-reconcile \ `ORPHAN_DIRECTORY` 只报告,不登记、不删除。 CLI 使用 `CF_V2_DB_HOST/PORT/USER/PASSWORD/NAME`;bootstrap 从现有 -`SEAFILE_MYSQL_DB_*` 变量传入,保持实例原数据库。首次缺表时 CLI 只幂等建立当前合同的 -`cf_library_storage`,不读取旧表。以上不是自动数据迁移;切换执行及 DEV/生产发布需按环境审批。 +`SEAFILE_MYSQL_DB_*` 变量传入,保持实例原数据库。 +CLI 的 `SEAFILE_DATA_DIR` 与 `SEAFILE_CENTRAL_CONF_DIR` 显式设为当前实例真实原生目录, +覆盖陈旧环境值,避免 Seafile Python 初始化读到其他实例。dry-run 严格只读;首次缺表仅报告 `SCHEMA_MISSING`,不是业务 BLOCKED。 +预检无异常后,apply 才幂等建立当前合同的 `cf_library_storage`,不读取旧表。 +`STATE_DRIFT` 仅报告、不阻断启动、不更新生命周期;保留 Server 的 reserved/active/retired 终判。以上不是自动数据迁移;切换执行及 DEV/生产发布需按环境审批。 ## 验证与边界 diff --git a/scripts/scripts_14.0/library_storage.py b/scripts/scripts_14.0/library_storage.py index ed049993..b4cc5d5f 100644 --- a/scripts/scripts_14.0/library_storage.py +++ b/scripts/scripts_14.0/library_storage.py @@ -122,6 +122,8 @@ def configure_auto_storage(conf, generated_body, installdir, environment=None): # Resolve that upstream layout without relaxing managed-root validation. native = (conf.parent.parent / 'seafile-data').resolve(strict=True) cli_env = dict(environment) + cli_env['SEAFILE_DATA_DIR'] = str(native) + cli_env['SEAFILE_CENTRAL_CONF_DIR'] = str(conf.parent.resolve(strict=True)) # The persisted Seafile configuration, rather than a future Compose value, # describes the current routing that must pass the transition preflight. cli_env.pop('CF_STORAGE_CLASSES_JSON', None) diff --git a/tests/test_library_storage_config.py b/tests/test_library_storage_config.py index 0982bd97..496ca18d 100644 --- a/tests/test_library_storage_config.py +++ b/tests/test_library_storage_config.py @@ -36,7 +36,8 @@ def setUp(self): self.calls = [] self.env = dict(CF_LIBRARY_STORAGE='auto', SEAFILE_MYSQL_DB_HOST='existing-db', SEAFILE_MYSQL_DB_PASSWORD='secret', SEAFILE_MYSQL_DB_SEAFILE_DB_NAME='original_db', - CF_STORAGE_CLASSES_JSON='new-env-must-not-override-persisted') + CF_STORAGE_CLASSES_JSON='new-env-must-not-override-persisted', + SEAFILE_DATA_DIR='/stale-data', SEAFILE_CENTRAL_CONF_DIR='/stale-conf') self.root_patch = patch.object(storage, 'LIBRARY_ROOT', str(self.root)) self.root_patch.start() self.addCleanup(self.root_patch.stop) @@ -51,6 +52,8 @@ def cli(self, command, **kwargs): self.assertEqual('original_db', kwargs['env']['CF_V2_DB_NAME']) self.assertNotIn('secret', command) self.assertNotIn('CF_STORAGE_CLASSES_JSON', kwargs['env']) + self.assertEqual(str((self.top / 'seafile-data').resolve()), kwargs['env']['SEAFILE_DATA_DIR']) + self.assertEqual(str(self.conf.parent.resolve()), kwargs['env']['SEAFILE_CENTRAL_CONF_DIR']) self.assertEqual(str((self.top / 'seafile-data').resolve()), command[command.index('--seafile-data') + 1]) self.calls.append((command, config)) return self.report(command) @@ -180,6 +183,20 @@ def test_explicit_transition_inventories_original_classes(self): self.assertEqual('multiple', config['block_backend']['name']) self.assertNotIn('existing.json', self.conf.read_text()) + def test_schema_missing_and_state_drift_allow_explicit_apply(self): + for status in ('SCHEMA_MISSING', 'STATE_DRIFT'): + with self.subTest(status=status): + modes = [] + def cli(command, **kwargs): + modes.append(command[-1]) + report = json.loads(self.cli(command, **kwargs).stdout) + report['summary'][status] = 1 + report['schema'] = {'status': status if status == 'SCHEMA_MISSING' else 'READY'} + return subprocess.CompletedProcess(command, 0, json.dumps(report), '') + self.configure(cli) + self.assertEqual(['--dry-run', '--apply'], modes) + self.assertIn('name = filesystem', self.conf.read_text()) + def test_preflight_blocked_or_pinned_does_not_write_or_apply(self): for blocked, pinned in ((1, 0), (0, 1)): with self.subTest(blocked=blocked, pinned=pinned): From 0f3c5717d5364ad9ff2eaaec11808fa953baea49 Mon Sep 17 00:00:00 2001 From: dev9-bb Date: Tue, 20 Oct 2026 17:47:53 +0800 Subject: [PATCH 4/4] fix(runtime): export native data and config paths to Hub services --- deploy/compose/docker-compose.yml | 5 ++++- docs/features/automatic-local-storage.md | 5 ++++- tests/test_v06_default_library_storage.py | 16 ++++++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/deploy/compose/docker-compose.yml b/deploy/compose/docker-compose.yml index 696123fb..7218cf14 100644 --- a/deploy/compose/docker-compose.yml +++ b/deploy/compose/docker-compose.yml @@ -278,7 +278,8 @@ services: # seafile-ai service below). Harmless with no file present: # seahub/config_parser.py logs a warning and returns {} rather than # raising, so this is safe to set even with CF_AI_ENABLED=false. - SEAFILE_CENTRAL_CONF_DIR: /opt/seafile/conf + SEAFILE_DATA_DIR: /shared/seafile/seafile-data + SEAFILE_CENTRAL_CONF_DIR: /shared/seafile/conf # -- SSO (inert unless CF_ENABLE_SSO is true) -- # No `:?` on any of these: compose interpolates the whole file whatever # profile is active, so a required marker here would break a plain @@ -429,6 +430,8 @@ services: REDIS_PORT: "6379" TIME_ZONE: ${TIME_ZONE:-Etc/UTC} SEAFILE_RPC_PIPE_PATH: /shared/seafile/runtime + SEAFILE_DATA_DIR: /shared/seafile/seafile-data + SEAFILE_CENTRAL_CONF_DIR: /shared/seafile/conf # Keep the worker's task registry in sync with the application. New # periodic extension tasks can therefore be enabled with the same .env. CF_ENABLE_SSO: ${CF_ENABLE_SSO:-false} diff --git a/docs/features/automatic-local-storage.md b/docs/features/automatic-local-storage.md index d9215135..8f34a5ac 100644 --- a/docs/features/automatic-local-storage.md +++ b/docs/features/automatic-local-storage.md @@ -54,7 +54,10 @@ python -m cloudfile_ext.v2 storage-reconcile \ CLI 使用 `CF_V2_DB_HOST/PORT/USER/PASSWORD/NAME`;bootstrap 从现有 `SEAFILE_MYSQL_DB_*` 变量传入,保持实例原数据库。 -CLI 的 `SEAFILE_DATA_DIR` 与 `SEAFILE_CENTRAL_CONF_DIR` 显式设为当前实例真实原生目录, +Compose 的 cloudfile/cf-worker 运行环境均明确设置 +`SEAFILE_DATA_DIR=/shared/seafile/seafile-data` 与 +`SEAFILE_CENTRAL_CONF_DIR=/shared/seafile/conf`,Hub 运行时和预检读取同一原生实例。 +CLI 的这两个变量也显式设为当前实例真实原生目录, 覆盖陈旧环境值,避免 Seafile Python 初始化读到其他实例。dry-run 严格只读;首次缺表仅报告 `SCHEMA_MISSING`,不是业务 BLOCKED。 预检无异常后,apply 才幂等建立当前合同的 `cf_library_storage`,不读取旧表。 `STATE_DRIFT` 仅报告、不阻断启动、不更新生命周期;保留 Server 的 reserved/active/retired 终判。以上不是自动数据迁移;切换执行及 DEV/生产发布需按环境审批。 diff --git a/tests/test_v06_default_library_storage.py b/tests/test_v06_default_library_storage.py index 0a09afb8..87f60ff8 100644 --- a/tests/test_v06_default_library_storage.py +++ b/tests/test_v06_default_library_storage.py @@ -34,6 +34,8 @@ def test_merged_new_install_compose_contract(self): worker = data["services"]["cf-worker"] for process in (server, worker): self.assertEqual("auto", process["environment"]["CF_LIBRARY_STORAGE"]) + self.assertEqual("/shared/seafile/seafile-data", process["environment"]["SEAFILE_DATA_DIR"]) + self.assertEqual("/shared/seafile/conf", process["environment"]["SEAFILE_CENTRAL_CONF_DIR"]) self.assertEqual("false", str(process["environment"]["CF_ENABLE_S3_STORAGE"]).lower()) self.assertEqual("", server["environment"]["SEAF_SERVER_STORAGE_TYPE"]) self.assertEqual("", server["environment"]["CF_STORAGE_CLASSES_JSON"]) @@ -46,6 +48,20 @@ def test_merged_new_install_compose_contract(self): self.assertEqual(False, str(server["environment"]["CF_ENABLE_CHECKOUT"]).lower() == "true") + @unittest.skipUnless(shutil.which("docker"), "Docker Compose CLI not installed") + def test_base_compose_exports_native_runtime_paths_without_storage_overlay(self): + proc = subprocess.run( + ["docker", "compose", "--env-file", ".env.example", "-f", "docker-compose.yml", "config", "--format", "json"], + cwd=ROOT, capture_output=True, text=True, timeout=30, check=True, + env={**os.environ, "CF_LIBRARY_STORAGE": "", "SEAFILE_DATA_DIR": "/stale-data", + "SEAFILE_CENTRAL_CONF_DIR": "/stale-conf"}) + services = json.loads(proc.stdout)["services"] + for name in ("cloudfile", "cf-worker"): + environment = services[name]["environment"] + self.assertEqual("/shared/seafile/seafile-data", environment["SEAFILE_DATA_DIR"]) + self.assertEqual("/shared/seafile/conf", environment["SEAFILE_CENTRAL_CONF_DIR"]) + self.assertEqual("", environment["CF_LIBRARY_STORAGE"]) + @unittest.skipUnless(shutil.which("docker"), "Docker Compose CLI not installed") def test_policy_flags_reach_both_processes(self): # .env values used to be silently dropped by Compose before bootstrap.