diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index c6a3f02a..ebcac019 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -58,5 +58,5 @@ jobs: - name: Test source selection rules without building images run: | - python3 -m unittest discover -s tests -p test_v2_source_assembly.py -q - python3 -m unittest discover -s tests -p test_v2_build_policy.py -q + python3 -m unittest discover -s tests -p test_build_receipt.py -q + python3 -m unittest discover -s tests -p test_current_build_policy.py -q diff --git a/.github/workflows/prod.yml b/.github/workflows/prod.yml index e4367b04..574608a0 100644 --- a/.github/workflows/prod.yml +++ b/.github/workflows/prod.yml @@ -1,366 +1,121 @@ -name: CloudFile production build +name: CloudFile receipt build -# Expensive production builds are release operations. Development pushes run -# checks only; an explicit dispatch starts this workflow. Keeping releases on -# one branch also lets GitHub reuse content-addressed caches across versions. +# Explicit integration build only. Routine pushes remain static and cheap. on: workflow_dispatch: inputs: version: - description: Build version - default: '14.0.0-cf.0' + description: Isolated test image version + default: 14.0.8-storage-test required: true - mode: - description: Build mode - type: choice - options: - - incremental - - full - default: incremental + server_ref: + description: Server branch (dev by default, a PR branch for pre-merge tests) + default: dev + required: true + hub_ref: + description: Hub branch + default: dev required: true - -concurrency: - group: ${{ github.workflow }}-${{ inputs.version || github.ref }} - cancel-in-progress: true - -env: - CF_VERSION: ${{ inputs.version || '14.0.0-cf.0' }} permissions: contents: read jobs: - # CE-compatible reference path: keep the original full build intact. Use it - # for cache-independent verification, upstream upgrades and release audits. - full: - if: inputs.mode == 'full' - needs: plan - runs-on: ubuntu-24.04 - timeout-minutes: 120 - env: - CF_FORCE_REBUILD: '1' - CF_SERVER_REF: ${{ needs.plan.outputs.server_sha }} - CF_HUB_REF: ${{ needs.plan.outputs.hub_sha }} - steps: - - uses: actions/checkout@v4 - with: - path: cloudfile-docker - fetch-depth: 1 - - - name: Free disk space - run: | - df -h / | tail -1 - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \ - /usr/local/share/boost "$AGENT_TOOLSDIRECTORY" || true - df -h / | tail -1 - - - uses: docker/setup-buildx-action@v3 - - - name: Restore full build cache - uses: actions/cache@v4 - with: - path: cloudfile-docker/build/cloudfile_14.0/.cache - key: cf-full-tools-v2-${{ needs.plan.outputs.backend_key }}-${{ needs.plan.outputs.frontend_key }}-${{ needs.plan.outputs.package_key }} - restore-keys: cf-full-tools-v2- - - - name: Build CE 14 base image - working-directory: cloudfile-docker/image/cloudfile_14.0 - run: ./base-build.sh - - - name: Build full release package - working-directory: cloudfile-docker/build/cloudfile_14.0 - run: ./build-in-docker.sh "$CF_VERSION" - - - name: Verify release artifact - run: | - cloudfile-docker/tools/verify-release-artifact.sh \ - "cloudfile-docker/build/cloudfile_14.0/seafile-server-$CF_VERSION" - - - name: Build production image - working-directory: cloudfile-docker/image/cloudfile_14.0 - run: ./docker-build.sh "$CF_VERSION" - - - name: Record build sources - run: | - cat "cloudfile-docker/build/cloudfile_14.0/seafile-server-$CF_VERSION/cloudfile-build-info.txt" \ - | tee -a "$GITHUB_STEP_SUMMARY" - - - name: Export image - run: | - docker save "cloudfile/cloudfile:$CF_VERSION" | gzip -1 > "cloudfile-$CF_VERSION.tar.gz" - ls -lh "cloudfile-$CF_VERSION.tar.gz" - - - uses: actions/upload-artifact@v4 - with: - name: cloudfile-production-full-image-${{ env.CF_VERSION }} - path: cloudfile-*.tar.gz - retention-days: 14 - plan: runs-on: ubuntu-24.04 outputs: server_sha: ${{ steps.plan.outputs.server_sha }} - hub_sha: ${{ steps.plan.outputs.hub_sha }} - hub_tree: ${{ steps.plan.outputs.hub_tree }} docker_sha: ${{ steps.plan.outputs.docker_sha }} - backend_key: ${{ steps.plan.outputs.backend_key }} - frontend_key: ${{ steps.plan.outputs.frontend_key }} - package_key: ${{ steps.plan.outputs.package_key }} steps: - uses: actions/checkout@v4 - with: - path: cloudfile-docker - fetch-depth: 1 - - - name: Restrict the historical release recipe to explicit maintenance refs - shell: bash - run: | - if [[ "$GITHUB_REF_NAME" != maintenance/v0.9/* ]]; then - echo "::error::Historical packaging is allowed only from an explicitly approved maintenance/v0.9/* ref; never from CloudFile dev or a detached feature snapshot." - exit 2 - fi - - - name: Read release refs - id: refs - working-directory: cloudfile-docker - run: | - server_ref=$(python3 build/cloudfile_14.0/read-manifest.py release.yaml server_commit) - hub_ref=$(python3 build/cloudfile_14.0/read-manifest.py release.yaml hub_commit) - # Never fall back to floating dev; the Docker commit is recorded - # from the checked-out workflow SHA, not self-pinned in this file. - for pin in "$server_ref" "$hub_ref"; do - if [[ ! $pin =~ ^[0-9a-f]{40}$ ]]; then - echo "::error::Historical release requires immutable server_commit and hub_commit pins" - exit 2 - fi - done - echo "server_ref=$server_ref" >> "$GITHUB_OUTPUT" - echo "hub_ref=$hub_ref" >> "$GITHUB_OUTPUT" - - - uses: actions/checkout@v4 - with: - repository: eaopen/cloudfile-server - ref: ${{ steps.refs.outputs.server_ref }} - path: cloudfile-server - fetch-depth: 1 - - - uses: actions/checkout@v4 - with: - repository: eaopen/cloudfile-hub - ref: ${{ steps.refs.outputs.hub_ref }} - path: cloudfile-hub - fetch-depth: 1 - - - name: Resolve immutable sources and component keys + - name: Resolve all three branches exactly once id: plan - run: | - ./cloudfile-docker/tools/release-build-plan.sh \ - "$CF_VERSION" cloudfile-server cloudfile-hub | tee -a "$GITHUB_OUTPUT" - - backend: - if: inputs.mode == 'incremental' - needs: plan - runs-on: ubuntu-24.04 - timeout-minutes: 75 - env: - CF_SERVER_REF: ${{ needs.plan.outputs.server_sha }} - CF_HUB_REF: ${{ needs.plan.outputs.hub_sha }} - CF_BUILD_TARGET: backend - CF_BUILD_JOBS: '2' - CF_CACHE_DIR: ${{ github.workspace }}/cloudfile-docker/build/cloudfile_14.0/.cache/backend - steps: - - uses: actions/checkout@v4 - with: - path: cloudfile-docker - - - name: Restore compiled backend - id: backend-artifact - uses: actions/cache@v4 - with: - path: cloudfile-backend.tar.gz - key: cf-backend-${{ needs.plan.outputs.backend_key }} - - - name: Free disk space - if: steps.backend-artifact.outputs.cache-hit != 'true' - run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \ - /usr/local/share/boost "$AGENT_TOOLSDIRECTORY" || true - - - uses: docker/setup-buildx-action@v3 - if: steps.backend-artifact.outputs.cache-hit != 'true' - - - name: Restore backend tool caches - if: steps.backend-artifact.outputs.cache-hit != 'true' - uses: actions/cache@v4 - with: - path: cloudfile-docker/build/cloudfile_14.0/.cache/backend - key: cf-backend-tools-${{ needs.plan.outputs.backend_key }} - restore-keys: cf-backend-tools- - - - name: Build CE 14 base image - if: steps.backend-artifact.outputs.cache-hit != 'true' - working-directory: cloudfile-docker/image/cloudfile_14.0 - run: ./base-build.sh - - - name: Compile backend - if: steps.backend-artifact.outputs.cache-hit != 'true' - working-directory: cloudfile-docker/build/cloudfile_14.0 - run: ./build-in-docker.sh "$CF_VERSION" - - - name: Pack backend artifact - if: steps.backend-artifact.outputs.cache-hit != 'true' - working-directory: cloudfile-docker/build/cloudfile_14.0 - run: | - tar -czf "$GITHUB_WORKSPACE/cloudfile-backend.tar.gz" \ - cloudfile-backend - + env: + VERSION: ${{ inputs.version }} + SERVER_REF: ${{ inputs.server_ref }} + HUB_REF: ${{ inputs.hub_ref }} + run: | + python3 build/seafile_14.0/build_receipt.py resolve \ + --manifest release.yaml --version "$VERSION" \ + --server-ref "$SERVER_REF" --hub-ref "$HUB_REF" \ + --docker-ref "$GITHUB_REF_NAME" --output build/cloudfile_14.0/build-receipt.json + python3 - <<'PY' >> "$GITHUB_OUTPUT" + import json + lock = json.load(open('build/cloudfile_14.0/build-receipt.json'))['lock'] + print('server_sha=' + lock['sources']['seafile-server']['ref']) + print('docker_sha=' + lock['docker_commit']) + PY - uses: actions/upload-artifact@v4 with: - name: backend-${{ needs.plan.outputs.backend_key }} - path: cloudfile-backend.tar.gz - retention-days: 1 + name: source-receipt + path: build/cloudfile_14.0/build-receipt.json - frontend: - if: inputs.mode == 'incremental' + full: needs: plan runs-on: ubuntu-24.04 - timeout-minutes: 60 + timeout-minutes: 120 env: - CF_SERVER_REF: ${{ needs.plan.outputs.server_sha }} - CF_HUB_REF: ${{ needs.plan.outputs.hub_sha }} - CF_BUILD_TARGET: frontend + CF_BUILD_RECEIPT: ${{ github.workspace }}/build/cloudfile_14.0/build-receipt.json CF_BUILD_JOBS: '2' - CF_CACHE_DIR: ${{ github.workspace }}/cloudfile-docker/build/cloudfile_14.0/.cache/frontend - NODE_OPTIONS: --max-old-space-size=4096 + VERSION: ${{ inputs.version }} steps: - uses: actions/checkout@v4 with: - path: cloudfile-docker - - - name: Restore compiled frontend - id: frontend-artifact - uses: actions/cache@v4 - with: - path: cloudfile-frontend.tar.gz - # The frontend artifact records the exact Hub SHA; a matching asset-tree - # cache from a different Hub commit must not pass package provenance. - key: cf-frontend-${{ needs.plan.outputs.frontend_key }}-${{ needs.plan.outputs.hub_tree }} - - - name: Free disk space - if: steps.frontend-artifact.outputs.cache-hit != 'true' - run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \ - /usr/local/share/boost "$AGENT_TOOLSDIRECTORY" || true - - - uses: docker/setup-buildx-action@v3 - if: steps.frontend-artifact.outputs.cache-hit != 'true' - - - name: Restore frontend tool caches - if: steps.frontend-artifact.outputs.cache-hit != 'true' - uses: actions/cache@v4 - with: - path: cloudfile-docker/build/cloudfile_14.0/.cache/frontend - key: cf-frontend-tools-${{ needs.plan.outputs.frontend_key }} - restore-keys: cf-frontend-tools- - - - name: Build CE 14 base image - if: steps.frontend-artifact.outputs.cache-hit != 'true' - working-directory: cloudfile-docker/image/cloudfile_14.0 - run: ./base-build.sh - - - name: Compile frontend - if: steps.frontend-artifact.outputs.cache-hit != 'true' - working-directory: cloudfile-docker/build/cloudfile_14.0 - run: ./build-in-docker.sh "$CF_VERSION" - - - name: Pack frontend artifact - if: steps.frontend-artifact.outputs.cache-hit != 'true' - working-directory: cloudfile-docker/build/cloudfile_14.0 - run: | - tar -czf "$GITHUB_WORKSPACE/cloudfile-frontend.tar.gz" \ - cloudfile-frontend - + ref: ${{ needs.plan.outputs.docker_sha }} + - uses: actions/download-artifact@v4 + with: + name: source-receipt + path: build/cloudfile_14.0 + - uses: actions/cache@v4 + with: + path: build/cloudfile_14.0/.cache + key: cf-receipt-tools-${{ runner.os }}-${{ hashFiles('release.yaml', '**/package-lock.json') }} + restore-keys: cf-receipt-tools-${{ runner.os }}- + - name: Prepare CE14 build image + run: bash image/cloudfile_14.0/base-build.sh + - name: Full C/Go/Web build and focused storage tests + run: bash build/cloudfile_14.0/build-in-docker.sh "$VERSION" + - name: Verify package and build isolated image + run: bash image/cloudfile_14.0/docker-build.sh "$VERSION" - uses: actions/upload-artifact@v4 + if: always() with: - name: frontend-${{ needs.plan.outputs.frontend_key }} - path: cloudfile-frontend.tar.gz - retention-days: 1 + name: full-build-evidence + path: | + build/cloudfile_14.0/build-receipt.json + build/cloudfile_14.0/test-results + if-no-files-found: error - package: - if: inputs.mode == 'incremental' - needs: [plan, backend, frontend] + bind-mounts: + needs: plan runs-on: ubuntu-24.04 - timeout-minutes: 60 - env: - CF_SERVER_REF: ${{ needs.plan.outputs.server_sha }} - CF_HUB_REF: ${{ needs.plan.outputs.hub_sha }} - CF_BUILD_TARGET: package - CF_BUILD_JOBS: '2' - CF_CACHE_DIR: ${{ github.workspace }}/cloudfile-docker/build/cloudfile_14.0/.cache/package + timeout-minutes: 45 steps: - uses: actions/checkout@v4 with: + ref: ${{ needs.plan.outputs.docker_sha }} path: cloudfile-docker - - - name: Free disk space - run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \ - /usr/local/share/boost "$AGENT_TOOLSDIRECTORY" || true - - - uses: docker/setup-buildx-action@v3 - - - uses: actions/download-artifact@v5 + - uses: actions/checkout@v4 with: - name: backend-${{ needs.plan.outputs.backend_key }} - path: cloudfile-docker/build/cloudfile_14.0 - - - uses: actions/download-artifact@v5 + repository: eaopen/cloudfile-server + ref: ${{ needs.plan.outputs.server_sha }} + path: cloudfile-server + - uses: actions/download-artifact@v4 with: - name: frontend-${{ needs.plan.outputs.frontend_key }} + name: source-receipt path: cloudfile-docker/build/cloudfile_14.0 - - - name: Restore component artifacts - working-directory: cloudfile-docker/build/cloudfile_14.0 - run: | - tar -xzf cloudfile-backend.tar.gz - tar -xzf cloudfile-frontend.tar.gz - - - name: Restore package dependency cache - uses: actions/cache@v4 - with: - path: cloudfile-docker/build/cloudfile_14.0/.cache/package - key: cf-package-tools-${{ needs.plan.outputs.package_key }} - restore-keys: cf-package-tools- - - - name: Build CE 14 base image - working-directory: cloudfile-docker/image/cloudfile_14.0 - run: ./base-build.sh - - - name: Assemble release package - working-directory: cloudfile-docker/build/cloudfile_14.0 - run: ./build-in-docker.sh "$CF_VERSION" - - - name: Verify release artifact - run: | - cloudfile-docker/tools/verify-release-artifact.sh \ - "cloudfile-docker/build/cloudfile_14.0/seafile-server-$CF_VERSION" - - - name: Build production image - working-directory: cloudfile-docker/image/cloudfile_14.0 - run: ./docker-build.sh "$CF_VERSION" - - - name: Record build sources - run: | - cat "cloudfile-docker/build/cloudfile_14.0/seafile-server-$CF_VERSION/cloudfile-build-info.txt" \ - | tee -a "$GITHUB_STEP_SUMMARY" - - - name: Export image - run: | - docker save "cloudfile/cloudfile:$CF_VERSION" | gzip -1 > "cloudfile-$CF_VERSION.tar.gz" - ls -lh "cloudfile-$CF_VERSION.tar.gz" - + - name: Prepare CE14 test image + run: bash cloudfile-docker/image/cloudfile_14.0/base-build.sh + - name: Privileged mount namespace (no deployed volumes) + run: | + docker run --rm --privileged --pull=never \ + -v "$PWD/cloudfile-docker:/work" -v "$PWD/cloudfile-server:/server:ro" \ + -e CF_TEST_BIND_MOUNTS=1 -w /work cloudfile-build-base:ce14-v2 \ + bash -ec 'git -C /tmp config --global --add safe.directory "*"; bash build/cloudfile_14.0/run-storage-tests.sh build/cloudfile_14.0/build-receipt.json /server' - uses: actions/upload-artifact@v4 + if: always() with: - name: cloudfile-production-image-${{ env.CF_VERSION }} - path: cloudfile-*.tar.gz - retention-days: 14 + name: bind-mount-evidence + path: | + cloudfile-docker/build/cloudfile_14.0/build-receipt.json + cloudfile-docker/build/cloudfile_14.0/test-results diff --git a/.gitignore b/.gitignore index 0893b305..0c6e20ab 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,9 @@ deploy/compose/.*.verify.yml # verify-local.sh ʱ Compose Ŀ¼ .local-verify/ + +# Generated source locks, receipts, and focused test logs. +build/seafile_14.0/release.json +build/*/build-receipt.json +build/*/captured-sources.json +build/*/test-results/ diff --git a/AGENTS.md b/AGENTS.md index 31639592..8d19d159 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,8 +22,8 @@ CloudFile Server 负责 C/Go 底层文件、权限与存储；Hub 负责 Web/API **版本与构建事实必须分开：** - `dev` 表示当前 CloudFile V2 **代码开发线**，不等于正在部署的镜像，也不等于已运行通过的组合。 -- `build/v2/source-set.json` 是其中一次固定 CE14/V2 源码快照；内部 `overlay_commit` 可能早于最新 `dev`，不能仅凭文件名推断为当前 HEAD。 -- 根目录 `release.yaml` 及 `build/cloudfile_14.0/` 保留较早扩展版发布配方与证据；在确认它们的有效源码组合前，**不得直接用其浮动 `ref: dev` 发布当前 V2**。 +- `docs/history/storage-build-20261020/v2/source-set.json` 是其中一次固定 CE14/V2 源码快照；内部 `overlay_commit` 可能早于最新 `dev`，不能仅凭文件名推断为当前 HEAD。 +- 根目录 `release.yaml` 与 `build/cloudfile_14.0/` 是当前唯一构建实现；三仓分支在开工时经远端解析一次，生成 `build-receipt.json`，后续仅按收据里的 SHA 构建和验证。旧白名单组装已归档到 `docs/history/storage-build-20261020/`，不作构建入口。 - 真正构建/交付时，须明确 Server、Hub、Docker 的具体 Commit SHA、上游 CE 源码基线与镜像 Digest；历史 V2-clean CI 成功不代表最新 `dev` 已联验。 ## 不扩大改动范围 diff --git a/README.md b/README.md index 59fb1b0f..a6bcdfc2 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ CloudFile Docker 负责三仓源码组装、镜像构建、部署配置、跨仓 ## V2 构建与旧版入口的区别 -- [V2 源码组装及开发运行说明](build/v2/README.md)：描述 `build/v2/` 路径。`source-set.json` 是一次**固定源码快照**，不能默认它等于三个仓库最新 `dev`。 +- [当前完整构建与收据](docs/storage-build.md)：从三仓 `dev` 解析固定 SHA 后构建，旧白名单组装仅在历史目录保留。 - 根目录 [`release.yaml`](release.yaml) 和 `build/cloudfile_14.0/` 保留较早版本的发布和 CE14 扩展构建方式；**不要未经核实就直接将它们用于当前 V2 `dev` 构建或部署**。 - [历史技术文档](docs/history/README.md) 仅用于追溯旧版行为。实际 V2 的授权、Tag、Checkout 等内置能力以当前 V2 代码/契约为准，不恢复旧 `CF_ENABLE_*` 参数。 @@ -28,6 +28,6 @@ CloudFile Docker 负责三仓源码组装、镜像构建、部署配置、跨仓 | [cloudfile-hub](https://github.com/eaopen/cloudfile-hub) | Web/API、SSO、目录/文件访问、可选增强能力 | | [cloudfile-docker](https://github.com/eaopen/cloudfile-docker) | 构建、运行、部署与跨仓版本组合 | -更多资料：[开发约束](AGENTS.md) · [部署文档](docs/deployment.md) · [V2 源码组装](build/v2/README.md) · [上游改动清单](docs/upstream-patches/)。 +更多资料：[开发约束](AGENTS.md) · [部署文档](docs/deployment.md) · [完整构建](docs/storage-build.md) · [上游改动清单](docs/upstream-patches/)。 本仓保留原上游 Seafile 组件的许可证及各自授权；请核对 [LICENSE.txt](LICENSE.txt) 和具体组件的许可证。 diff --git a/build/cloudfile_14.0/build-in-docker.sh b/build/cloudfile_14.0/build-in-docker.sh index 6c057108..30ea8870 100755 --- a/build/cloudfile_14.0/build-in-docker.sh +++ b/build/cloudfile_14.0/build-in-docker.sh @@ -28,6 +28,19 @@ fi version=$1 here=$(cd "$(dirname "$0")" && pwd) repo_root=$(cd "$here/../.." && pwd) +receipt_tool="$repo_root/build/seafile_14.0/build_receipt.py" +receipt=${CF_BUILD_RECEIPT:-$here/build-receipt.json} +if [[ -z ${CF_BUILD_RECEIPT:-} ]]; then + resolve_args=(--manifest "$repo_root/release.yaml" --version "$version" --output "$receipt") + for component in SERVER HUB DOCKER; do + key="CF_${component}_REF" + option=$(printf '%s' "$component" | tr '[:upper:]' '[:lower:]') + [[ -z ${!key:-} ]] || resolve_args+=("--${option}-ref" "${!key}") + done + python3 "$receipt_tool" resolve "${resolve_args[@]}" +fi +python3 "$receipt_tool" verify "$receipt" --recipe "$repo_root" --version "$version" >/dev/null +receipt=$(cd "$(dirname "$receipt")" && pwd)/$(basename "$receipt") source "$repo_root/tools/build-platform.sh" base_image=${CF_BASE_IMAGE:-$(python3 "$here/read-manifest.py" \ "$repo_root/release.yaml" build_base_image)} @@ -101,6 +114,7 @@ fi # 把可覆盖的 ref 透传进容器，方便构建特性分支；CF_FORCE_REBUILD=1 # 强制全量重建、忽略 cloudfile-build.sh 的分层缓存。 env_args=() +env_args+=(-e "CF_BUILD_RECEIPT=/cf-receipts/$(basename "$receipt")") for v in CF_SERVER_REF CF_HUB_REF CF_SERVER_URL CF_HUB_URL \ CF_SEAFOBJ_REF CF_SEAFDAV_REF CF_SEAFEVENTS_REF \ CF_LIBSEARPC_REF CF_LIBEVHTP_REF CF_FORCE_REBUILD \ @@ -176,13 +190,14 @@ docker run --rm -i "${pull_arg[@]}" \ "${env_args[@]}" \ "${cache_args[@]}" \ "${mount_args[@]+"${mount_args[@]}"}" \ + -v "$(dirname "$receipt"):/cf-receipts" \ -v "$repo_root:/work" \ -w /work/build/cloudfile_14.0 \ "$base_image" \ bash -c " set -e export TZ=Etc/UTC - git config --global --add safe.directory '*' + git -C /tmp config --global --add safe.directory '*' ccache --zero-stats trap 'ccache --show-stats || true' EXIT ./cloudfile-build.sh '$version' diff --git a/build/cloudfile_14.0/cloudfile-build.sh b/build/cloudfile_14.0/cloudfile-build.sh index 303ac9b6..50e4d7e9 100755 --- a/build/cloudfile_14.0/cloudfile-build.sh +++ b/build/cloudfile_14.0/cloudfile-build.sh @@ -16,7 +16,7 @@ # CF_SERVER_REF=feature/my-branch ./cloudfile-build.sh 14.0.0-cf.0 # CF_BUILD_JOBS=8 ./cloudfile-build.sh 14.0.0-cf.0 (parallel C/Go jobs) -set -e +set -euo pipefail if [[ $# != 1 ]]; then echo '' @@ -56,18 +56,31 @@ function manifest_get_optional() { python3 "${current_dir}/read-manifest.py" "$manifest" "$1" 2>/dev/null || true } -cloudfile_server_url=${CF_SERVER_URL:-$(manifest_get 'forks.cloudfile_server.url')} -cloudfile_hub_url=${CF_HUB_URL:-$(manifest_get 'forks.cloudfile_hub.url')} -manifest_server_commit=$(manifest_get_optional 'server_commit') -manifest_hub_commit=$(manifest_get_optional 'hub_commit') -cloudfile_server_ref=${CF_SERVER_REF:-${manifest_server_commit:-$(manifest_get 'forks.cloudfile_server.ref')}} -cloudfile_hub_ref=${CF_HUB_REF:-${manifest_hub_commit:-$(manifest_get 'forks.cloudfile_hub.ref')}} - -seafobj_ref=${CF_SEAFOBJ_REF:-$(manifest_get 'upstream.seafobj')} -seafdav_ref=${CF_SEAFDAV_REF:-$(manifest_get 'upstream.seafdav')} -seafevents_ref=${CF_SEAFEVENTS_REF:-$(manifest_get 'upstream.seafevents')} -libsearpc_ref=${CF_LIBSEARPC_REF:-$(manifest_get 'upstream.libsearpc')} -libevhtp_ref=${CF_LIBEVHTP_REF:-$(manifest_get 'upstream.libevhtp')} +receipt_tool="$repo_root/build/seafile_14.0/build_receipt.py" +receipt=${CF_BUILD_RECEIPT:-$current_dir/build-receipt.json} +if [[ -z ${CF_BUILD_RECEIPT:-} ]]; then + args=(--manifest "$manifest" --version "$version" --output "$receipt") + for component in SERVER HUB DOCKER; do + key="CF_${component}_REF" + option=$(printf '%s' "$component" | tr '[:upper:]' '[:lower:]') + [[ -z ${!key:-} ]] || args+=("--${option}-ref" "${!key}") + done + python3 "$receipt_tool" resolve "${args[@]}" +fi +export CF_BUILD_RECEIPT="$receipt" +python3 "$receipt_tool" verify "$receipt" --version "$version" >/dev/null +locked_manifest="$repo_root/build/seafile_14.0/release.json" +python3 "$receipt_tool" lock "$receipt" "$locked_manifest" +lock_reader="$repo_root/build/seafile_14.0/source_manifest.py" +cloudfile_server_url=${CF_SERVER_URL:-$(python3 "$lock_reader" "$receipt" url seafile-server)} +cloudfile_hub_url=${CF_HUB_URL:-$(python3 "$lock_reader" "$receipt" url seahub)} +cloudfile_server_ref=$(python3 "$lock_reader" "$receipt" ref seafile-server) +cloudfile_hub_ref=$(python3 "$lock_reader" "$receipt" ref seahub) +seafobj_ref=$(python3 "$lock_reader" "$receipt" ref seafobj) +seafdav_ref=$(python3 "$lock_reader" "$receipt" ref seafdav) +seafevents_ref=$(python3 "$lock_reader" "$receipt" ref seafevents) +libsearpc_ref=$(python3 "$lock_reader" "$receipt" ref libsearpc) +libevhtp_ref=$(python3 "$lock_reader" "$receipt" ref libevhtp) # ============================================================================ # 分层构建（layered build） @@ -381,27 +394,14 @@ function checkout_ref() { git clean -xfd fi - # Five upstream components are immutable full SHAs. Once their objects are - # present, fetching the same repositories on every build only adds network - # latency. Fork refs are branches, so they still fetch on every run and can - # never silently reuse a stale local branch. - if [[ $ref =~ ^[0-9a-fA-F]{40}$ ]] && git cat-file -e "${ref}^{commit}" 2>/dev/null; then - echo "Using locally cached commit ${ref}" - elif git fetch --no-tags origin "$ref"; then - target=FETCH_HEAD - else - # Compatibility fallback for servers that reject fetching a raw SHA or - # require tags to be advertised before the ref can be resolved. - git fetch --tags origin - if git show-ref --verify --quiet "refs/remotes/origin/${ref}"; then - target="origin/${ref}" - fi - fi - if ! git checkout --detach "$target" 2>/dev/null; then - # A SHA that predates the shallow fetch, or a branch not yet local. - git fetch origin "$ref" - git checkout --detach FETCH_HEAD + # Branches were resolved before any build. Cached objects are safe only + # when they match that exact receipt SHA. A failed fetch has no fallback. + [[ $ref =~ ^[0-9a-f]{40}$ ]] || { echo "Unresolved build input: $dir" >&2; exit 2; } + if ! git cat-file -e "${ref}^{commit}" 2>/dev/null; then + git fetch --no-tags origin "$ref" fi + git checkout --detach "$ref" + [[ $(git rev-parse HEAD) == "$ref" ]] || exit 2 cd "${code_path}" } @@ -426,6 +426,15 @@ function fetch() { checkout_ref seafile-server "$cloudfile_server_ref" checkout_ref seahub "$cloudfile_hub_ref" + # Capture pristine receipt-matched sources before approved dependency + # patches and generated build files modify the worktrees. + python3 - "$receipt" "$code_path" "$current_dir/captured-sources.json" "$repo_root" <<'PY_CAPTURE' +import json, sys +from pathlib import Path +sys.path.insert(0, str(Path(sys.argv[4]) / 'build/seafile_14.0')) +from package_provenance import capture_sources +Path(sys.argv[3]).write_text(json.dumps(capture_sources(sys.argv[1], sys.argv[2]))) +PY_CAPTURE # Python readers must resolve the same automatic per-key roots as C/Go. apply_patches seafobj apply_patches seafdav @@ -473,16 +482,26 @@ function write_build_info() { # A cached distribution may already carry a provenance record from an # earlier recipe commit. Rewriting build-info invalidates that record; # the image builder must verify and stamp the current bytes again. + if [[ -e "${package_dir}/cloudfile-build.json" || -L "${package_dir}/cloudfile-build.json" ]]; then + python3 "$repo_root/build/seafile_14.0/package_provenance.py" "$package_dir" "$receipt" >/dev/null + fi rm -f "${package_dir}/cloudfile-build.json" { echo "product: ${version}" echo "architecture: $(uname -m)" echo "jobs: ${CF_BUILD_JOBS}" - echo "cloudfile-docker: $(git -C "$repo_root" rev-parse HEAD)" + echo "cloudfile-docker: $(python3 -c 'import json,sys;print(json.load(open(sys.argv[1]))["lock"]["docker_commit"])' "$receipt")" for d in seafile-server seahub seafobj seafdav seafevents libsearpc libevhtp; do echo "${d}: $(git -C "${code_path}/${d}" rev-parse HEAD)" done } > "$out" + python3 - "$package_dir" "$receipt" "$current_dir/captured-sources.json" "$repo_root" <<'PY_STAMP' +import json, sys +from pathlib import Path +sys.path.insert(0, str(Path(sys.argv[4]) / 'build/seafile_14.0')) +from package_provenance import write_provenance +write_provenance(sys.argv[1], json.loads(Path(sys.argv[3]).read_text()), sys.argv[2]) +PY_STAMP echo '' cat "$out" } @@ -742,6 +761,7 @@ function dist_fingerprint() { echo "arch: $(uname -m)" echo "script: $(sha256sum "${current_dir}/cloudfile-build.sh" | cut -d' ' -f1)" echo "builder: $(sha256sum "${current_dir}/cloudfile-build.py" | cut -d' ' -f1)" + python3 -c 'import json,sys; print("docker-recipe=" + json.load(open(sys.argv[1]))["lock"]["docker_commit"])' "$receipt" } | sha256sum | cut -d' ' -f1 } @@ -928,6 +948,11 @@ clone_code fetch layer_done source +if [[ $CF_BUILD_TARGET == all || $CF_BUILD_TARGET == backend ]]; then + # Even cached builds need evidence attached to this build's receipt. + bash "$current_dir/run-storage-tests.sh" "$receipt" "$code_path/seafile-server" +fi + if [[ $CF_BUILD_TARGET == backend ]]; then layer_start 3 backend "C/Go 编译（独立进程，不与前端构建争抢内存）" build_compile diff --git a/build/cloudfile_14.0/run-storage-tests.sh b/build/cloudfile_14.0/run-storage-tests.sh new file mode 100755 index 00000000..b7ab03b6 --- /dev/null +++ b/build/cloudfile_14.0/run-storage-tests.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Required focused tests against precisely the checked-out Server receipt SHA. +set -euo pipefail +receipt=${1:?build receipt required} +server=${2:?Server source required} +here=$(cd "$(dirname "$0")" && pwd) +tool="$here/../seafile_14.0/build_receipt.py" +export CF_AUTO_STORAGE_CASES="$here/../../docs/features/automatic-local-storage-cases.json" +expected=$(python3 "$here/../seafile_14.0/source_manifest.py" "$receipt" ref seafile-server) +[[ $(git -C "$server" rev-parse HEAD) == "$expected" ]] || exit 2 +logs="$here/test-results" +mkdir -p "$logs" +run_test() { + local name=$1 command=$2 status=passed + if ! (cd "$server" && bash -euo pipefail -c "$command") >"$logs/$name.log" 2>&1; then + status=failed + fi + cat "$logs/$name.log" + digest=$(sha256sum "$logs/$name.log" | cut -d' ' -f1) + python3 "$tool" test "$receipt" "$name" "$status" --command "$command" --log-sha256 "$digest" >/dev/null + [[ $status == passed ]] +} +if [[ ${CF_TEST_BIND_MOUNTS:-0} == 1 ]]; then + run_test bind_mounts 'cd fileserver && CF_TEST_BIND_MOUNTS=1 go test -v ./objstore/... -run TestLocalAtomicBindMount -count=1' +else + run_test local_storage 'bash tests/local_storage/run.sh' + run_test objstore 'cd fileserver && go test -v ./objstore/... -count=1' +fi diff --git a/build/cloudfile_14.0/stamp-current-package.py b/build/cloudfile_14.0/stamp-current-package.py index 53c471dc..cf1e8aae 100644 --- a/build/cloudfile_14.0/stamp-current-package.py +++ b/build/cloudfile_14.0/stamp-current-package.py @@ -1,115 +1,65 @@ #!/usr/bin/env python3 -"""Bind a CE14 dev distribution to its actual built source commits and bytes.""" - +"""Verify installed package bytes exclusively against its build receipt.""" import argparse -import importlib.util import json import os +from pathlib import Path import re import subprocess import sys -import tempfile -from pathlib import Path ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / "build/seafile_14.0")) -from package_provenance import ( # noqa: E402 - RECORD_NAME, load_manifest, verify_package, write_provenance, -) +from build_receipt import load_receipt, verify_recipe +from package_provenance import verify_package -READER_SPEC = importlib.util.spec_from_file_location("cloudfile_yaml_reader", ROOT / "build/cloudfile_14.0/read-manifest.py") -READER = importlib.util.module_from_spec(READER_SPEC) -READER_SPEC.loader.exec_module(READER) - -SOURCE_NAMES = ("seafile-server", "seahub", "seafobj", "seafdav", "seafevents", - "libsearpc", "libevhtp") +SOURCE_NAMES = ("seafile-server", "seahub", "seafobj", "seafdav", "seafevents", "libsearpc", "libevhtp") COMMIT = re.compile(r"[0-9a-f]{40}\Z") def build_sources(package, version): - package = Path(package) - info = package / "cloudfile-build-info.txt" + info = Path(package) / "cloudfile-build-info.txt" if not info.is_file() or info.is_symlink() or info.stat().st_size > 4096: raise ValueError("missing immutable build source record") values = {} - for line in info.read_text(encoding="utf-8").splitlines(): + for line in info.read_text().splitlines(): key, separator, value = line.partition(": ") if not separator or key in values: raise ValueError("invalid or duplicate build source field") values[key] = value - if values.get("product") != version or not re.fullmatch(r"[A-Za-z0-9._-]+", version): + if values.get("product") != version: raise ValueError("build version differs from selected distribution") - if set(SOURCE_NAMES) - set(values) or not all(COMMIT.fullmatch(values[key]) for key in SOURCE_NAMES): - raise ValueError("incomplete build source commits") - # Python 3.6 on dev162 predates text=True; preserve decoded Git output. - docker_commit = subprocess.check_output( - ["git", "rev-parse", "HEAD"], cwd=ROOT, universal_newlines=True).strip() - if values.get("cloudfile-docker") != docker_commit: - raise ValueError("distribution was built with a different Docker recipe commit") - return {key: values[key] for key in SOURCE_NAMES} - - -def current_manifest(sources): - """Resolve release.yaml inputs without changing the locked release record.""" - configured = READER.load(ROOT / "release.yaml") - manifest = load_manifest(ROOT / "build/seafile_14.0/release.json") - manifest["seafile_version"] = configured["ce_anchor.version"][1:].split("-server")[0] - for name, commit in sources.items(): - fork = {"seafile-server": "cloudfile_server", "seahub": "cloudfile_hub"}.get(name) - if fork: - checkout = ROOT.parent / name.replace("seafile-server", "cloudfile-server").replace("seahub", "cloudfile-hub") - ref = configured["forks." + fork + ".ref"] - manifest["sources"][name]["url"] = configured["forks." + fork + ".url"] - # Incremental package CI has immutable inputs from the plan job, - # but no matching checkout of each build source. - plan_key = "CF_SERVER_REF" if name == "seafile-server" else "CF_HUB_REF" - expected = os.environ.get(plan_key, "") - if expected and not COMMIT.fullmatch(expected): - raise ValueError("invalid immutable plan source: " + name) - else: - checkout = ROOT / "build/cloudfile_14.0/src" / name - ref = configured["upstream." + name] - expected = ref if COMMIT.fullmatch(ref) else "" - if not expected: - # Preserve local-building support for moving refs, but require - # exact SHA equality when an immutable plan pin is supplied. - # Keep moving-ref resolution compatible with the same host Python. - expected = subprocess.check_output( - ["git", "rev-parse", ref + "^{commit}"], - cwd=checkout, universal_newlines=True).strip() - if expected != commit: - raise ValueError("distribution differs from release.yaml input: " + name) - manifest["sources"][name]["ref"] = commit - return manifest + for name in (*SOURCE_NAMES, "cloudfile-docker"): + if not COMMIT.fullmatch(values.get(name, "")): + raise ValueError("incomplete build source commits") + return values -def stamp(package, version): - package = Path(package) - sources = build_sources(package, version) - manifest = current_manifest(sources) - # The locked release.json remains release evidence. This dev build gets - # an independent immutable manifest and the same package-byte validation. - with tempfile.TemporaryDirectory(prefix="cloudfile-provenance-") as temporary: - manifest_path = Path(temporary) / "release.json" - manifest_path.write_text(json.dumps(manifest), encoding="utf-8") - record = package / RECORD_NAME - if record.exists() or record.is_symlink(): - result = verify_package(package, manifest_path) - else: - result = write_provenance(package, {"schema": 1, "manifest": manifest, - "source_commits": sources}, manifest_path) - return result["package_sha256"] +def stamp(package, version, receipt_path=None): + receipt_path = Path(receipt_path or os.environ.get("CF_BUILD_RECEIPT", ROOT / "build/cloudfile_14.0/build-receipt.json")) + receipt = load_receipt(receipt_path) + verify_recipe(receipt, ROOT) + if receipt["version"] != version: + raise ValueError("receipt version differs from distribution") + values = build_sources(package, version) + if values["cloudfile-docker"] != receipt["lock"]["docker_commit"]: + raise ValueError("distribution has a different Docker source commit") + for name, source in receipt["lock"]["sources"].items(): + if values[name] != source["ref"]: + raise ValueError("distribution differs from build receipt: " + name) + return verify_package(package, receipt_path)["package_sha256"] def main(): parser = argparse.ArgumentParser() parser.add_argument("package") parser.add_argument("version") + parser.add_argument("--receipt") args = parser.parse_args() try: - print(stamp(args.package, args.version)) - except (OSError, ValueError, subprocess.CalledProcessError): - parser.exit(1, "CloudFile distribution source or byte verification failed\n") + print(stamp(args.package, args.version, args.receipt)) + except (OSError, ValueError, subprocess.CalledProcessError) as error: + parser.exit(1, "CloudFile receipt or byte verification failed: %s\n" % error) if __name__ == "__main__": diff --git a/build/seafile_14.0/build-in-docker.sh b/build/seafile_14.0/build-in-docker.sh index 1f543096..cd58022b 100755 --- a/build/seafile_14.0/build-in-docker.sh +++ b/build/seafile_14.0/build-in-docker.sh @@ -1,70 +1,5 @@ -#!/bin/bash -# Reuse v0.1's prebuilt toolchain, native architecture and persistent caches. -# Only clean, pinned local CloudFile commits are mounted; no source worktree edits. +#!/usr/bin/env bash +# Compatibility name; resolve dev once using the canonical receipt builder. set -euo pipefail here=$(cd "$(dirname "$0")" && pwd) -repo=$(cd "${here}/../.." && pwd) -workspace=$(dirname "${repo}") -if [[ $# -gt 1 ]]; then - echo 'Usage: build-in-docker.sh [seafile-version]' >&2 - exit 2 -fi -version=$(python3 "${here}/source_manifest.py" "${here}/release.json" seafile_version) -if [[ $# -eq 1 && "$1" != "${version}" ]]; then - echo 'Version must match release.json' >&2 - exit 2 -fi -native_platform=$(python3 "${here}/image_platform.py" resolve native) -platform=$(python3 "${here}/image_platform.py" resolve "${CF_PLATFORM:-native}") -native_arch=${native_platform#linux/} -package_suffix= -if [[ -n "${CF_PLATFORM:-}" && "${CF_PLATFORM}" != "native" ]]; then - package_suffix=-${platform#linux/} -fi -base=${CF_BASE_IMAGE:-cloudfile-build-base:ce14-v2} -base_arch=$(docker image inspect --format '{{.Architecture}}' "${base}") -base_id=$(docker image inspect --format '{{.Id}}' "${base}") -if [[ "${base_arch}" != "${platform#linux/}" ]]; then - echo 'Build base architecture differs from requested platform' >&2 - exit 2 -fi -if [[ "${platform#linux/}" != "${native_arch}" ]]; then - echo 'Explicit cross-architecture build uses emulation; prefer a native builder.' >&2 -fi -jobs=${CF_BUILD_JOBS:-4} -if [[ ! "${jobs}" =~ ^[1-9][0-9]*$ ]]; then - echo 'CF_BUILD_JOBS must be a positive integer' >&2 - exit 2 -fi -legacy_cache="${workspace}/.v0.1-local/cloudfile-docker/cloudfile_14.0/.cache" -if [[ -n "${CF_CACHE_DIR:-}" ]]; then - cache=${CF_CACHE_DIR} -elif [[ -d "${legacy_cache}" ]]; then - cache=${legacy_cache} - echo 'Reuse existing v0.1 compiler/module/package download caches.' -else - cache=${here}/.cache -fi -mkdir -p "${cache}/ccache" "${cache}/gocache" "${cache}/gomodcache" "${cache}/pip" -cache=$(cd "${cache}" && pwd) -sources=() -for item in 'seafile-server:cloudfile-server:CLOUDFILE_SERVER_SOURCE' 'seahub:cloudfile-hub:CLOUDFILE_HUB_SOURCE'; do - source_name=${item%%:*} - remaining=${item#*:} - checkout_name=${remaining%%:*} - variable=${remaining#*:} - source=${!variable:-${workspace}/${checkout_name}} - pin=$(python3 "${here}/source_manifest.py" "${here}/release.json" ref "${source_name}") - validated=$(python3 "${here}/source_override.py" "${source}" "${pin}") - sources+=(-v "${validated}:/sources/${checkout_name}:ro" -e "${variable}=/sources/${checkout_name}") -done -docker run --rm -i --pull=never --platform "${platform}" \ - -v "${repo}:/work" -v "${cache}:/cache" "${sources[@]}" \ - -w /work/build/seafile_14.0 \ - -e 'PATH=/usr/lib/ccache:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' \ - -e CCACHE_DIR=/cache/ccache -e CCACHE_COMPRESS=true -e CCACHE_MAXSIZE=5G \ - -e GOCACHE=/cache/gocache -e GOMODCACHE=/cache/gomodcache -e PIP_CACHE_DIR=/cache/pip \ - -e "CLOUDFILE_BUILD_BASE_ID=${base_id}" -e "CF_BUILD_JOBS=${jobs}" \ - -e "CF_PACKAGE_SUFFIX=${package_suffix}" \ - -e "CF_FORCE_THIRDPART_REFRESH=${CF_FORCE_THIRDPART_REFRESH:-false}" \ - "${base_id}" bash -c 'set -e; test "${CLOUDFILE_BUILD_BASE:-}" = true; git config --global --add safe.directory /sources/cloudfile-server; git config --global --add safe.directory /sources/cloudfile-hub; git config --global --add safe.directory /work/build/seafile_14.0/src/libevhtp; for source in libsearpc seafile-server seafobj seafdav seafevents seahub; do git config --global --add safe.directory "/work/build/seafile_14.0/src/$source"; done; ./seafile-build.sh' +exec bash "$here/../cloudfile_14.0/build-in-docker.sh" "$@" diff --git a/build/seafile_14.0/build-local-image.sh b/build/seafile_14.0/build-local-image.sh index 590d38d3..47bf21e0 100755 --- a/build/seafile_14.0/build-local-image.sh +++ b/build/seafile_14.0/build-local-image.sh @@ -1,115 +1,4 @@ -#!/bin/bash - -# Assemble an isolated Docker context from the verified package and shared -# runtime assets. This avoids generated copies inside the tracked image tree. +#!/usr/bin/env bash set -euo pipefail - -SCRIPT=$(readlink -f "$0") -current_dir=$(dirname "${SCRIPT}") -repo_dir=$(readlink -f "${current_dir}/../..") -manifest="${current_dir}/release.json" -manifest_reader="${current_dir}/source_manifest.py" -provenance_reader="${current_dir}/package_provenance.py" -version=$(python3 "${manifest_reader}" "${manifest}" seafile_version) -product_version=$(python3 "${manifest_reader}" "${manifest}" product_version) -server_ref=$(python3 "${manifest_reader}" "${manifest}" ref seafile-server) -hub_ref=$(python3 "${manifest_reader}" "${manifest}" ref seahub) -if [[ $# -gt 1 ]]; then - echo 'Usage: build-local-image.sh [image-tag]' >&2 - exit 2 -fi -platform=$(python3 "${current_dir}/image_platform.py" resolve "${CF_PLATFORM:-native}") -arch=${platform#linux/} -if [[ -z "${CF_PLATFORM:-}" || "${CF_PLATFORM}" == "native" ]]; then - default_tag="cloudfile/cloudfile:${version}-${product_version}-local" -else - default_tag="cloudfile/cloudfile:${version}-${product_version}-${arch}-local" -fi -tag=${1:-${default_tag}} -legacy_package="${current_dir}/seafile-server-${version}" -arch_package="${current_dir}/seafile-server-${version}-${arch}" -if [[ -z "${CF_PLATFORM:-}" || "${CF_PLATFORM}" == "native" ]]; then - default_package="${legacy_package}" - [[ -d "${default_package}" ]] || default_package="${arch_package}" -else - default_package="${arch_package}" - [[ -d "${default_package}" ]] || default_package="${legacy_package}" -fi -package="${CLOUDFILE_PACKAGE_DIR:-${default_package}}" - -if [[ ! -d "${package}" ]]; then - echo "Missing verified package: ${package}" >&2 - exit 1 -fi -package_digest=$(python3 "${provenance_reader}" "${package}" "${manifest}") -python3 "${current_dir}/image_platform.py" verify "${platform}" "${package}" >/dev/null -if ! command -v docker >/dev/null 2>&1; then - echo "docker is required to build the local image" >&2 - exit 1 -fi - -context=$(mktemp -d) -trap 'rm -rf "${context}"' EXIT -cp "${repo_dir}/image/seafile_14.0/Dockerfile" "${context}/Dockerfile" -cp "${repo_dir}/image/seafile_14.0/Dockerfile.runtime-base" "${context}/Dockerfile.runtime-base" -cp -a "${repo_dir}/base_scripts" "${context}/base_scripts" -cp -a "${repo_dir}/scripts/scripts_14.0" "${context}/scripts_14.0" -cp -a "${repo_dir}/services" "${context}/services" -cp -a "${package}" "${context}/seafile-server-${version}" -# Validate the copied context too, so a changed source package cannot be labeled -# with the current pins between the original check and context assembly. -copied_digest=$(python3 "${provenance_reader}" "${context}/seafile-server-${version}" "${manifest}") -if [[ "${copied_digest}" != "${package_digest}" ]]; then - echo "Native package changed while preparing the image context" >&2 - exit 1 -fi - -# Reuse v0.1's dependency/application split. BuildKit can reuse the existing -# identical CE14 APT/pip layers for this first extraction too. Refresh is explicit. -legacy_base="cloudfile/runtime-base:${version}-local" -if [[ -z "${CF_PLATFORM:-}" || "${CF_PLATFORM}" == "native" ]]; then - default_base="${legacy_base}" -else - default_base="cloudfile/runtime-base:${version}-${arch}-local" -fi -runtime_base=${CLOUDFILE_RUNTIME_BASE:-${default_base}} -if [[ "${runtime_base}" != "${legacy_base}" && -z "${CLOUDFILE_RUNTIME_BASE:-}" ]] && \ - ! docker image inspect "${runtime_base}" >/dev/null 2>&1 && \ - docker image inspect "${legacy_base}" >/dev/null 2>&1 && \ - [[ "$(docker image inspect --format '{{.Architecture}}' "${legacy_base}")" == "${arch}" ]]; then - docker tag "${legacy_base}" "${runtime_base}" -fi -if [[ "${CLOUDFILE_REFRESH_RUNTIME_BASE:-false}" == "true" ]] || \ - ! docker image inspect "${runtime_base}" >/dev/null 2>&1; then - docker build --platform "${platform}" --pull=false --file "${context}/Dockerfile.runtime-base" \ - --build-arg "server_version=${version}" \ - --label "com.cloudfile.runtime-base.version=${version}" \ - -t "${runtime_base}" "${context}" -fi -base_version=$(docker image inspect --format '{{ index .Config.Labels "com.cloudfile.runtime-base.version" }}' "${runtime_base}") -if [[ "${base_version}" != "${version}" ]]; then - echo "Runtime base does not match CE release ${version}" >&2 - exit 1 -fi -base_arch=$(docker image inspect --format '{{.Architecture}}' "${runtime_base}") -if [[ "${base_arch}" != "${arch}" ]]; then - echo "Runtime base architecture ${base_arch} does not match ${platform}" >&2 - exit 1 -fi -base_id=$(docker image inspect --format '{{.Id}}' "${runtime_base}") -base_pin="cloudfile/runtime-base:${arch}-sha-${base_id#sha256:}" -docker tag "${base_id}" "${base_pin}" -docker build --platform "${platform}" --pull=false \ - --build-arg "CLOUDFILE_RUNTIME_BASE=${base_pin}" \ - --build-arg "server_version=${version}" \ - --label "com.cloudfile.runtime-base.image=${base_id}" \ - --label "org.opencontainers.image.version=${product_version}" \ - --label "com.cloudfile.seafile.version=${version}" \ - --label "com.cloudfile.source.seafile-server=${server_ref}" \ - --label "com.cloudfile.source.seahub=${hub_ref}" \ - --label "com.cloudfile.package.sha256=${package_digest}" \ - --label "com.cloudfile.image.platform=${platform}" \ - -t "${tag}" \ - "${context}" - -echo "Built ${tag}" +here=$(cd "$(dirname "$0")" && pwd) +exec bash "$here/../../image/cloudfile_14.0/docker-build.sh" "$@" diff --git a/build/seafile_14.0/build_receipt.py b/build/seafile_14.0/build_receipt.py new file mode 100644 index 00000000..e0fc9dac --- /dev/null +++ b/build/seafile_14.0/build_receipt.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +"""Resolve branch configuration once, then consume an immutable source lock.""" +import argparse +from datetime import datetime, timezone +import importlib.util +import json +import os +from pathlib import Path +import re +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +SPEC = importlib.util.spec_from_file_location("release_yaml", ROOT / "build/cloudfile_14.0/read-manifest.py") +READER = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(READER) +SHA = re.compile(r"[0-9a-f]{40}\Z") +FORKS = {"seafile-server": "cloudfile_server", "seahub": "cloudfile_hub", + "cloudfile-docker": "cloudfile_docker"} +DEPENDENCIES = ("libevhtp", "libsearpc", "seafobj", "seafdav", "seafevents") + + +def atomic_json(path, value): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + fd, temporary = tempfile.mkstemp(prefix="." + path.name, dir=path.parent) + try: + with os.fdopen(fd, "w") as stream: + json.dump(value, stream, indent=2, sort_keys=True) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + finally: + if os.path.exists(temporary): + os.unlink(temporary) + + +def resolve(manifest, version, overrides=None): + if not re.fullmatch(r"[0-9]+(?:\.[0-9]+)+(?:-[A-Za-z0-9.-]+)?", version): + raise ValueError("invalid build version") + config = READER.load(manifest) + overrides = overrides or {} + branches, commits, urls = {}, {}, {} + # Each branch is resolved exactly once. A remote error is fatal even if + # a previous build or a local object database contains a usable commit. + for name, key in FORKS.items(): + ref = overrides.get(name) or config["forks." + key + ".ref"] + if (not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9/_.-]*", ref) + or ".." in ref or ref.endswith("/") or SHA.fullmatch(ref)): + raise ValueError("select a branch name, not a cached commit: " + name) + url = config["forks." + key + ".url"] + result = subprocess.check_output( + ["git", "ls-remote", "--exit-code", url, "refs/heads/" + ref], text=True) + lines = result.strip().splitlines() + if len(lines) != 1: + raise ValueError("remote branch is absent or ambiguous: " + name) + commit, remote_ref = lines[0].split() + if not SHA.fullmatch(commit) or remote_ref != "refs/heads/" + ref: + raise ValueError("invalid remote branch result: " + name) + branches[name], commits[name], urls[name] = ref, commit, url + sources = {name: {"url": urls[name], "ref": commits[name]} + for name in ("seafile-server", "seahub")} + for name in DEPENDENCIES: + pin = config["upstream." + name] + if not SHA.fullmatch(pin): + raise ValueError("CE dependencies require immutable source commits: " + name) + sources[name] = {"url": "https://github.com/haiwen/" + name + ".git", "ref": pin} + lock = {"schema": 1, "product_version": "v2", "seafile_version": + config["ce_anchor.version"].removeprefix("v").removesuffix("-server"), + "sources": sources, "docker_commit": commits["cloudfile-docker"]} + return {"schema": 2, "created_at": datetime.now(timezone.utc).isoformat(), + "version": version, "requested_branches": branches, "lock": lock, + "tests": {}, "image": None} + + +def load_receipt(path): + receipt = json.loads(Path(path).read_text()) + if receipt.get("schema") != 2 or not isinstance(receipt.get("lock"), dict): + raise ValueError("a generated build receipt is required") + from source_manifest import validate_manifest + validate_manifest(receipt["lock"]) + if not SHA.fullmatch(receipt["lock"].get("docker_commit", "")): + raise ValueError("receipt is missing the Docker source commit") + if not isinstance(receipt.get("tests"), dict): + raise ValueError("receipt has no test results") + return receipt + + +def verify_recipe(receipt, repo): + actual = subprocess.check_output(["git", "-C", str(repo), "rev-parse", "HEAD"], text=True).strip() + if actual != receipt["lock"]["docker_commit"]: + raise ValueError("Docker checkout differs from the resolved build receipt") + dirty = subprocess.check_output( + ["git", "-C", str(repo), "status", "--porcelain", "--untracked-files=normal"], text=True) + if dirty: + raise ValueError("Docker build recipe must have a clean worktree") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + create = sub.add_parser("resolve") + create.add_argument("--manifest", type=Path, required=True) + create.add_argument("--version", required=True) + create.add_argument("--output", type=Path, required=True) + for key in ("server", "hub", "docker"): + create.add_argument("--" + key + "-ref") + verify = sub.add_parser("verify") + verify.add_argument("receipt", type=Path) + verify.add_argument("--recipe", type=Path) + verify.add_argument("--version") + lock = sub.add_parser("lock") + lock.add_argument("receipt", type=Path) + lock.add_argument("output", type=Path) + result = sub.add_parser("test") + result.add_argument("receipt", type=Path) + result.add_argument("name") + result.add_argument("status", choices=("passed", "failed", "not-run")) + result.add_argument("--command", dest="test_command", default="") + result.add_argument("--log-sha256", default="") + image = sub.add_parser("image") + image.add_argument("receipt", type=Path) + image.add_argument("tag") + args = parser.parse_args() + if args.command == "resolve": + overrides = {"seafile-server": args.server_ref, "seahub": args.hub_ref, + "cloudfile-docker": args.docker_ref} + receipt = resolve(args.manifest, args.version, overrides) + verify_recipe(receipt, ROOT) + atomic_json(args.output, receipt) + return + receipt = load_receipt(args.receipt) + if args.command == "verify": + if args.version and args.version != receipt.get("version"): + raise ValueError("build version differs from the resolved receipt") + if args.recipe: + verify_recipe(receipt, args.recipe) + elif args.command == "lock": + atomic_json(args.output, receipt["lock"]) + elif args.command == "test": + receipt["tests"][args.name] = {"status": args.status, "command": args.test_command, + "log_sha256": args.log_sha256} + atomic_json(args.receipt, receipt) + elif args.command == "image": + info = json.loads(subprocess.check_output(["docker", "image", "inspect", args.tag], text=True))[0] + labels = info.get("Config", {}).get("Labels", {}) or {} + for name, commit in (("seafile-server", receipt["lock"]["sources"]["seafile-server"]["ref"]), + ("seahub", receipt["lock"]["sources"]["seahub"]["ref"]), + ("cloudfile-docker", receipt["lock"]["docker_commit"])): + if labels.get("com.cloudfile.source." + name) != commit: + raise ValueError("built image differs from the source receipt: " + name) + if labels.get("org.opencontainers.image.version") != receipt["version"]: + raise ValueError("built image version differs from the receipt") + digest = info["Id"] + if not re.fullmatch(r"sha256:[0-9a-f]{64}", digest): + raise ValueError("invalid image content digest") + receipt["image"] = {"tag": args.tag, "digest": digest, + "digest_kind": "local-image-config", "repo_digests": info.get("RepoDigests", [])} + atomic_json(args.receipt, receipt) + print(json.dumps(receipt, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/build/seafile_14.0/package_provenance.py b/build/seafile_14.0/package_provenance.py index bca2e0b5..cbd167f4 100644 --- a/build/seafile_14.0/package_provenance.py +++ b/build/seafile_14.0/package_provenance.py @@ -18,6 +18,7 @@ REQUIRED_FILES = ( "seafile/bin/seaf-server", "seafile/bin/fileserver", "seafile/bin/seaf-fuse", "seafile/lib/libcloudfile_acl.so.1", "seahub/cloudfile_extensions/apps.py", + "seahub/thirdpart/bin/gunicorn", ) @@ -128,6 +129,13 @@ def write_provenance(package, captured, manifest_path): def verify_package(package, manifest_path): manifest = load_manifest(manifest_path) + raw = json.loads(Path(manifest_path).read_text()) + if raw.get("schema") == 2: + for name in ("local_storage", "objstore"): + result = raw.get("tests", {}).get(name, {}) + if (result.get("status") != "passed" or not result.get("command") + or not re.fullmatch(r"[0-9a-f]{64}", result.get("log_sha256", ""))): + raise ValueError("receipt lacks successful focused test evidence: " + name) target = Path(package) / RECORD_NAME if not target.is_file() or target.is_symlink(): raise ValueError("native package lacks build provenance; rebuild before image assembly") diff --git a/build/seafile_14.0/seafile-build.py b/build/seafile_14.0/seafile-build.py old mode 100755 new mode 100644 index 2175a54d..fe70a3c1 --- a/build/seafile_14.0/seafile-build.py +++ b/build/seafile_14.0/seafile-build.py @@ -1,744 +1,3 @@ #!/usr/bin/env python3 -# coding: UTF-8 - -import sys - -#################### -# Requires Python 3.6+ -#################### -if sys.version_info[0] == 2: - print('Python 2 not supported yet. Quit now.') - sys.exit(1) -if sys.version_info[1] < 6: - print('Python 3.6 or above is required. Quit now.') - sys.exit(1) - -import os -import glob -import subprocess -import tempfile -import shutil -import re -import subprocess -import optparse - -from package_provenance import capture_sources, write_provenance - -#################### -# Global variables -#################### - -# command line configuration -conf = {} - -# key names in the conf dictionary. -CONF_VERSION = 'version' -CONF_SRCDIR = 'srcdir' -CONF_BUILDDIR = 'builddir' -CONF_THIRDPARTDIR = 'thirdpartdir' -CONF_JOBS = 'jobs' -CONF_MYSQL_CONFIG = 'mysql_config' - -#################### -# Common helper functions -#################### -def highlight(content, is_error=False): - '''Add ANSI color to content to get it highlighted on terminal''' - if is_error: - return '\x1b[1;31m%s\x1b[m' % content - else: - return '\x1b[1;32m%s\x1b[m' % content - - -def info(msg): - print(highlight('[INFO] ') + msg) - - -def error(msg=None, usage=None): - if msg: - print(highlight('[ERROR] ') + msg) - if usage: - print(usage) - sys.exit(1) - - -def run_argv(argv, - cwd=None, - env=None, - suppress_stdout=False, - suppress_stderr=False): - '''Run a program and wait it to finish, and return its exit code. The - standard output of this program is suppressed. - - ''' - with open(os.devnull, 'w') as devnull: - if suppress_stdout: - stdout = devnull - else: - stdout = sys.stdout - - if suppress_stderr: - stderr = devnull - else: - stderr = sys.stderr - - proc = subprocess.Popen(argv, - cwd=cwd, - stdout=stdout, - stderr=stderr, - env=env) - return proc.wait() - - -def run(cmdline, - cwd=None, - env=None, - suppress_stdout=False, - suppress_stderr=False): - '''Like run_argv but specify a command line string instead of argv''' - with open(os.devnull, 'w') as devnull: - if suppress_stdout: - stdout = devnull - else: - stdout = sys.stdout - - if suppress_stderr: - stderr = devnull - else: - stderr = sys.stderr - - proc = subprocess.Popen(cmdline, - cwd=cwd, - stdout=stdout, - stderr=stderr, - env=env, - shell=True) - return proc.wait() - - -def must_mkdir(path): - '''Create a directory, exit on failure''' - if os.path.exists(path): - return - - try: - os.makedirs(path) - except OSError as e: - error('failed to create directory %s:%s' % (path, e)) - - -def must_copy(src, dst): - '''Copy src to dst, exit on failure''' - try: - shutil.copy(src, dst) - except Exception as e: - error('failed to copy %s to %s: %s' % (src, dst, e)) - - -def must_copytree(src, dst, with_hidden=False): - '''must_copytree(a, b) copies every file/dir under a/ to b/''' - pattern = os.path.join(src, '*') - try: - for path in glob.glob(pattern): - target_path = os.path.join(dst, os.path.basename(path)) - if os.path.isdir(path): - shutil.copytree(path, target_path, ignore=shutil.ignore_patterns( - '.git', '__pycache__', '*.pyc')) - else: - shutil.copy(path, target_path) - except Exception as e: - error('failed to copy seahub thirdpart libs: %s' % e) - - if with_hidden: - # for hidden dir: .libs_pylibmc - hidden_pattern = os.path.join(src, '.*') - try: - for path in glob.glob(hidden_pattern): - target_path = os.path.join(dst, os.path.basename(path)) - if os.path.isdir(path): - shutil.copytree(path, target_path, ignore=shutil.ignore_patterns( - '.git', '__pycache__', '*.pyc')) - else: - shutil.copy(path, target_path) - except Exception as e: - error('failed to copy seahub thirdpart hidden libs: %s' % e) - - -class Project(object): - '''Base class for a project''' - # Project name, i.e. libseaprc/seafile/seahub - name = '' - - # A list of shell commands to configure/build the project - build_commands = [] - - def __init__(self): - # the path to pass to --prefix=/ - self.prefix = os.path.join(conf[CONF_BUILDDIR], 'seafile-server', - 'seafile') - self.projdir = os.path.join(conf[CONF_SRCDIR], '%s' % self.name) - - def build(self): - '''Build the source''' - info('Building %s' % self.name) - for cmd in self.build_commands: - if run(cmd, cwd=self.projdir) != 0: - error('error when running command:\n\t%s\n' % cmd) - - -class Libsearpc(Project): - name = 'libsearpc' - - def __init__(self): - Project.__init__(self) - self.build_commands = [ - './autogen.sh', - './configure --prefix=%s' % self.prefix, - 'make -j%s' % conf[CONF_JOBS], 'make install' - ] - - -class Libevhtp(Project): - name = 'libevhtp' - - def __init__(self): - Project.__init__(self) - self.build_commands = [ - 'cmake -DEVHTP_DISABLE_SSL=ON -DEVHTP_BUILD_SHARED=OFF .', - 'make', - 'make install', - 'ldconfig' - ] - - -class Seafile(Project): - name = 'seafile-server' - - def __init__(self): - Project.__init__(self) - - configure_command = './configure --prefix=%s' % self.prefix - if conf[CONF_MYSQL_CONFIG]: - configure_command += ' --with-mysql=%s' % conf[CONF_MYSQL_CONFIG] - self.build_commands = [ - './autogen.sh', - configure_command, - 'make -j%s' % conf[CONF_JOBS], - 'make install', - - 'cd fileserver && go build && cd -', - - 'cd notification-server && go build && cd -', - ] - - -class Seahub(Project): - name = 'seahub' - - def __init__(self): - Project.__init__(self) - # nothing to do for seahub - self.build_commands = [] - - def build(self): - self.write_version_to_settings_py() - - Project.build(self) - - def write_version_to_settings_py(self): - '''Write the version of current seafile server to seahub''' - settings_py = os.path.join(self.projdir, 'seahub', 'settings.py') - - line = '\nSEAFILE_VERSION = "%s"\n' % conf[CONF_VERSION] - with open(settings_py, 'a+') as fp: - fp.write(line) - - -def validate_args(usage, options): - required_args = [ - CONF_VERSION, - CONF_SRCDIR, - CONF_THIRDPARTDIR, - ] - - # fist check required args - for optname in required_args: - if getattr(options, optname, None) == None: - error('%s must be specified' % optname, usage=usage) - - def get_option(optname): - return getattr(options, optname) - - # [ version ] - def check_project_version(version): - '''A valid version must be like 1.2.2, 1.3''' - if not re.match('^([0-9])+(\.([0-9])+)+$', version): - error('%s is not a valid version' % version, usage=usage) - - version = get_option(CONF_VERSION) - check_project_version(version) - - # [ srcdir ] - srcdir = get_option(CONF_SRCDIR) - - # [ builddir ] - builddir = get_option(CONF_BUILDDIR) - if not os.path.exists(builddir): - error('%s does not exist' % builddir, usage=usage) - - # [ thirdpartdir ] - thirdpartdir = get_option(CONF_THIRDPARTDIR) - - # [ JOBS ] - jobs = get_option(CONF_JOBS) - - mysql_config_path = get_option(CONF_MYSQL_CONFIG) - - conf[CONF_VERSION] = version - - conf[CONF_BUILDDIR] = builddir - conf[CONF_SRCDIR] = srcdir - conf[CONF_THIRDPARTDIR] = thirdpartdir - conf[CONF_JOBS] = jobs - conf[CONF_MYSQL_CONFIG] = mysql_config_path - - show_build_info() - - prepare_builddir(builddir) - - -def show_build_info(): - '''Print all conf information. Confirm before continue.''' - info('------------------------------------------') - info('Seafile Server Professional %s: BUILD INFO' % conf[CONF_VERSION]) - info('------------------------------------------') - info('builddir: %s' % conf[CONF_BUILDDIR]) - info('source dir: %s' % conf[CONF_SRCDIR]) - info('thirdpart dir: %s' % conf[CONF_THIRDPARTDIR]) - info('jobs: %s' % conf[CONF_JOBS]) - info('------------------------------------------') - - -def prepare_builddir(builddir): - must_mkdir(builddir) - - versioned = os.path.join(builddir, 'seafile-server-' + conf[CONF_VERSION]) - staging = os.path.join(builddir, 'seafile-server') - if os.path.lexists(versioned) or os.path.lexists(staging): - error('choose a fresh build directory; existing packages must not be overlaid') - - os.chdir(builddir) - - must_mkdir(os.path.join(builddir, 'seafile-server')) - must_mkdir(os.path.join(builddir, 'seafile-server', 'seafile')) - - -def parse_args(): - parser = optparse.OptionParser() - - def long_opt(opt): - return '--' + opt - - parser.add_option( - long_opt(CONF_THIRDPARTDIR), - dest=CONF_THIRDPARTDIR, - nargs=1, - help='where to find the thirdpart libs for seahub') - - parser.add_option( - long_opt(CONF_VERSION), - dest=CONF_VERSION, - nargs=1, - help='the version to build. Must be digits delimited by dots, like 1.3.0') - - parser.add_option( - long_opt(CONF_BUILDDIR), - dest=CONF_BUILDDIR, - nargs=1, - help='the directory to build the source. Defaults to /tmp', - default=tempfile.gettempdir()) - - parser.add_option( - long_opt(CONF_SRCDIR), - dest=CONF_SRCDIR, - nargs=1, - help='''Source code must be placed in this directory.''') - - parser.add_option(long_opt(CONF_JOBS), dest=CONF_JOBS, default=2, type=int) - - parser.add_option(long_opt(CONF_MYSQL_CONFIG), - dest=CONF_MYSQL_CONFIG, - nargs=1, - help='''Absolute path to mysql_config or mariadb_config program.''') - - usage = parser.format_help() - options, remain = parser.parse_args() - if remain: - error(usage=usage) - - validate_args(usage, options) - - -def setup_build_env(): - '''Setup environment variables, such as export PATH=$BUILDDDIR/bin:$PATH''' - prefix = os.path.join(conf[CONF_BUILDDIR], 'seafile-server', 'seafile') - - def prepend_env_value(name, value, seperator=':'): - '''append a new value to a list''' - try: - current_value = os.environ[name] - except KeyError: - current_value = '' - - new_value = value - if current_value: - new_value += seperator + current_value - - os.environ[name] = new_value - - prepend_env_value('CPPFLAGS', - '-I%s' % os.path.join(prefix, 'include'), - seperator=' ') - - prepend_env_value('LDFLAGS', - '-L%s' % os.path.join(prefix, 'lib'), - seperator=' ') - - prepend_env_value('LDFLAGS', - '-L%s' % os.path.join(prefix, 'lib64'), - seperator=' ') - - prepend_env_value('PATH', os.path.join(prefix, 'bin')) - prepend_env_value('PKG_CONFIG_PATH', os.path.join(prefix, 'lib', - 'pkgconfig')) - prepend_env_value('PKG_CONFIG_PATH', os.path.join(prefix, 'lib64', - 'pkgconfig')) - - -def copy_pro_libs(): - '''Copy pro.py and python libs for Seafile Professional to - seafile-server/pro/python - - ''' - builddir = conf[CONF_BUILDDIR] - pro_program_dir = os.path.join(builddir, 'seafile-server', 'pro') - if not os.path.exists(pro_program_dir): - must_mkdir(pro_program_dir) - - pro_misc_dir = os.path.join(pro_program_dir, 'misc') - if not os.path.exists(pro_misc_dir): - must_mkdir(pro_misc_dir) - - pro_libs_dir = os.path.join(pro_program_dir, 'python') - must_mkdir(pro_libs_dir) - - pro_py = os.path.join(Seahub().projdir, 'scripts', 'pro.py') - must_copy(pro_py, pro_program_dir) - - copy_seafevents() - - -def copy_seafevents(): - builddir = conf[CONF_BUILDDIR] - pro_libs_dir = os.path.join(builddir, 'seafile-server', 'pro', 'python') - must_mkdir(os.path.join(pro_libs_dir, 'seafevents')) - - events_dir = os.path.join(conf[CONF_SRCDIR], 'seafevents') - - must_copytree(events_dir, os.path.join(pro_libs_dir, 'seafevents')) - - -def copy_seafdav(): - dst_dir = os.path.join(conf[CONF_BUILDDIR], 'seafile-server', 'seahub', - 'thirdpart') - must_mkdir(os.path.join(dst_dir, 'wsgidav')) - must_mkdir(os.path.join(dst_dir, 'seafobj')) - must_mkdir(os.path.join(dst_dir, 'objwrapper')) - - dav_dir = os.path.join(conf[CONF_SRCDIR], 'seafdav', 'wsgidav') - seafobj_dir = os.path.join(conf[CONF_SRCDIR], 'seafobj', 'seafobj') - objwrapper_dir = os.path.join(conf[CONF_SRCDIR], 'seafobj', 'objwrapper') - - must_copytree(dav_dir, os.path.join(dst_dir, 'wsgidav')) - must_copytree(seafobj_dir, os.path.join(dst_dir, 'seafobj')) - must_copytree(objwrapper_dir, os.path.join(dst_dir, 'objwrapper')) - - -def copy_user_manuals(): - builddir = conf[CONF_BUILDDIR] - src_pattern = os.path.join(builddir, Seafile().projdir, 'doc', - 'seafile-tutorial.doc') - dst_dir = os.path.join(builddir, 'seafile-server', 'seafile', 'docs') - - must_mkdir(dst_dir) - - for path in glob.glob(src_pattern): - must_copy(path, dst_dir) - - -def copy_fileserver(): - '''copy fileserver''' - builddir = conf[CONF_BUILDDIR] - dst_dir = os.path.join(builddir, 'seafile-server', 'seafile', 'bin') - fileserver_srcdir = os.path.join(builddir, Seafile().projdir, 'fileserver') - must_copy(os.path.join(fileserver_srcdir, 'fileserver'), dst_dir) - - -def copy_notification_server(): - '''copy notification-server''' - builddir = conf[CONF_BUILDDIR] - dst_dir = os.path.join(builddir, 'seafile-server', 'seafile', 'bin') - notification_server_srcdir = os.path.join(builddir, Seafile().projdir, 'notification-server') - must_copy(os.path.join(notification_server_srcdir, 'notification-server'), dst_dir) - - -def move_python_packages(): - '''move python packages - seafile/lib64/python3.8/site-packages -> seafile/lib64/python3/site-packages - ''' - python_version = 'python' + str(sys.version_info[0]) + '.' + str(sys.version_info[1]) - builddir = conf[CONF_BUILDDIR] - src_parent_dir = os.path.join(builddir, 'seafile-server', 'seafile', 'lib', python_version) - src_dir = os.path.join(src_parent_dir, 'site-packages') - dst_dir = os.path.join(builddir, 'seafile-server', 'seafile', 'lib', 'python3') - src_parent_dir_64 = os.path.join(builddir, 'seafile-server', 'seafile', 'lib64', python_version) - src_dir_64 = os.path.join(src_parent_dir_64, 'site-packages') - dst_dir_64 = os.path.join(builddir, 'seafile-server', 'seafile', 'lib64', 'python3') - if os.path.exists(src_dir): - must_mkdir(dst_dir) - shutil.move(src_dir, dst_dir) - os.removedirs(src_parent_dir) - if os.path.exists(src_dir_64): - must_mkdir(dst_dir_64) - shutil.move(src_dir_64, dst_dir_64) - os.removedirs(src_parent_dir_64) - - -def copy_scripts_and_libs(): - '''Copy server release scripts and shared libs, as well as seahub - thirdpart libs - - ''' - builddir = conf[CONF_BUILDDIR] - scripts_srcdir = os.path.join(builddir, Seahub().projdir, 'scripts') - serverdir = os.path.join(builddir, 'seafile-server') - - must_copy(os.path.join(scripts_srcdir, 'setup-seafile.sh'), serverdir) - must_copy( - os.path.join(scripts_srcdir, 'setup-seafile-mysql.sh'), serverdir) - must_copy( - os.path.join(scripts_srcdir, 'setup-seafile-mysql.py'), serverdir) - must_copy(os.path.join(scripts_srcdir, 'seafile.sh'), serverdir) - must_copy(os.path.join(scripts_srcdir, 'seahub.sh'), serverdir) - must_copy(os.path.join(scripts_srcdir, 'reset-admin.sh'), serverdir) - must_copy(os.path.join(scripts_srcdir, 'seaf-fuse.sh'), serverdir) - must_copy(os.path.join(scripts_srcdir, 'seaf-gc.sh'), serverdir) - must_copy(os.path.join(scripts_srcdir, 'seaf-fsck.sh'), serverdir) - must_copy(os.path.join(scripts_srcdir, 'check_init_admin.py'), serverdir) - must_copy(os.path.join(scripts_srcdir, 'seafile-monitor.sh'), serverdir) - must_copy(os.path.join(scripts_srcdir, 'migrate_ldapusers.py'), serverdir) - - # copy update scripts - update_scriptsdir = os.path.join(scripts_srcdir, 'upgrade') - dst_update_scriptsdir = os.path.join(serverdir, 'upgrade') - try: - shutil.copytree(update_scriptsdir, dst_update_scriptsdir) - except Exception as e: - error('failed to copy upgrade scripts: %s' % e) - - # copy sql scripts - sql_scriptsdir = os.path.join(builddir, Seafile().projdir, 'scripts', 'sql') - dst_sql_scriptsdir = os.path.join(serverdir, 'sql') - try: - shutil.copytree(sql_scriptsdir, dst_sql_scriptsdir) - except Exception as e: - error('failed to copy sql scripts: %s' % e) - - # copy runtime/seahub.conf - runtimedir = os.path.join(serverdir, 'runtime') - must_mkdir(runtimedir) - must_copy(os.path.join(scripts_srcdir, 'seahub.conf'), runtimedir) - - # copy seahub to seafile-server/seahub - src_seahubdir = Seahub().projdir - dst_seahubdir = os.path.join(serverdir, 'seahub') - try: - shutil.copytree(src_seahubdir, dst_seahubdir, ignore=shutil.ignore_patterns( - '.git', '__pycache__', '*.pyc')) - except Exception as e: - error('failed to copy seahub to seafile-server/seahub: %s' % e) - - # copy seahub thirdpart libs - seahub_thirdpart = os.path.join(dst_seahubdir, 'thirdpart') - copy_seahub_thirdpart_libs(seahub_thirdpart) - copy_seafdav() - - # copy pro libs - copy_pro_libs() - - move_python_packages() - - # copy fileserver - copy_fileserver() - # copy notification-server - copy_notification_server() - - # copy shared c libs - copy_cloudfile_acl_core() - copy_shared_libs() - copy_user_manuals() - - -def get_dependent_libs(executable): - syslibs = ['libsearpc', 'libseafile', 'libpthread.so', - 'libc.so', 'libm.so', 'librt.so', 'libdl.so', 'libselinux.so', - 'libresolv.so'] - - def is_syslib(lib): - for syslib in syslibs: - if syslib in lib: - return True - return False - - ldd_output = subprocess.getoutput('ldd %s' % executable) - ret = set() - for line in ldd_output.splitlines(): - tokens = line.split() - if len(tokens) != 4: - continue - if is_syslib(tokens[0]): - continue - ret.add(tokens[2]) - - return ret - - -def copy_cloudfile_acl_core(): - """Package the Server-installed ABI1 core for both native and Hub consumers. - - Do not depend on ldd discovering an unversioned development symlink, or - silently package a runtime that cannot load its policy core. - """ - prefix = os.path.join(conf[CONF_BUILDDIR], 'seafile-server', 'seafile') - candidates = [os.path.join(prefix, directory, 'seafile', 'libcloudfile_acl.so.1') - for directory in ('lib', 'lib64')] - installed = [path for path in candidates if os.path.isfile(path)] - if len(installed) != 1: - error('CloudFile ABI1 permission core must be installed exactly once') - source = installed[0] - destination = os.path.join(prefix, 'lib', 'libcloudfile_acl.so.1') - must_copy(source, destination) - - -def copy_shared_libs(): - '''copy shared c libs, such as libevent, glib, libmysqlclient''' - builddir = conf[CONF_BUILDDIR] - - dst_dir = os.path.join(builddir, 'seafile-server', 'seafile', 'lib') - - seafile_path = os.path.join(builddir, 'seafile-server', 'seafile', 'bin', - 'seaf-server') - - seaf_fuse_path = os.path.join(builddir, 'seafile-server', 'seafile', 'bin', - 'seaf-fuse') - - libs = set() - libs.update(get_dependent_libs(seafile_path)) - libs.update(get_dependent_libs(seaf_fuse_path)) - - for lib in libs: - if lib == 'not': - continue - dst_file = os.path.join(dst_dir, os.path.basename(lib)) - if os.path.exists(dst_file): - continue - info('Copying %s' % lib) - shutil.copy(lib, dst_dir) - - -def copy_seahub_thirdpart_libs(seahub_thirdpart): - '''copy django/djblets/gunicorn from ${thirdpartdir} to - seahub/thirdpart - - ''' - src = conf[CONF_THIRDPARTDIR] - dst = seahub_thirdpart - - must_copytree(src, dst) - - -def strip_symbols(): - def do_strip(fn): - run('chmod u+w %s' % fn) - info('stripping: %s' % fn) - run('strip "%s"' % fn) - - def remove_static_lib(fn): - info('removing: %s' % fn) - os.remove(fn) - - for parent, dnames, fnames in os.walk('seafile-server/seafile'): - dummy = dnames # avoid pylint 'unused' warning - for fname in fnames: - fn = os.path.join(parent, fname) - if os.path.isdir(fn): - continue - - if fn.endswith(".a") or fn.endswith(".la"): - remove_static_lib(fn) - continue - - if os.path.islink(fn): - continue - - finfo = subprocess.getoutput('file "%s"' % fn) - - if 'not stripped' in finfo: - do_strip(fn) - - -def strip_and_rename(): - # strip symbols of libraries to reduce size - try: - strip_symbols() - except Exception as e: - error('failed to strip symbols: %s' % e) - - version = conf[CONF_VERSION] - serverdir = 'seafile-server' - versioned_serverdir = 'seafile-server-' + version - - # move seafile-server to seafile-server-${version} - try: - shutil.move(serverdir, versioned_serverdir) - except Exception as e: - error('failed to move %s to %s: %s' % - (serverdir, versioned_serverdir, e)) - -def main(): - parse_args() - manifest_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'release.json') - sources = capture_sources(manifest_path, conf[CONF_SRCDIR]) - setup_build_env() - - libsearpc = Libsearpc() - libevhtp = Libevhtp() - seafile = Seafile() - seahub = Seahub() - - libsearpc.build() - libevhtp.build() - seafile.build() - seahub.build() - - copy_scripts_and_libs() - strip_and_rename() - package = os.path.join(conf[CONF_BUILDDIR], 'seafile-server-' + conf[CONF_VERSION]) - write_provenance(package, sources, manifest_path) - print('---------------------------------------------') - print('The build is successful; package provenance recorded.') - print('---------------------------------------------') - - -if __name__ == '__main__': - main() +"""Historical entry point; use the complete dev build and its receipt.""" +raise SystemExit('Use build/cloudfile_14.0/build-in-docker.sh ; release.json is generated, never hand maintained') diff --git a/build/seafile_14.0/seafile-build.sh b/build/seafile_14.0/seafile-build.sh index abd07268..9813ee3c 100755 --- a/build/seafile_14.0/seafile-build.sh +++ b/build/seafile_14.0/seafile-build.sh @@ -1,254 +1,5 @@ -#!/bin/bash - -# A partial native package is unusable; stop on the first failed clone, -# dependency install, compile, or packaging command instead of printing success. +#!/usr/bin/env bash +# Compatibility name; the full dev builder is the single implementation. set -euo pipefail - -if [[ $# -gt 1 ]]; then - echo '' - echo 'Usage: ./seafile-build.sh [seafile-version]' - echo '' - exit 2 -fi - -SCRIPT=$(readlink -f "$0") -current_dir=$(dirname "${SCRIPT}") -code_path=$current_dir/src -manifest=$current_dir/release.json -manifest_reader=$current_dir/source_manifest.py -override_validator=$current_dir/source_override.py -version=$(python3 "${manifest_reader}" "${manifest}" seafile_version) || exit 1 -product_version=$(python3 "${manifest_reader}" "${manifest}" product_version) || exit 1 -package_suffix=${CF_PACKAGE_SUFFIX:-} -case "${package_suffix}" in ''|-arm64|-amd64) ;; *) echo 'Invalid CF_PACKAGE_SUFFIX' >&2; exit 2 ;; esac -if [[ $# -eq 1 && "$1" != "${version}" ]]; then - echo "Requested Seafile version $1 does not match release manifest ${version}" >&2 - exit 2 -fi -mkdir -p "${code_path}" - -function manifest_value() { - python3 "${manifest_reader}" "${manifest}" "$2" "$1" -} - -function source_override() { - # Local overrides are intentionally limited to CloudFile-owned repositories. - # This lets an operator build exact local commits before a human pushes them. - case "$1" in - seafile-server) - echo "${CLOUDFILE_SERVER_SOURCE:-}" - ;; - seahub) - echo "${CLOUDFILE_HUB_SOURCE:-}" - ;; - *) - echo "" - ;; - esac -} - -function install_dependencies() { - if [[ "${CLOUDFILE_BUILD_BASE:-false}" == "true" ]]; then - echo 'Reuse the prebuilt CE14 toolchain; skip APT installation.' - return - fi - apt-get update && apt-get upgrade -y - apt-get install -y build-essential - export DEBIAN_FRONTEND=noninteractive && apt-get install -y tzdata - apt-get install -y \ - ca-certificates \ - cargo \ - cmake \ - git \ - golang-go \ - intltool \ - libarchive-dev \ - libcurl4-openssl-dev \ - libevent-dev \ - libffi-dev \ - libfuse-dev \ - libglib2.0-dev \ - libjansson-dev \ - libjpeg-dev \ - libjwt-dev \ - libldap2-dev \ - libmariadbclient-dev-compat \ - libonig-dev \ - libpq-dev \ - libsqlite3-dev \ - libssl-dev \ - libtool \ - libxml2-dev \ - libxslt-dev \ - python3 \ - python3-distro \ - python3-lxml \ - python3-ldap \ - python3-pip \ - python3-setuptools \ - python3-wheel \ - uuid-dev \ - valac \ - wget \ - curl \ - libunwind-dev \ - libhiredis-dev \ - google-perftools \ - libgoogle-perftools-dev \ - libargon2-dev -} - -function install_python_dependencies() { - cat ${code_path}/seafevents/requirements.txt ${code_path}/seafdav/requirements.txt ${code_path}/seahub/requirements.txt >${code_path}/requirements-thirdpart.txt - cd ${code_path} - - # seafevents ignore - sed -i 's/SQLAlchemy/# SQLAlchemy/' requirements-thirdpart.txt - sed -i 's/mock/# mock/' requirements-thirdpart.txt - sed -i 's/pytest/# pytest/' requirements-thirdpart.txt - sed -i 's/gevent/# gevent/' requirements-thirdpart.txt - sed -i 's/numpy/# numpy/' requirements-thirdpart.txt - sed -i 's/scikit-learn/# scikit-learn/' requirements-thirdpart.txt - - # seafdav ignore - sed -i 's/Jinja2/# Jinja2/' requirements-thirdpart.txt - sed -i 's/sqlalchemy/# sqlalchemy/' requirements-thirdpart.txt - - # seahub ignore - sed -i 's/django_simple_captcha/# django_simple_captcha/' requirements-thirdpart.txt - sed -i 's/^captcha/# captcha/' requirements-thirdpart.txt - sed -i 's/mysqlclient/# mysqlclient/' requirements-thirdpart.txt - sed -i 's/pillow/# pillow/' requirements-thirdpart.txt - sed -i 's/pycryptodome/# pycryptodome/' requirements-thirdpart.txt - sed -i 's/djangosaml2/# djangosaml2/' requirements-thirdpart.txt - sed -i 's/pysaml2/# pysaml2/' requirements-thirdpart.txt - sed -i 's/cffi/# cffi/' requirements-thirdpart.txt - sed -i 's/python-ldap/# python-ldap/' requirements-thirdpart.txt - sed -i 's/PyMuPDF/# PyMuPDF/' requirements-thirdpart.txt - sed -i 's/cairosvg/# cairosvg/' requirements-thirdpart.txt - - # pymysql for scripts - sed -i '$a\pymysql' requirements-thirdpart.txt - # Reuse installed Python dependencies only for this exact requirement set, - # toolchain image and interpreter. A partial/old directory lacks the marker. - dependency_key=$(python3 - <<'PY' -import hashlib, os, platform, sys -from pathlib import Path -value = Path('requirements-thirdpart.txt').read_bytes() -value += repr((sys.version, platform.machine(), os.environ.get('CLOUDFILE_BUILD_BASE_ID', 'unmanaged'))).encode() -print(hashlib.sha256(value).hexdigest()) -PY -) - marker="${code_path}/thirdpartdir/.cloudfile-requirements.sha256" - if [[ "${CF_FORCE_THIRDPART_REFRESH:-false}" != "true" && -f "${marker}" && \ - "$(cat "${marker}")" == "${dependency_key}" ]]; then - echo 'Reuse matching installed Python dependencies.' - return - fi - thirdpart_output=$(mktemp -d "${code_path}/thirdpart-build.XXXXXX") - pip3 install -r requirements-thirdpart.txt -t "${thirdpart_output}" - echo "${dependency_key}" > "${thirdpart_output}/.cloudfile-requirements.sha256" - if [[ -e "${code_path}/thirdpartdir" ]]; then - previous_dependencies=$(mktemp -d "${code_path}/thirdpart-previous.XXXXXX") - mv "${code_path}/thirdpartdir" "${previous_dependencies}/thirdpartdir" - fi - mv "${thirdpart_output}" "${code_path}/thirdpartdir" -} - -function clone_code() { - cd "${code_path}" || exit 1 - - for source in libevhtp libsearpc seafile-server seafobj seafdav seafevents seahub; do - source_url=$(manifest_value "${source}" url) || exit 1 - source_ref=$(manifest_value "${source}" ref) || exit 1 - override=$(source_override "${source}") - if [[ ! -e "${source}" ]]; then - if [[ -n "${override}" ]]; then - validated_override=$(python3 "${override_validator}" "${override}" "${source_ref}") || exit 1 - git clone --no-hardlinks "${validated_override}" "${source}" || exit 1 - else - git clone "${source_url}" "${source}" || exit 1 - fi - elif [[ -d "${source}/.git" ]]; then - if [[ -n "${override}" ]]; then - validated_override=$(python3 "${override_validator}" "${override}" "${source_ref}") || exit 1 - git -C "${source}" remote set-url origin "${validated_override}" || exit 1 - else - git -C "${source}" remote set-url origin "${source_url}" || exit 1 - fi - else - echo "Existing source path is not a Git repository: ${code_path}/${source}" >&2 - exit 1 - fi - done -} - -function fetch() { - cd "${code_path}" || exit 1 - - echo "Start fetch" - echo '' - - for source in libevhtp libsearpc seafile-server seafobj seafdav seafevents seahub; do - echo "Fetch ${source}" - source_ref=$(manifest_value "${source}" ref) || exit 1 - override=$(source_override "${source}") - cd "${code_path}/${source}" || exit 1 - git reset --hard || exit 1 - git clean -xffd || exit 1 - if [[ -n "${override}" ]]; then - validated_override=$(python3 "${override_validator}" "${override}" "${source_ref}") || exit 1 - git remote set-url origin "${validated_override}" || exit 1 - fi - if [[ "$(git rev-parse HEAD)" != "${source_ref}" ]]; then - git fetch --force origin "${source_ref}" || exit 1 - git checkout --detach FETCH_HEAD || exit 1 - fi - cd "${code_path}" || exit 1 - done -} - -function build() { - cd "${current_dir}" - # Never overlay an old completed package or a previous partial build. - # Failed output stays isolated for diagnosis; the last package stays intact. - build_output=$(mktemp -d "${current_dir}/build-output.XXXXXX") - python3 ./seafile-build.py --version="${version}" --builddir="${build_output}" \ - --srcdir="${code_path}" --thirdpartdir="${code_path}/thirdpartdir" \ - --mysql_config=/usr/bin/mariadb_config --jobs="${CF_BUILD_JOBS:-4}" - python3 ./package_provenance.py "${build_output}/seafile-server-${version}" "${manifest}" - package="${current_dir}/seafile-server-${version}${package_suffix}" - if [[ -e "${package}" || -L "${package}" ]]; then - previous=$(mktemp -d "${current_dir}/seafile-server-${version}.previous.XXXXXX") - mv "${package}" "${previous}/package" - fi - if ! mv "${build_output}/seafile-server-${version}" "${package}"; then - if [[ -n "${previous:-}" ]]; then - mv "${previous}/package" "${package}" - fi - exit 1 - fi - rmdir "${build_output}" -} - -echo '' -echo "Info: CloudFile ${product_version}, Seafile ${version}" -echo '' - -install_dependencies -wait - -clone_code -wait - -fetch -wait - -install_python_dependencies -wait - -build - -echo '' -echo "Info: Successfully built ${package}" -echo '' +here=$(cd "$(dirname "$0")" && pwd) +exec bash "$here/../cloudfile_14.0/cloudfile-build.sh" "$@" diff --git a/build/seafile_14.0/source_manifest.py b/build/seafile_14.0/source_manifest.py index 24625356..566fc4ae 100644 --- a/build/seafile_14.0/source_manifest.py +++ b/build/seafile_14.0/source_manifest.py @@ -24,10 +24,18 @@ def load_manifest(path): with Path(path).open(encoding="utf-8") as manifest_file: manifest = json.load(manifest_file) + # Configuration chooses branches; validation consumes only the generated + # immutable lock in a receipt (or its generated release.json projection). + if manifest.get("schema") == 2: + manifest = manifest["lock"] + return validate_manifest(manifest) + + +def validate_manifest(manifest): if manifest.get("schema") != 1: raise ValueError("release manifest schema must be 1") - if manifest.get("product_version") != "v0.2": - raise ValueError("release manifest product_version must be v0.2") + if manifest.get("product_version") not in ("v0.2", "v2"): + raise ValueError("unknown locked product version") version = manifest.get("seafile_version") if not isinstance(version, str) or not re.fullmatch(r"14\.0\.\d+", version): raise ValueError("release manifest must select a Seafile 14.0 patch version") diff --git a/build/seafile_14.0/source_override.py b/build/seafile_14.0/source_override.py index fbbe5abc..6c3577c4 100644 --- a/build/seafile_14.0/source_override.py +++ b/build/seafile_14.0/source_override.py @@ -49,9 +49,19 @@ def validate_override(path, expected_commit): def main(): parser = argparse.ArgumentParser() parser.add_argument("path") - parser.add_argument("expected_commit") + parser.add_argument("expected_commit", nargs="?") + parser.add_argument("--receipt", help="Generated receipt, never branch configuration") + parser.add_argument("--source", choices=("seafile-server", "seahub")) arguments = parser.parse_args() - print(validate_override(arguments.path, arguments.expected_commit)) + expected = arguments.expected_commit + if arguments.receipt: + from source_manifest import load_manifest + if not arguments.source: + parser.error("--source is required with --receipt") + expected = load_manifest(arguments.receipt)["sources"][arguments.source]["ref"] + if expected is None: + parser.error("a receipt or an immutable expected commit is required") + print(validate_override(arguments.path, expected)) if __name__ == "__main__": diff --git a/docs/history/storage-build-20261020/README.md b/docs/history/storage-build-20261020/README.md new file mode 100644 index 00000000..166dfe4e --- /dev/null +++ b/docs/history/storage-build-20261020/README.md @@ -0,0 +1,9 @@ +# 历史构建链归档 + +此目录保留旧 CE14 白名单叠加组装、独立 V2 DEV 启动器和旧原生构建入口，供追溯。 +它们不再是可执行的当前构建流程，移动后的相对路径也不作为运行合同。 +其中 source-set.json 的原 SHA 原样保留，未通过改旧 pin 修复当前构建。 + +当前唯一实现为根目录 release.yaml 与 build/cloudfile_14.0/cloudfile-build.sh； +容器包装器 build-in-docker.sh 和镜像包装器 docker-build.sh 使用同一份生成收据。 +旧 build/seafile_14.0/release.json 从 Git 删除，构建按收据生成，禁止手工维护。 diff --git a/docs/history/storage-build-20261020/native-build/build-in-docker.sh b/docs/history/storage-build-20261020/native-build/build-in-docker.sh new file mode 100755 index 00000000..1f543096 --- /dev/null +++ b/docs/history/storage-build-20261020/native-build/build-in-docker.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Reuse v0.1's prebuilt toolchain, native architecture and persistent caches. +# Only clean, pinned local CloudFile commits are mounted; no source worktree edits. +set -euo pipefail +here=$(cd "$(dirname "$0")" && pwd) +repo=$(cd "${here}/../.." && pwd) +workspace=$(dirname "${repo}") +if [[ $# -gt 1 ]]; then + echo 'Usage: build-in-docker.sh [seafile-version]' >&2 + exit 2 +fi +version=$(python3 "${here}/source_manifest.py" "${here}/release.json" seafile_version) +if [[ $# -eq 1 && "$1" != "${version}" ]]; then + echo 'Version must match release.json' >&2 + exit 2 +fi +native_platform=$(python3 "${here}/image_platform.py" resolve native) +platform=$(python3 "${here}/image_platform.py" resolve "${CF_PLATFORM:-native}") +native_arch=${native_platform#linux/} +package_suffix= +if [[ -n "${CF_PLATFORM:-}" && "${CF_PLATFORM}" != "native" ]]; then + package_suffix=-${platform#linux/} +fi +base=${CF_BASE_IMAGE:-cloudfile-build-base:ce14-v2} +base_arch=$(docker image inspect --format '{{.Architecture}}' "${base}") +base_id=$(docker image inspect --format '{{.Id}}' "${base}") +if [[ "${base_arch}" != "${platform#linux/}" ]]; then + echo 'Build base architecture differs from requested platform' >&2 + exit 2 +fi +if [[ "${platform#linux/}" != "${native_arch}" ]]; then + echo 'Explicit cross-architecture build uses emulation; prefer a native builder.' >&2 +fi +jobs=${CF_BUILD_JOBS:-4} +if [[ ! "${jobs}" =~ ^[1-9][0-9]*$ ]]; then + echo 'CF_BUILD_JOBS must be a positive integer' >&2 + exit 2 +fi +legacy_cache="${workspace}/.v0.1-local/cloudfile-docker/cloudfile_14.0/.cache" +if [[ -n "${CF_CACHE_DIR:-}" ]]; then + cache=${CF_CACHE_DIR} +elif [[ -d "${legacy_cache}" ]]; then + cache=${legacy_cache} + echo 'Reuse existing v0.1 compiler/module/package download caches.' +else + cache=${here}/.cache +fi +mkdir -p "${cache}/ccache" "${cache}/gocache" "${cache}/gomodcache" "${cache}/pip" +cache=$(cd "${cache}" && pwd) +sources=() +for item in 'seafile-server:cloudfile-server:CLOUDFILE_SERVER_SOURCE' 'seahub:cloudfile-hub:CLOUDFILE_HUB_SOURCE'; do + source_name=${item%%:*} + remaining=${item#*:} + checkout_name=${remaining%%:*} + variable=${remaining#*:} + source=${!variable:-${workspace}/${checkout_name}} + pin=$(python3 "${here}/source_manifest.py" "${here}/release.json" ref "${source_name}") + validated=$(python3 "${here}/source_override.py" "${source}" "${pin}") + sources+=(-v "${validated}:/sources/${checkout_name}:ro" -e "${variable}=/sources/${checkout_name}") +done +docker run --rm -i --pull=never --platform "${platform}" \ + -v "${repo}:/work" -v "${cache}:/cache" "${sources[@]}" \ + -w /work/build/seafile_14.0 \ + -e 'PATH=/usr/lib/ccache:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' \ + -e CCACHE_DIR=/cache/ccache -e CCACHE_COMPRESS=true -e CCACHE_MAXSIZE=5G \ + -e GOCACHE=/cache/gocache -e GOMODCACHE=/cache/gomodcache -e PIP_CACHE_DIR=/cache/pip \ + -e "CLOUDFILE_BUILD_BASE_ID=${base_id}" -e "CF_BUILD_JOBS=${jobs}" \ + -e "CF_PACKAGE_SUFFIX=${package_suffix}" \ + -e "CF_FORCE_THIRDPART_REFRESH=${CF_FORCE_THIRDPART_REFRESH:-false}" \ + "${base_id}" bash -c 'set -e; test "${CLOUDFILE_BUILD_BASE:-}" = true; git config --global --add safe.directory /sources/cloudfile-server; git config --global --add safe.directory /sources/cloudfile-hub; git config --global --add safe.directory /work/build/seafile_14.0/src/libevhtp; for source in libsearpc seafile-server seafobj seafdav seafevents seahub; do git config --global --add safe.directory "/work/build/seafile_14.0/src/$source"; done; ./seafile-build.sh' diff --git a/docs/history/storage-build-20261020/native-build/build-local-image.sh b/docs/history/storage-build-20261020/native-build/build-local-image.sh new file mode 100755 index 00000000..590d38d3 --- /dev/null +++ b/docs/history/storage-build-20261020/native-build/build-local-image.sh @@ -0,0 +1,115 @@ +#!/bin/bash + +# Assemble an isolated Docker context from the verified package and shared +# runtime assets. This avoids generated copies inside the tracked image tree. +set -euo pipefail + +SCRIPT=$(readlink -f "$0") +current_dir=$(dirname "${SCRIPT}") +repo_dir=$(readlink -f "${current_dir}/../..") +manifest="${current_dir}/release.json" +manifest_reader="${current_dir}/source_manifest.py" +provenance_reader="${current_dir}/package_provenance.py" +version=$(python3 "${manifest_reader}" "${manifest}" seafile_version) +product_version=$(python3 "${manifest_reader}" "${manifest}" product_version) +server_ref=$(python3 "${manifest_reader}" "${manifest}" ref seafile-server) +hub_ref=$(python3 "${manifest_reader}" "${manifest}" ref seahub) +if [[ $# -gt 1 ]]; then + echo 'Usage: build-local-image.sh [image-tag]' >&2 + exit 2 +fi +platform=$(python3 "${current_dir}/image_platform.py" resolve "${CF_PLATFORM:-native}") +arch=${platform#linux/} +if [[ -z "${CF_PLATFORM:-}" || "${CF_PLATFORM}" == "native" ]]; then + default_tag="cloudfile/cloudfile:${version}-${product_version}-local" +else + default_tag="cloudfile/cloudfile:${version}-${product_version}-${arch}-local" +fi +tag=${1:-${default_tag}} +legacy_package="${current_dir}/seafile-server-${version}" +arch_package="${current_dir}/seafile-server-${version}-${arch}" +if [[ -z "${CF_PLATFORM:-}" || "${CF_PLATFORM}" == "native" ]]; then + default_package="${legacy_package}" + [[ -d "${default_package}" ]] || default_package="${arch_package}" +else + default_package="${arch_package}" + [[ -d "${default_package}" ]] || default_package="${legacy_package}" +fi +package="${CLOUDFILE_PACKAGE_DIR:-${default_package}}" + +if [[ ! -d "${package}" ]]; then + echo "Missing verified package: ${package}" >&2 + exit 1 +fi +package_digest=$(python3 "${provenance_reader}" "${package}" "${manifest}") +python3 "${current_dir}/image_platform.py" verify "${platform}" "${package}" >/dev/null +if ! command -v docker >/dev/null 2>&1; then + echo "docker is required to build the local image" >&2 + exit 1 +fi + +context=$(mktemp -d) +trap 'rm -rf "${context}"' EXIT +cp "${repo_dir}/image/seafile_14.0/Dockerfile" "${context}/Dockerfile" +cp "${repo_dir}/image/seafile_14.0/Dockerfile.runtime-base" "${context}/Dockerfile.runtime-base" +cp -a "${repo_dir}/base_scripts" "${context}/base_scripts" +cp -a "${repo_dir}/scripts/scripts_14.0" "${context}/scripts_14.0" +cp -a "${repo_dir}/services" "${context}/services" +cp -a "${package}" "${context}/seafile-server-${version}" +# Validate the copied context too, so a changed source package cannot be labeled +# with the current pins between the original check and context assembly. +copied_digest=$(python3 "${provenance_reader}" "${context}/seafile-server-${version}" "${manifest}") +if [[ "${copied_digest}" != "${package_digest}" ]]; then + echo "Native package changed while preparing the image context" >&2 + exit 1 +fi + +# Reuse v0.1's dependency/application split. BuildKit can reuse the existing +# identical CE14 APT/pip layers for this first extraction too. Refresh is explicit. +legacy_base="cloudfile/runtime-base:${version}-local" +if [[ -z "${CF_PLATFORM:-}" || "${CF_PLATFORM}" == "native" ]]; then + default_base="${legacy_base}" +else + default_base="cloudfile/runtime-base:${version}-${arch}-local" +fi +runtime_base=${CLOUDFILE_RUNTIME_BASE:-${default_base}} +if [[ "${runtime_base}" != "${legacy_base}" && -z "${CLOUDFILE_RUNTIME_BASE:-}" ]] && \ + ! docker image inspect "${runtime_base}" >/dev/null 2>&1 && \ + docker image inspect "${legacy_base}" >/dev/null 2>&1 && \ + [[ "$(docker image inspect --format '{{.Architecture}}' "${legacy_base}")" == "${arch}" ]]; then + docker tag "${legacy_base}" "${runtime_base}" +fi +if [[ "${CLOUDFILE_REFRESH_RUNTIME_BASE:-false}" == "true" ]] || \ + ! docker image inspect "${runtime_base}" >/dev/null 2>&1; then + docker build --platform "${platform}" --pull=false --file "${context}/Dockerfile.runtime-base" \ + --build-arg "server_version=${version}" \ + --label "com.cloudfile.runtime-base.version=${version}" \ + -t "${runtime_base}" "${context}" +fi +base_version=$(docker image inspect --format '{{ index .Config.Labels "com.cloudfile.runtime-base.version" }}' "${runtime_base}") +if [[ "${base_version}" != "${version}" ]]; then + echo "Runtime base does not match CE release ${version}" >&2 + exit 1 +fi +base_arch=$(docker image inspect --format '{{.Architecture}}' "${runtime_base}") +if [[ "${base_arch}" != "${arch}" ]]; then + echo "Runtime base architecture ${base_arch} does not match ${platform}" >&2 + exit 1 +fi +base_id=$(docker image inspect --format '{{.Id}}' "${runtime_base}") +base_pin="cloudfile/runtime-base:${arch}-sha-${base_id#sha256:}" +docker tag "${base_id}" "${base_pin}" +docker build --platform "${platform}" --pull=false \ + --build-arg "CLOUDFILE_RUNTIME_BASE=${base_pin}" \ + --build-arg "server_version=${version}" \ + --label "com.cloudfile.runtime-base.image=${base_id}" \ + --label "org.opencontainers.image.version=${product_version}" \ + --label "com.cloudfile.seafile.version=${version}" \ + --label "com.cloudfile.source.seafile-server=${server_ref}" \ + --label "com.cloudfile.source.seahub=${hub_ref}" \ + --label "com.cloudfile.package.sha256=${package_digest}" \ + --label "com.cloudfile.image.platform=${platform}" \ + -t "${tag}" \ + "${context}" + +echo "Built ${tag}" diff --git a/docs/history/storage-build-20261020/native-build/seafile-build.py b/docs/history/storage-build-20261020/native-build/seafile-build.py new file mode 100755 index 00000000..c6bfc367 --- /dev/null +++ b/docs/history/storage-build-20261020/native-build/seafile-build.py @@ -0,0 +1,744 @@ +#!/usr/bin/env python3 +# coding: UTF-8 + +import sys + +#################### +# Requires Python 3.6+ +#################### +if sys.version_info[0] == 2: + print('Python 2 not supported yet. Quit now.') + sys.exit(1) +if sys.version_info[1] < 6: + print('Python 3.6 or above is required. Quit now.') + sys.exit(1) + +import os +import glob +import subprocess +import tempfile +import shutil +import re +import subprocess +import optparse + +from package_provenance import capture_sources, write_provenance + +#################### +# Global variables +#################### + +# command line configuration +conf = {} + +# key names in the conf dictionary. +CONF_VERSION = 'version' +CONF_SRCDIR = 'srcdir' +CONF_BUILDDIR = 'builddir' +CONF_THIRDPARTDIR = 'thirdpartdir' +CONF_JOBS = 'jobs' +CONF_MYSQL_CONFIG = 'mysql_config' + +#################### +# Common helper functions +#################### +def highlight(content, is_error=False): + '''Add ANSI color to content to get it highlighted on terminal''' + if is_error: + return '\x1b[1;31m%s\x1b[m' % content + else: + return '\x1b[1;32m%s\x1b[m' % content + + +def info(msg): + print(highlight('[INFO] ') + msg) + + +def error(msg=None, usage=None): + if msg: + print(highlight('[ERROR] ') + msg) + if usage: + print(usage) + sys.exit(1) + + +def run_argv(argv, + cwd=None, + env=None, + suppress_stdout=False, + suppress_stderr=False): + '''Run a program and wait it to finish, and return its exit code. The + standard output of this program is suppressed. + + ''' + with open(os.devnull, 'w') as devnull: + if suppress_stdout: + stdout = devnull + else: + stdout = sys.stdout + + if suppress_stderr: + stderr = devnull + else: + stderr = sys.stderr + + proc = subprocess.Popen(argv, + cwd=cwd, + stdout=stdout, + stderr=stderr, + env=env) + return proc.wait() + + +def run(cmdline, + cwd=None, + env=None, + suppress_stdout=False, + suppress_stderr=False): + '''Like run_argv but specify a command line string instead of argv''' + with open(os.devnull, 'w') as devnull: + if suppress_stdout: + stdout = devnull + else: + stdout = sys.stdout + + if suppress_stderr: + stderr = devnull + else: + stderr = sys.stderr + + proc = subprocess.Popen(cmdline, + cwd=cwd, + stdout=stdout, + stderr=stderr, + env=env, + shell=True) + return proc.wait() + + +def must_mkdir(path): + '''Create a directory, exit on failure''' + if os.path.exists(path): + return + + try: + os.makedirs(path) + except OSError as e: + error('failed to create directory %s:%s' % (path, e)) + + +def must_copy(src, dst): + '''Copy src to dst, exit on failure''' + try: + shutil.copy(src, dst) + except Exception as e: + error('failed to copy %s to %s: %s' % (src, dst, e)) + + +def must_copytree(src, dst, with_hidden=False): + '''must_copytree(a, b) copies every file/dir under a/ to b/''' + pattern = os.path.join(src, '*') + try: + for path in glob.glob(pattern): + target_path = os.path.join(dst, os.path.basename(path)) + if os.path.isdir(path): + shutil.copytree(path, target_path, ignore=shutil.ignore_patterns( + '.git', '__pycache__', '*.pyc')) + else: + shutil.copy(path, target_path) + except Exception as e: + error('failed to copy seahub thirdpart libs: %s' % e) + + if with_hidden: + # for hidden dir: .libs_pylibmc + hidden_pattern = os.path.join(src, '.*') + try: + for path in glob.glob(hidden_pattern): + target_path = os.path.join(dst, os.path.basename(path)) + if os.path.isdir(path): + shutil.copytree(path, target_path, ignore=shutil.ignore_patterns( + '.git', '__pycache__', '*.pyc')) + else: + shutil.copy(path, target_path) + except Exception as e: + error('failed to copy seahub thirdpart hidden libs: %s' % e) + + +class Project(object): + '''Base class for a project''' + # Project name, i.e. libseaprc/seafile/seahub + name = '' + + # A list of shell commands to configure/build the project + build_commands = [] + + def __init__(self): + # the path to pass to --prefix=/ + self.prefix = os.path.join(conf[CONF_BUILDDIR], 'seafile-server', + 'seafile') + self.projdir = os.path.join(conf[CONF_SRCDIR], '%s' % self.name) + + def build(self): + '''Build the source''' + info('Building %s' % self.name) + for cmd in self.build_commands: + if run(cmd, cwd=self.projdir) != 0: + error('error when running command:\n\t%s\n' % cmd) + + +class Libsearpc(Project): + name = 'libsearpc' + + def __init__(self): + Project.__init__(self) + self.build_commands = [ + './autogen.sh', + './configure --prefix=%s' % self.prefix, + 'make -j%s' % conf[CONF_JOBS], 'make install' + ] + + +class Libevhtp(Project): + name = 'libevhtp' + + def __init__(self): + Project.__init__(self) + self.build_commands = [ + 'cmake -DEVHTP_DISABLE_SSL=ON -DEVHTP_BUILD_SHARED=OFF .', + 'make', + 'make install', + 'ldconfig' + ] + + +class Seafile(Project): + name = 'seafile-server' + + def __init__(self): + Project.__init__(self) + + configure_command = './configure --prefix=%s' % self.prefix + if conf[CONF_MYSQL_CONFIG]: + configure_command += ' --with-mysql=%s' % conf[CONF_MYSQL_CONFIG] + self.build_commands = [ + './autogen.sh', + configure_command, + 'make -j%s' % conf[CONF_JOBS], + 'make install', + + 'cd fileserver && go build && cd -', + + 'cd notification-server && go build && cd -', + ] + + +class Seahub(Project): + name = 'seahub' + + def __init__(self): + Project.__init__(self) + # nothing to do for seahub + self.build_commands = [] + + def build(self): + self.write_version_to_settings_py() + + Project.build(self) + + def write_version_to_settings_py(self): + '''Write the version of current seafile server to seahub''' + settings_py = os.path.join(self.projdir, 'seahub', 'settings.py') + + line = '\nSEAFILE_VERSION = "%s"\n' % conf[CONF_VERSION] + with open(settings_py, 'a+') as fp: + fp.write(line) + + +def validate_args(usage, options): + required_args = [ + CONF_VERSION, + CONF_SRCDIR, + CONF_THIRDPARTDIR, + ] + + # fist check required args + for optname in required_args: + if getattr(options, optname, None) == None: + error('%s must be specified' % optname, usage=usage) + + def get_option(optname): + return getattr(options, optname) + + # [ version ] + def check_project_version(version): + '''A valid version must be like 1.2.2, 1.3''' + if not re.match('^([0-9])+(\.([0-9])+)+$', version): + error('%s is not a valid version' % version, usage=usage) + + version = get_option(CONF_VERSION) + check_project_version(version) + + # [ srcdir ] + srcdir = get_option(CONF_SRCDIR) + + # [ builddir ] + builddir = get_option(CONF_BUILDDIR) + if not os.path.exists(builddir): + error('%s does not exist' % builddir, usage=usage) + + # [ thirdpartdir ] + thirdpartdir = get_option(CONF_THIRDPARTDIR) + + # [ JOBS ] + jobs = get_option(CONF_JOBS) + + mysql_config_path = get_option(CONF_MYSQL_CONFIG) + + conf[CONF_VERSION] = version + + conf[CONF_BUILDDIR] = builddir + conf[CONF_SRCDIR] = srcdir + conf[CONF_THIRDPARTDIR] = thirdpartdir + conf[CONF_JOBS] = jobs + conf[CONF_MYSQL_CONFIG] = mysql_config_path + + show_build_info() + + prepare_builddir(builddir) + + +def show_build_info(): + '''Print all conf information. Confirm before continue.''' + info('------------------------------------------') + info('Seafile Server Professional %s: BUILD INFO' % conf[CONF_VERSION]) + info('------------------------------------------') + info('builddir: %s' % conf[CONF_BUILDDIR]) + info('source dir: %s' % conf[CONF_SRCDIR]) + info('thirdpart dir: %s' % conf[CONF_THIRDPARTDIR]) + info('jobs: %s' % conf[CONF_JOBS]) + info('------------------------------------------') + + +def prepare_builddir(builddir): + must_mkdir(builddir) + + versioned = os.path.join(builddir, 'seafile-server-' + conf[CONF_VERSION]) + staging = os.path.join(builddir, 'seafile-server') + if os.path.lexists(versioned) or os.path.lexists(staging): + error('choose a fresh build directory; existing packages must not be overlaid') + + os.chdir(builddir) + + must_mkdir(os.path.join(builddir, 'seafile-server')) + must_mkdir(os.path.join(builddir, 'seafile-server', 'seafile')) + + +def parse_args(): + parser = optparse.OptionParser() + + def long_opt(opt): + return '--' + opt + + parser.add_option( + long_opt(CONF_THIRDPARTDIR), + dest=CONF_THIRDPARTDIR, + nargs=1, + help='where to find the thirdpart libs for seahub') + + parser.add_option( + long_opt(CONF_VERSION), + dest=CONF_VERSION, + nargs=1, + help='the version to build. Must be digits delimited by dots, like 1.3.0') + + parser.add_option( + long_opt(CONF_BUILDDIR), + dest=CONF_BUILDDIR, + nargs=1, + help='the directory to build the source. Defaults to /tmp', + default=tempfile.gettempdir()) + + parser.add_option( + long_opt(CONF_SRCDIR), + dest=CONF_SRCDIR, + nargs=1, + help='''Source code must be placed in this directory.''') + + parser.add_option(long_opt(CONF_JOBS), dest=CONF_JOBS, default=2, type=int) + + parser.add_option(long_opt(CONF_MYSQL_CONFIG), + dest=CONF_MYSQL_CONFIG, + nargs=1, + help='''Absolute path to mysql_config or mariadb_config program.''') + + usage = parser.format_help() + options, remain = parser.parse_args() + if remain: + error(usage=usage) + + validate_args(usage, options) + + +def setup_build_env(): + '''Setup environment variables, such as export PATH=$BUILDDDIR/bin:$PATH''' + prefix = os.path.join(conf[CONF_BUILDDIR], 'seafile-server', 'seafile') + + def prepend_env_value(name, value, seperator=':'): + '''append a new value to a list''' + try: + current_value = os.environ[name] + except KeyError: + current_value = '' + + new_value = value + if current_value: + new_value += seperator + current_value + + os.environ[name] = new_value + + prepend_env_value('CPPFLAGS', + '-I%s' % os.path.join(prefix, 'include'), + seperator=' ') + + prepend_env_value('LDFLAGS', + '-L%s' % os.path.join(prefix, 'lib'), + seperator=' ') + + prepend_env_value('LDFLAGS', + '-L%s' % os.path.join(prefix, 'lib64'), + seperator=' ') + + prepend_env_value('PATH', os.path.join(prefix, 'bin')) + prepend_env_value('PKG_CONFIG_PATH', os.path.join(prefix, 'lib', + 'pkgconfig')) + prepend_env_value('PKG_CONFIG_PATH', os.path.join(prefix, 'lib64', + 'pkgconfig')) + + +def copy_pro_libs(): + '''Copy pro.py and python libs for Seafile Professional to + seafile-server/pro/python + + ''' + builddir = conf[CONF_BUILDDIR] + pro_program_dir = os.path.join(builddir, 'seafile-server', 'pro') + if not os.path.exists(pro_program_dir): + must_mkdir(pro_program_dir) + + pro_misc_dir = os.path.join(pro_program_dir, 'misc') + if not os.path.exists(pro_misc_dir): + must_mkdir(pro_misc_dir) + + pro_libs_dir = os.path.join(pro_program_dir, 'python') + must_mkdir(pro_libs_dir) + + pro_py = os.path.join(Seahub().projdir, 'scripts', 'pro.py') + must_copy(pro_py, pro_program_dir) + + copy_seafevents() + + +def copy_seafevents(): + builddir = conf[CONF_BUILDDIR] + pro_libs_dir = os.path.join(builddir, 'seafile-server', 'pro', 'python') + must_mkdir(os.path.join(pro_libs_dir, 'seafevents')) + + events_dir = os.path.join(conf[CONF_SRCDIR], 'seafevents') + + must_copytree(events_dir, os.path.join(pro_libs_dir, 'seafevents')) + + +def copy_seafdav(): + dst_dir = os.path.join(conf[CONF_BUILDDIR], 'seafile-server', 'seahub', + 'thirdpart') + must_mkdir(os.path.join(dst_dir, 'wsgidav')) + must_mkdir(os.path.join(dst_dir, 'seafobj')) + must_mkdir(os.path.join(dst_dir, 'objwrapper')) + + dav_dir = os.path.join(conf[CONF_SRCDIR], 'seafdav', 'wsgidav') + seafobj_dir = os.path.join(conf[CONF_SRCDIR], 'seafobj', 'seafobj') + objwrapper_dir = os.path.join(conf[CONF_SRCDIR], 'seafobj', 'objwrapper') + + must_copytree(dav_dir, os.path.join(dst_dir, 'wsgidav')) + must_copytree(seafobj_dir, os.path.join(dst_dir, 'seafobj')) + must_copytree(objwrapper_dir, os.path.join(dst_dir, 'objwrapper')) + + +def copy_user_manuals(): + builddir = conf[CONF_BUILDDIR] + src_pattern = os.path.join(builddir, Seafile().projdir, 'doc', + 'seafile-tutorial.doc') + dst_dir = os.path.join(builddir, 'seafile-server', 'seafile', 'docs') + + must_mkdir(dst_dir) + + for path in glob.glob(src_pattern): + must_copy(path, dst_dir) + + +def copy_fileserver(): + '''copy fileserver''' + builddir = conf[CONF_BUILDDIR] + dst_dir = os.path.join(builddir, 'seafile-server', 'seafile', 'bin') + fileserver_srcdir = os.path.join(builddir, Seafile().projdir, 'fileserver') + must_copy(os.path.join(fileserver_srcdir, 'fileserver'), dst_dir) + + +def copy_notification_server(): + '''copy notification-server''' + builddir = conf[CONF_BUILDDIR] + dst_dir = os.path.join(builddir, 'seafile-server', 'seafile', 'bin') + notification_server_srcdir = os.path.join(builddir, Seafile().projdir, 'notification-server') + must_copy(os.path.join(notification_server_srcdir, 'notification-server'), dst_dir) + + +def move_python_packages(): + '''move python packages + seafile/lib64/python3.8/site-packages -> seafile/lib64/python3/site-packages + ''' + python_version = 'python' + str(sys.version_info[0]) + '.' + str(sys.version_info[1]) + builddir = conf[CONF_BUILDDIR] + src_parent_dir = os.path.join(builddir, 'seafile-server', 'seafile', 'lib', python_version) + src_dir = os.path.join(src_parent_dir, 'site-packages') + dst_dir = os.path.join(builddir, 'seafile-server', 'seafile', 'lib', 'python3') + src_parent_dir_64 = os.path.join(builddir, 'seafile-server', 'seafile', 'lib64', python_version) + src_dir_64 = os.path.join(src_parent_dir_64, 'site-packages') + dst_dir_64 = os.path.join(builddir, 'seafile-server', 'seafile', 'lib64', 'python3') + if os.path.exists(src_dir): + must_mkdir(dst_dir) + shutil.move(src_dir, dst_dir) + os.removedirs(src_parent_dir) + if os.path.exists(src_dir_64): + must_mkdir(dst_dir_64) + shutil.move(src_dir_64, dst_dir_64) + os.removedirs(src_parent_dir_64) + + +def copy_scripts_and_libs(): + '''Copy server release scripts and shared libs, as well as seahub + thirdpart libs + + ''' + builddir = conf[CONF_BUILDDIR] + scripts_srcdir = os.path.join(builddir, Seahub().projdir, 'scripts') + serverdir = os.path.join(builddir, 'seafile-server') + + must_copy(os.path.join(scripts_srcdir, 'setup-seafile.sh'), serverdir) + must_copy( + os.path.join(scripts_srcdir, 'setup-seafile-mysql.sh'), serverdir) + must_copy( + os.path.join(scripts_srcdir, 'setup-seafile-mysql.py'), serverdir) + must_copy(os.path.join(scripts_srcdir, 'seafile.sh'), serverdir) + must_copy(os.path.join(scripts_srcdir, 'seahub.sh'), serverdir) + must_copy(os.path.join(scripts_srcdir, 'reset-admin.sh'), serverdir) + must_copy(os.path.join(scripts_srcdir, 'seaf-fuse.sh'), serverdir) + must_copy(os.path.join(scripts_srcdir, 'seaf-gc.sh'), serverdir) + must_copy(os.path.join(scripts_srcdir, 'seaf-fsck.sh'), serverdir) + must_copy(os.path.join(scripts_srcdir, 'check_init_admin.py'), serverdir) + must_copy(os.path.join(scripts_srcdir, 'seafile-monitor.sh'), serverdir) + must_copy(os.path.join(scripts_srcdir, 'migrate_ldapusers.py'), serverdir) + + # copy update scripts + update_scriptsdir = os.path.join(scripts_srcdir, 'upgrade') + dst_update_scriptsdir = os.path.join(serverdir, 'upgrade') + try: + shutil.copytree(update_scriptsdir, dst_update_scriptsdir) + except Exception as e: + error('failed to copy upgrade scripts: %s' % e) + + # copy sql scripts + sql_scriptsdir = os.path.join(builddir, Seafile().projdir, 'scripts', 'sql') + dst_sql_scriptsdir = os.path.join(serverdir, 'sql') + try: + shutil.copytree(sql_scriptsdir, dst_sql_scriptsdir) + except Exception as e: + error('failed to copy sql scripts: %s' % e) + + # copy runtime/seahub.conf + runtimedir = os.path.join(serverdir, 'runtime') + must_mkdir(runtimedir) + must_copy(os.path.join(scripts_srcdir, 'seahub.conf'), runtimedir) + + # copy seahub to seafile-server/seahub + src_seahubdir = Seahub().projdir + dst_seahubdir = os.path.join(serverdir, 'seahub') + try: + shutil.copytree(src_seahubdir, dst_seahubdir, ignore=shutil.ignore_patterns( + '.git', '__pycache__', '*.pyc')) + except Exception as e: + error('failed to copy seahub to seafile-server/seahub: %s' % e) + + # copy seahub thirdpart libs + seahub_thirdpart = os.path.join(dst_seahubdir, 'thirdpart') + copy_seahub_thirdpart_libs(seahub_thirdpart) + copy_seafdav() + + # copy pro libs + copy_pro_libs() + + move_python_packages() + + # copy fileserver + copy_fileserver() + # copy notification-server + copy_notification_server() + + # copy shared c libs + copy_cloudfile_acl_core() + copy_shared_libs() + copy_user_manuals() + + +def get_dependent_libs(executable): + syslibs = ['libsearpc', 'libseafile', 'libpthread.so', + 'libc.so', 'libm.so', 'librt.so', 'libdl.so', 'libselinux.so', + 'libresolv.so'] + + def is_syslib(lib): + for syslib in syslibs: + if syslib in lib: + return True + return False + + ldd_output = subprocess.getoutput('ldd %s' % executable) + ret = set() + for line in ldd_output.splitlines(): + tokens = line.split() + if len(tokens) != 4: + continue + if is_syslib(tokens[0]): + continue + ret.add(tokens[2]) + + return ret + + +def copy_cloudfile_acl_core(): + """Package the Server-installed ABI1 core for both native and Hub consumers. + + Do not depend on ldd discovering an unversioned development symlink, or + silently package a runtime that cannot load its policy core. + """ + prefix = os.path.join(conf[CONF_BUILDDIR], 'seafile-server', 'seafile') + candidates = [os.path.join(prefix, directory, 'seafile', 'libcloudfile_acl.so.1') + for directory in ('lib', 'lib64')] + installed = [path for path in candidates if os.path.isfile(path)] + if len(installed) != 1: + error('CloudFile ABI1 permission core must be installed exactly once') + source = installed[0] + destination = os.path.join(prefix, 'lib', 'libcloudfile_acl.so.1') + must_copy(source, destination) + + +def copy_shared_libs(): + '''copy shared c libs, such as libevent, glib, libmysqlclient''' + builddir = conf[CONF_BUILDDIR] + + dst_dir = os.path.join(builddir, 'seafile-server', 'seafile', 'lib') + + seafile_path = os.path.join(builddir, 'seafile-server', 'seafile', 'bin', + 'seaf-server') + + seaf_fuse_path = os.path.join(builddir, 'seafile-server', 'seafile', 'bin', + 'seaf-fuse') + + libs = set() + libs.update(get_dependent_libs(seafile_path)) + libs.update(get_dependent_libs(seaf_fuse_path)) + + for lib in libs: + if lib == 'not': + continue + dst_file = os.path.join(dst_dir, os.path.basename(lib)) + if os.path.exists(dst_file): + continue + info('Copying %s' % lib) + shutil.copy(lib, dst_dir) + + +def copy_seahub_thirdpart_libs(seahub_thirdpart): + '''copy django/djblets/gunicorn from ${thirdpartdir} to + seahub/thirdpart + + ''' + src = conf[CONF_THIRDPARTDIR] + dst = seahub_thirdpart + + must_copytree(src, dst) + + +def strip_symbols(): + def do_strip(fn): + run('chmod u+w %s' % fn) + info('stripping: %s' % fn) + run('strip "%s"' % fn) + + def remove_static_lib(fn): + info('removing: %s' % fn) + os.remove(fn) + + for parent, dnames, fnames in os.walk('seafile-server/seafile'): + dummy = dnames # avoid pylint 'unused' warning + for fname in fnames: + fn = os.path.join(parent, fname) + if os.path.isdir(fn): + continue + + if fn.endswith(".a") or fn.endswith(".la"): + remove_static_lib(fn) + continue + + if os.path.islink(fn): + continue + + finfo = subprocess.getoutput('file "%s"' % fn) + + if 'not stripped' in finfo: + do_strip(fn) + + +def strip_and_rename(): + # strip symbols of libraries to reduce size + try: + strip_symbols() + except Exception as e: + error('failed to strip symbols: %s' % e) + + version = conf[CONF_VERSION] + serverdir = 'seafile-server' + versioned_serverdir = 'seafile-server-' + version + + # move seafile-server to seafile-server-${version} + try: + shutil.move(serverdir, versioned_serverdir) + except Exception as e: + error('failed to move %s to %s: %s' % + (serverdir, versioned_serverdir, e)) + +def main(): + parse_args() + manifest_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'release.json') + sources = capture_sources(manifest_path, conf[CONF_SRCDIR]) + setup_build_env() + + libsearpc = Libsearpc() + libevhtp = Libevhtp() + seafile = Seafile() + seahub = Seahub() + + libsearpc.build() + libevhtp.build() + seafile.build() + seahub.build() + + copy_scripts_and_libs() + strip_and_rename() + package = os.path.join(conf[CONF_BUILDDIR], 'seafile-server-' + conf[CONF_VERSION]) + write_provenance(package, sources, manifest_path) + print('---------------------------------------------') + print('The build is successful; package provenance recorded.') + print('---------------------------------------------') + + +if __name__ == '__main__': + main() diff --git a/docs/history/storage-build-20261020/native-build/seafile-build.sh b/docs/history/storage-build-20261020/native-build/seafile-build.sh new file mode 100755 index 00000000..abd07268 --- /dev/null +++ b/docs/history/storage-build-20261020/native-build/seafile-build.sh @@ -0,0 +1,254 @@ +#!/bin/bash + +# A partial native package is unusable; stop on the first failed clone, +# dependency install, compile, or packaging command instead of printing success. +set -euo pipefail + +if [[ $# -gt 1 ]]; then + echo '' + echo 'Usage: ./seafile-build.sh [seafile-version]' + echo '' + exit 2 +fi + +SCRIPT=$(readlink -f "$0") +current_dir=$(dirname "${SCRIPT}") +code_path=$current_dir/src +manifest=$current_dir/release.json +manifest_reader=$current_dir/source_manifest.py +override_validator=$current_dir/source_override.py +version=$(python3 "${manifest_reader}" "${manifest}" seafile_version) || exit 1 +product_version=$(python3 "${manifest_reader}" "${manifest}" product_version) || exit 1 +package_suffix=${CF_PACKAGE_SUFFIX:-} +case "${package_suffix}" in ''|-arm64|-amd64) ;; *) echo 'Invalid CF_PACKAGE_SUFFIX' >&2; exit 2 ;; esac +if [[ $# -eq 1 && "$1" != "${version}" ]]; then + echo "Requested Seafile version $1 does not match release manifest ${version}" >&2 + exit 2 +fi +mkdir -p "${code_path}" + +function manifest_value() { + python3 "${manifest_reader}" "${manifest}" "$2" "$1" +} + +function source_override() { + # Local overrides are intentionally limited to CloudFile-owned repositories. + # This lets an operator build exact local commits before a human pushes them. + case "$1" in + seafile-server) + echo "${CLOUDFILE_SERVER_SOURCE:-}" + ;; + seahub) + echo "${CLOUDFILE_HUB_SOURCE:-}" + ;; + *) + echo "" + ;; + esac +} + +function install_dependencies() { + if [[ "${CLOUDFILE_BUILD_BASE:-false}" == "true" ]]; then + echo 'Reuse the prebuilt CE14 toolchain; skip APT installation.' + return + fi + apt-get update && apt-get upgrade -y + apt-get install -y build-essential + export DEBIAN_FRONTEND=noninteractive && apt-get install -y tzdata + apt-get install -y \ + ca-certificates \ + cargo \ + cmake \ + git \ + golang-go \ + intltool \ + libarchive-dev \ + libcurl4-openssl-dev \ + libevent-dev \ + libffi-dev \ + libfuse-dev \ + libglib2.0-dev \ + libjansson-dev \ + libjpeg-dev \ + libjwt-dev \ + libldap2-dev \ + libmariadbclient-dev-compat \ + libonig-dev \ + libpq-dev \ + libsqlite3-dev \ + libssl-dev \ + libtool \ + libxml2-dev \ + libxslt-dev \ + python3 \ + python3-distro \ + python3-lxml \ + python3-ldap \ + python3-pip \ + python3-setuptools \ + python3-wheel \ + uuid-dev \ + valac \ + wget \ + curl \ + libunwind-dev \ + libhiredis-dev \ + google-perftools \ + libgoogle-perftools-dev \ + libargon2-dev +} + +function install_python_dependencies() { + cat ${code_path}/seafevents/requirements.txt ${code_path}/seafdav/requirements.txt ${code_path}/seahub/requirements.txt >${code_path}/requirements-thirdpart.txt + cd ${code_path} + + # seafevents ignore + sed -i 's/SQLAlchemy/# SQLAlchemy/' requirements-thirdpart.txt + sed -i 's/mock/# mock/' requirements-thirdpart.txt + sed -i 's/pytest/# pytest/' requirements-thirdpart.txt + sed -i 's/gevent/# gevent/' requirements-thirdpart.txt + sed -i 's/numpy/# numpy/' requirements-thirdpart.txt + sed -i 's/scikit-learn/# scikit-learn/' requirements-thirdpart.txt + + # seafdav ignore + sed -i 's/Jinja2/# Jinja2/' requirements-thirdpart.txt + sed -i 's/sqlalchemy/# sqlalchemy/' requirements-thirdpart.txt + + # seahub ignore + sed -i 's/django_simple_captcha/# django_simple_captcha/' requirements-thirdpart.txt + sed -i 's/^captcha/# captcha/' requirements-thirdpart.txt + sed -i 's/mysqlclient/# mysqlclient/' requirements-thirdpart.txt + sed -i 's/pillow/# pillow/' requirements-thirdpart.txt + sed -i 's/pycryptodome/# pycryptodome/' requirements-thirdpart.txt + sed -i 's/djangosaml2/# djangosaml2/' requirements-thirdpart.txt + sed -i 's/pysaml2/# pysaml2/' requirements-thirdpart.txt + sed -i 's/cffi/# cffi/' requirements-thirdpart.txt + sed -i 's/python-ldap/# python-ldap/' requirements-thirdpart.txt + sed -i 's/PyMuPDF/# PyMuPDF/' requirements-thirdpart.txt + sed -i 's/cairosvg/# cairosvg/' requirements-thirdpart.txt + + # pymysql for scripts + sed -i '$a\pymysql' requirements-thirdpart.txt + # Reuse installed Python dependencies only for this exact requirement set, + # toolchain image and interpreter. A partial/old directory lacks the marker. + dependency_key=$(python3 - <<'PY' +import hashlib, os, platform, sys +from pathlib import Path +value = Path('requirements-thirdpart.txt').read_bytes() +value += repr((sys.version, platform.machine(), os.environ.get('CLOUDFILE_BUILD_BASE_ID', 'unmanaged'))).encode() +print(hashlib.sha256(value).hexdigest()) +PY +) + marker="${code_path}/thirdpartdir/.cloudfile-requirements.sha256" + if [[ "${CF_FORCE_THIRDPART_REFRESH:-false}" != "true" && -f "${marker}" && \ + "$(cat "${marker}")" == "${dependency_key}" ]]; then + echo 'Reuse matching installed Python dependencies.' + return + fi + thirdpart_output=$(mktemp -d "${code_path}/thirdpart-build.XXXXXX") + pip3 install -r requirements-thirdpart.txt -t "${thirdpart_output}" + echo "${dependency_key}" > "${thirdpart_output}/.cloudfile-requirements.sha256" + if [[ -e "${code_path}/thirdpartdir" ]]; then + previous_dependencies=$(mktemp -d "${code_path}/thirdpart-previous.XXXXXX") + mv "${code_path}/thirdpartdir" "${previous_dependencies}/thirdpartdir" + fi + mv "${thirdpart_output}" "${code_path}/thirdpartdir" +} + +function clone_code() { + cd "${code_path}" || exit 1 + + for source in libevhtp libsearpc seafile-server seafobj seafdav seafevents seahub; do + source_url=$(manifest_value "${source}" url) || exit 1 + source_ref=$(manifest_value "${source}" ref) || exit 1 + override=$(source_override "${source}") + if [[ ! -e "${source}" ]]; then + if [[ -n "${override}" ]]; then + validated_override=$(python3 "${override_validator}" "${override}" "${source_ref}") || exit 1 + git clone --no-hardlinks "${validated_override}" "${source}" || exit 1 + else + git clone "${source_url}" "${source}" || exit 1 + fi + elif [[ -d "${source}/.git" ]]; then + if [[ -n "${override}" ]]; then + validated_override=$(python3 "${override_validator}" "${override}" "${source_ref}") || exit 1 + git -C "${source}" remote set-url origin "${validated_override}" || exit 1 + else + git -C "${source}" remote set-url origin "${source_url}" || exit 1 + fi + else + echo "Existing source path is not a Git repository: ${code_path}/${source}" >&2 + exit 1 + fi + done +} + +function fetch() { + cd "${code_path}" || exit 1 + + echo "Start fetch" + echo '' + + for source in libevhtp libsearpc seafile-server seafobj seafdav seafevents seahub; do + echo "Fetch ${source}" + source_ref=$(manifest_value "${source}" ref) || exit 1 + override=$(source_override "${source}") + cd "${code_path}/${source}" || exit 1 + git reset --hard || exit 1 + git clean -xffd || exit 1 + if [[ -n "${override}" ]]; then + validated_override=$(python3 "${override_validator}" "${override}" "${source_ref}") || exit 1 + git remote set-url origin "${validated_override}" || exit 1 + fi + if [[ "$(git rev-parse HEAD)" != "${source_ref}" ]]; then + git fetch --force origin "${source_ref}" || exit 1 + git checkout --detach FETCH_HEAD || exit 1 + fi + cd "${code_path}" || exit 1 + done +} + +function build() { + cd "${current_dir}" + # Never overlay an old completed package or a previous partial build. + # Failed output stays isolated for diagnosis; the last package stays intact. + build_output=$(mktemp -d "${current_dir}/build-output.XXXXXX") + python3 ./seafile-build.py --version="${version}" --builddir="${build_output}" \ + --srcdir="${code_path}" --thirdpartdir="${code_path}/thirdpartdir" \ + --mysql_config=/usr/bin/mariadb_config --jobs="${CF_BUILD_JOBS:-4}" + python3 ./package_provenance.py "${build_output}/seafile-server-${version}" "${manifest}" + package="${current_dir}/seafile-server-${version}${package_suffix}" + if [[ -e "${package}" || -L "${package}" ]]; then + previous=$(mktemp -d "${current_dir}/seafile-server-${version}.previous.XXXXXX") + mv "${package}" "${previous}/package" + fi + if ! mv "${build_output}/seafile-server-${version}" "${package}"; then + if [[ -n "${previous:-}" ]]; then + mv "${previous}/package" "${package}" + fi + exit 1 + fi + rmdir "${build_output}" +} + +echo '' +echo "Info: CloudFile ${product_version}, Seafile ${version}" +echo '' + +install_dependencies +wait + +clone_code +wait + +fetch +wait + +install_python_dependencies +wait + +build + +echo '' +echo "Info: Successfully built ${package}" +echo '' diff --git a/tests/test_build_cache.py b/docs/history/storage-build-20261020/tests/test_build_cache.py similarity index 97% rename from tests/test_build_cache.py rename to docs/history/storage-build-20261020/tests/test_build_cache.py index f65a4a63..78151153 100644 --- a/tests/test_build_cache.py +++ b/docs/history/storage-build-20261020/tests/test_build_cache.py @@ -9,7 +9,7 @@ ROOT = Path(__file__).resolve().parents[1] -SCRIPT = ROOT / 'build/seafile_14.0/seafile-build.sh' +SCRIPT = ROOT / 'docs/history/storage-build-20261020/native-build/seafile-build.sh' FUNCTION = SCRIPT.read_text().split('function install_python_dependencies() {', 1)[1].split( 'function clone_code()', 1)[0] FUNCTION = 'function install_python_dependencies() {' + FUNCTION diff --git a/tools/prepare-v2-source.py b/docs/history/storage-build-20261020/tools/prepare-v2-source.py similarity index 100% rename from tools/prepare-v2-source.py rename to docs/history/storage-build-20261020/tools/prepare-v2-source.py diff --git a/build/v2/Dockerfile.dev b/docs/history/storage-build-20261020/v2/Dockerfile.dev similarity index 100% rename from build/v2/Dockerfile.dev rename to docs/history/storage-build-20261020/v2/Dockerfile.dev diff --git a/build/v2/Dockerfile.dev.dockerignore b/docs/history/storage-build-20261020/v2/Dockerfile.dev.dockerignore similarity index 100% rename from build/v2/Dockerfile.dev.dockerignore rename to docs/history/storage-build-20261020/v2/Dockerfile.dev.dockerignore diff --git a/build/v2/README.md b/docs/history/storage-build-20261020/v2/README.md similarity index 100% rename from build/v2/README.md rename to docs/history/storage-build-20261020/v2/README.md diff --git a/build/v2/build-frontend.cjs b/docs/history/storage-build-20261020/v2/build-frontend.cjs similarity index 100% rename from build/v2/build-frontend.cjs rename to docs/history/storage-build-20261020/v2/build-frontend.cjs diff --git a/build/v2/build-image.sh b/docs/history/storage-build-20261020/v2/build-image.sh similarity index 100% rename from build/v2/build-image.sh rename to docs/history/storage-build-20261020/v2/build-image.sh diff --git a/build/v2/build-native.sh b/docs/history/storage-build-20261020/v2/build-native.sh similarity index 100% rename from build/v2/build-native.sh rename to docs/history/storage-build-20261020/v2/build-native.sh diff --git a/build/v2/native_config.py b/docs/history/storage-build-20261020/v2/native_config.py similarity index 100% rename from build/v2/native_config.py rename to docs/history/storage-build-20261020/v2/native_config.py diff --git a/build/v2/nginx-dev.conf b/docs/history/storage-build-20261020/v2/nginx-dev.conf similarity index 100% rename from build/v2/nginx-dev.conf rename to docs/history/storage-build-20261020/v2/nginx-dev.conf diff --git a/build/v2/source-set.json b/docs/history/storage-build-20261020/v2/source-set.json similarity index 100% rename from build/v2/source-set.json rename to docs/history/storage-build-20261020/v2/source-set.json diff --git a/build/v2/start-dev.py b/docs/history/storage-build-20261020/v2/start-dev.py similarity index 100% rename from build/v2/start-dev.py rename to docs/history/storage-build-20261020/v2/start-dev.py diff --git a/build/v2/web_paths.py b/docs/history/storage-build-20261020/v2/web_paths.py similarity index 100% rename from build/v2/web_paths.py rename to docs/history/storage-build-20261020/v2/web_paths.py diff --git a/.github/workflows/cf-edit-native.yml b/docs/history/storage-build-20261020/workflows/cf-edit-native.yml similarity index 100% rename from .github/workflows/cf-edit-native.yml rename to docs/history/storage-build-20261020/workflows/cf-edit-native.yml diff --git a/.github/workflows/v2-clean.yml b/docs/history/storage-build-20261020/workflows/v2-clean.yml similarity index 100% rename from .github/workflows/v2-clean.yml rename to docs/history/storage-build-20261020/workflows/v2-clean.yml diff --git a/.github/workflows/v2-dev-bundle.yml b/docs/history/storage-build-20261020/workflows/v2-dev-bundle.yml similarity index 100% rename from .github/workflows/v2-dev-bundle.yml rename to docs/history/storage-build-20261020/workflows/v2-dev-bundle.yml diff --git a/docs/storage-build.md b/docs/storage-build.md new file mode 100644 index 00000000..2c49c270 --- /dev/null +++ b/docs/storage-build.md @@ -0,0 +1,43 @@ +# 当前存储构建与验收 + +当前唯一实现为 `release.yaml` + `build/cloudfile_14.0/cloudfile-build.sh`。 +`build-in-docker.sh` 只包装同一实现，使用已有 CE14 Linux 工具链与持久缓存； +`image/cloudfile_14.0/docker-build.sh` 验证同一收据及产物后构建测试镜像，不推送或部署。 + +```sh +bash build/cloudfile_14.0/build-in-docker.sh 14.0.8-storage-test +bash image/cloudfile_14.0/docker-build.sh 14.0.8-storage-test +``` + +构建开始对三仓分支各执行一次远端 `git ls-remote --exit-code`，默认均为 `dev`。 +远端失败即停止，不采用缓存分支或历史锁。随后仅按生成的 40 位 SHA 获取源码， +已缓存的同 SHA 对象与工具链可复用。PR 测试可显式设置 `CF_SERVER_REF`、 +`CF_HUB_REF`、`CF_DOCKER_REF` 为对应短分支；当前 Docker 工作树必须干净且 +恰好等于解析出的 Docker SHA。 + +`build/cloudfile_14.0/build-receipt.json` 包括三仓 SHA、五项 CE 依赖 SHA、 +执行命令/状态/日志 SHA256 和镜像摘要。原 `build/seafile_14.0/release.json` +从同一收据生成，不再手工维护或用其旧 pin 校验当前构建。 +已解析收据可通过 `CF_BUILD_RECEIPT=/absolute/path/build-receipt.json` 供后续阶段消费； +此时不得重新解析浮动分支。收据与编译日志都属于构建产物，不提交 Git。 + +镜像未推送注册表时，`image.digest` 是本地 image config 的 SHA256， +`digest_kind=local-image-config`；`repo_digests` 原样记录已存在的注册表摘要。 +不把本地 image ID 冒称注册表 manifest digest。 + +完整构建在编译前执行 `bash tests/local_storage/run.sh` 与 +`cd fileserver && go test -v ./objstore/... -count=1`；失败写入收据并停止。 +缓存命中也需要本次测试证据，旧产物必须通过原有字节记录才能继续组装。 +`workflow_dispatch` 的 `CloudFile receipt build` 单独运行完整编译和 Linux +特权挂载任务，push/PR 只跑轻量源码检查。 + +Linux 特权任务使用临时挂载命名空间和 `CF_TEST_BIND_MOUNTS=1`，覆盖存储类、 +库、分片三层挂载。数据、Repo ID、原生对象格式与旧存储登记表均不在构建中变更。 + +旧白名单组装及独立 DEV 启动器归档于 +`docs/history/storage-build-20261020/`。旧 feature 分支的配置块位置保留、Web +console 启动文件检查在当前 dev 重新实现；旧 release pin 与平行 native permission +API 不搬入。旧 feature 分支不会自动合并或更名。 + +DEV 真实数据核对与旧库上传、下载、历史、恢复、回收站、GC dry-run、fsck +按用户后续指示在发布后单独验收；构建和隔离测试不能替代该验收。 diff --git a/docs/upstream-patches/cloudfile-server.txt b/docs/upstream-patches/cloudfile-server.txt index a5fde503..733b3a00 100644 --- a/docs/upstream-patches/cloudfile-server.txt +++ b/docs/upstream-patches/cloudfile-server.txt @@ -59,3 +59,4 @@ common/user-mgr.c common/user-mgr.h fileserver/fileserver.go +server/gc/seafile-session.c diff --git a/image/cloudfile_14.0/docker-build.sh b/image/cloudfile_14.0/docker-build.sh index c2a03527..5dab7a49 100755 --- a/image/cloudfile_14.0/docker-build.sh +++ b/image/cloudfile_14.0/docker-build.sh @@ -13,7 +13,7 @@ # # Usage: ./docker-build.sh e.g. 14.0.0-cf.0 -set -e +set -euo pipefail if [[ $# != 1 ]]; then echo '' @@ -26,6 +26,8 @@ version=$1 here=$(cd "$(dirname "$0")" && pwd) repo_root=$(cd "$here/../.." && pwd) +receipt=${CF_BUILD_RECEIPT:-$repo_root/build/cloudfile_14.0/build-receipt.json} +python3 "$repo_root/build/seafile_14.0/build_receipt.py" verify "$receipt" --recipe "$repo_root" >/dev/null source "$repo_root/tools/build-platform.sh" dist=$repo_root/build/cloudfile_14.0/seafile-server-${version} @@ -36,7 +38,7 @@ if [[ ! -d $dist ]]; then fi "$repo_root/tools/verify-release-artifact.sh" "$dist" -package_digest=$(python3 "$repo_root/build/cloudfile_14.0/stamp-current-package.py" "$dist" "$version") +package_digest=$(python3 "$repo_root/build/cloudfile_14.0/stamp-current-package.py" "$dist" "$version" --receipt "$receipt") server_commit=$(sed -n 's/^seafile-server: //p' "$dist/cloudfile-build-info.txt") hub_commit=$(sed -n 's/^seahub: //p' "$dist/cloudfile-build-info.txt") docker_commit=$(sed -n 's/^cloudfile-docker: //p' "$dist/cloudfile-build-info.txt") @@ -119,5 +121,6 @@ else -t "$image" "$context" fi +python3 "$repo_root/build/seafile_14.0/build_receipt.py" image "$receipt" "$image" echo '' echo "Built ${image}" diff --git a/release.yaml b/release.yaml index 214dd0e2..f0c3db9b 100644 --- a/release.yaml +++ b/release.yaml @@ -1,4 +1,4 @@ -# Historical CE14 extension release recipe. V2 dev uses build/v2/source-set.json and the V2 current-bundle CI; do not release V2 images from this manifest. +# Current single CloudFile build entry: full Server/Hub dev trees. # CloudFile release manifest. # # Upstream CE 14 is released now: haiwen/seafile-server and haiwen/seahub both @@ -9,24 +9,17 @@ # no CE tag to point at. # # The build script (build/cloudfile_14.0/cloudfile-build.sh) reads this file and -# checks out exactly these refs. Bump the SHAs here, never in the build script. +# resolves these branch refs once into build-receipt.json before any build. product: 14.0.0-cf.0 image: cloudfile/cloudfile:14.0.0-cf.0 build_base_image: cloudfile-build-base:ce14-v2 ubuntu_base_image: ubuntu:24.04 -# Filled in at release time with the CloudFile commit each image was built from. -server_commit: "" -hub_commit: "" -docker_commit: "" - # CloudFile forks. # -# `dev` is the extension baseline: seams, build and deploy machinery, and no -# capabilities. A capability build points these at its own long-lived branch, -# e.g. ref: feature/dir-acl on both. CF_SERVER_REF / CF_HUB_REF override them -# without editing this file. +# dev is the sole active V2 line. Explicit PR refs may be selected for tests; +# the resulting immutable source lock is generated, never maintained here. forks: cloudfile_server: url: https://github.com/eaopen/cloudfile-server.git @@ -34,6 +27,9 @@ forks: cloudfile_hub: url: https://github.com/eaopen/cloudfile-hub.git ref: dev + cloudfile_docker: + url: https://github.com/eaopen/cloudfile-docker.git + ref: dev # Not forked. These are the upstream commits each fork is currently synced to. # tools/check-upstream-patches.sh uses them as its diff base in CI, where the diff --git a/build/seafile_14.0/release.json b/tests/fixtures/source-lock.json similarity index 57% rename from build/seafile_14.0/release.json rename to tests/fixtures/source-lock.json index 576ef6c5..520f2e56 100644 --- a/build/seafile_14.0/release.json +++ b/tests/fixtures/source-lock.json @@ -1,35 +1,36 @@ { "schema": 1, - "product_version": "v0.2", + "product_version": "v2", "seafile_version": "14.0.8", + "docker_commit": "dddddddddddddddddddddddddddddddddddddddd", "sources": { + "seafile-server": { + "url": "https://github.com/eaopen/cloudfile-server.git", + "ref": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, + "seahub": { + "url": "https://github.com/eaopen/cloudfile-hub.git", + "ref": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, "libevhtp": { "url": "https://github.com/haiwen/libevhtp.git", - "ref": "91071e2f20749cd469b87ac2ef1c158dc2a6806f" + "ref": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, "libsearpc": { "url": "https://github.com/haiwen/libsearpc.git", - "ref": "23d5df62815fb3b7825bf4e1a28b29af701e8cb2" - }, - "seafile-server": { - "url": "https://github.com/eaopen/cloudfile-server.git", - "ref": "38aff59227a20f8ec0c0e4a1f34f571cffe7969f" + "ref": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, "seafobj": { "url": "https://github.com/haiwen/seafobj.git", - "ref": "794428d5f455338f2f2f472c1f0ee2eebf623103" + "ref": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, "seafdav": { "url": "https://github.com/haiwen/seafdav.git", - "ref": "1ce0b05ea7d04495a1ca9121fc58c30d63b71e36" + "ref": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, "seafevents": { "url": "https://github.com/haiwen/seafevents.git", - "ref": "d07b9e885de866454b5c42bed5834ca82959f303" - }, - "seahub": { - "url": "https://github.com/eaopen/cloudfile-hub.git", - "ref": "7ab65e70b11f1acaa0c6df5fe8fe4a94c5eadadc" + "ref": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" } } } diff --git a/tests/test_build_receipt.py b/tests/test_build_receipt.py new file mode 100644 index 00000000..bbae34b0 --- /dev/null +++ b/tests/test_build_receipt.py @@ -0,0 +1,88 @@ +"""Remote resolution, fail-closed source locks and test evidence.""" +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'build/seafile_14.0')) +import build_receipt as receipt +from source_manifest import load_manifest + + +class ReceiptTests(unittest.TestCase): + def resolve(self): + def remote(argv, **kwargs): + return 'a' * 40 + '\t' + argv[-1] + '\n' + with patch.object(receipt.subprocess, 'check_output', side_effect=remote) as call: + result = receipt.resolve(ROOT / 'release.yaml', '14.0.8-storage-test') + self.assertEqual(call.call_count, 3) + self.assertTrue(all(c.args[0][1:3] == ['ls-remote', '--exit-code'] for c in call.call_args_list)) + self.assertEqual(set(result['requested_branches'].values()), {'dev'}) + return result + + def test_resolve_once_and_generate_only_immutable_source_lock(self): + result = self.resolve() + with tempfile.TemporaryDirectory() as directory: + target = Path(directory) / 'build-receipt.json' + receipt.atomic_json(target, result) + loaded = receipt.load_receipt(target) + self.assertEqual(load_manifest(target), loaded['lock']) + self.assertEqual(loaded['lock']['docker_commit'], 'a' * 40) + self.assertEqual(len(loaded['lock']['sources']), 7) + self.assertEqual(loaded['tests'], {}) + self.assertIsNone(loaded['image']) + + def test_remote_failure_never_returns_cached_sha(self): + with patch.object(receipt.subprocess, 'check_output', + side_effect=subprocess.CalledProcessError(128, 'git')): + with self.assertRaises(subprocess.CalledProcessError): + receipt.resolve(ROOT / 'release.yaml', '14.0.8-storage-test') + + def test_ambiguous_missing_or_wrong_ref_is_rejected(self): + for result in ('', 'a' * 40 + '\trefs/heads/other\n', + 'a' * 40 + '\trefs/heads/dev\n' + 'b' * 40 + '\trefs/heads/dev\n'): + with self.subTest(result=result), patch.object(receipt.subprocess, 'check_output', return_value=result): + with self.assertRaises(ValueError): + receipt.resolve(ROOT / 'release.yaml', '14.0.8-storage-test') + + def test_exact_recipe_and_clean_worktree_required(self): + resolved = self.resolve() + for replies in (['b' * 40], ['a' * 40, ' M release.yaml\n']): + with patch.object(receipt.subprocess, 'check_output', side_effect=replies): + with self.assertRaises(ValueError): + receipt.verify_recipe(resolved, ROOT) + with patch.object(receipt.subprocess, 'check_output', side_effect=['a' * 40, '']): + receipt.verify_recipe(resolved, ROOT) + + def test_bad_receipt_cannot_masquerade_as_branch_configuration(self): + result = self.resolve() + result['lock']['sources']['seahub']['ref'] = 'dev' + with tempfile.TemporaryDirectory() as directory: + target = Path(directory) / 'receipt.json' + receipt.atomic_json(target, result) + with self.assertRaisesRegex(ValueError, 'immutable'): + receipt.load_receipt(target) + + def test_test_status_cli_and_log_digest_survive_reloading(self): + with tempfile.TemporaryDirectory() as directory: + target = Path(directory) / 'receipt.json' + receipt.atomic_json(target, self.resolve()) + subprocess.run([sys.executable, str(ROOT / 'build/seafile_14.0/build_receipt.py'), + 'test', str(target), 'local_storage', 'failed', '--command', + 'bash tests/local_storage/run.sh', '--log-sha256', 'b' * 64], + check=True, capture_output=True) + loaded = receipt.load_receipt(target) + self.assertEqual(loaded['tests']['local_storage']['status'], 'failed') + self.assertEqual(loaded['tests']['local_storage']['log_sha256'], 'b' * 64) + + def test_ref_inputs_cannot_select_old_sha_as_default_branch(self): + with self.assertRaises(ValueError): + receipt.resolve(ROOT / 'release.yaml', '14.0.8-storage-test', {'seafile-server': 'a' * 40}) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_current_build_policy.py b/tests/test_current_build_policy.py new file mode 100644 index 00000000..2b9f9fe8 --- /dev/null +++ b/tests/test_current_build_policy.py @@ -0,0 +1,36 @@ +"""Current builds have one implementation and no automatic heavy CI.""" +from pathlib import Path +import unittest + +ROOT = Path(__file__).resolve().parents[1] + + +class CurrentBuildPolicy(unittest.TestCase): + def test_legacy_assembler_and_workflows_are_archived_without_changing_pins(self): + self.assertFalse((ROOT / 'build/v2/source-set.json').exists()) + self.assertFalse((ROOT / 'tools/prepare-v2-source.py').exists()) + history = ROOT / 'docs/history/storage-build-20261020' + self.assertTrue((history / 'v2/source-set.json').is_file()) + for name in ('v2-clean.yml', 'v2-dev-bundle.yml'): + self.assertFalse((ROOT / '.github/workflows' / name).exists()) + self.assertTrue((history / 'workflows' / name).is_file()) + + def test_full_and_privileged_tests_only_run_on_dispatch(self): + workflow = (ROOT / '.github/workflows/prod.yml').read_text() + trigger = workflow.split('permissions:', 1)[0] + self.assertIn('workflow_dispatch:', trigger) + self.assertNotIn(' push:', trigger) + self.assertNotIn(' pull_request:', trigger) + self.assertIn('--privileged', workflow) + self.assertIn('CF_TEST_BIND_MOUNTS=1', workflow) + quick = (ROOT / '.github/workflows/dev.yml').read_text() + self.assertNotIn('docker run', quick) + self.assertNotIn('build-in-docker.sh', quick) + + def test_generated_lock_has_no_hand_maintained_source_shas(self): + ignored = (ROOT / '.gitignore').read_text(errors='replace') + self.assertIn('build/seafile_14.0/release.json', ignored) + builder = (ROOT / 'build/cloudfile_14.0/cloudfile-build.sh').read_text() + self.assertIn('build_receipt.py', builder) + self.assertNotIn('target="origin/${ref}"', builder) + self.assertNotIn("manifest_get_optional 'server_commit'", builder) diff --git a/tests/test_current_package_stamp.py b/tests/test_current_package_stamp.py index 3ccc9f00..6a1028fa 100644 --- a/tests/test_current_package_stamp.py +++ b/tests/test_current_package_stamp.py @@ -1,73 +1,33 @@ +"""Receipt source checks do not consult stale local dev branches.""" import importlib.util +import json from pathlib import Path import tempfile import unittest from unittest.mock import patch - -SCRIPT = Path(__file__).resolve().parents[1] / 'build/cloudfile_14.0/stamp-current-package.py' -SPEC = importlib.util.spec_from_file_location('stamp_current_package', SCRIPT) +ROOT = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location('stamp', ROOT / 'build/cloudfile_14.0/stamp-current-package.py') STAMP = importlib.util.module_from_spec(SPEC) SPEC.loader.exec_module(STAMP) class CurrentPackageSourceTest(unittest.TestCase): - def test_build_record_rejects_duplicate_missing_and_foreign_recipe(self): - commit = 'a' * 40 - entries = [f'{name}: {commit}' for name in STAMP.SOURCE_NAMES] - base = '\n'.join(['product: 14.0.8-cf.0-test', - 'cloudfile-docker: ' + commit, *entries]) + '\n' - with tempfile.TemporaryDirectory() as temporary: - info = Path(temporary) / 'cloudfile-build-info.txt' - with patch.object(STAMP.subprocess, 'check_output', return_value=commit + '\n') as git: - info.write_text(base) - self.assertEqual(STAMP.build_sources(temporary, '14.0.8-cf.0-test'), - dict.fromkeys(STAMP.SOURCE_NAMES, commit)) - git.assert_called_with(['git', 'rev-parse', 'HEAD'], cwd=STAMP.ROOT, text=True) - for invalid in (base + entries[0] + '\n', - base.replace(entries[0] + '\n', ''), - base.replace('cloudfile-docker: ' + commit, - 'cloudfile-docker: ' + 'b' * 40)): - info.write_text(invalid) - with self.assertRaises(ValueError): - STAMP.build_sources(temporary, '14.0.8-cf.0-test') - - - def test_dev_manifest_uses_current_inputs_without_changing_locked_release(self): - locked = STAMP.ROOT / 'build/seafile_14.0/release.json' - before = locked.read_bytes() + def test_duplicate_missing_source_records_are_rejected(self): commit = 'a' * 40 - configured = STAMP.READER.load(STAMP.ROOT / 'release.yaml') - sources = {name: (commit if name in ('seafile-server', 'seahub') - else configured['upstream.' + name]) - for name in STAMP.SOURCE_NAMES} - # Incremental CI only has plan-locked Server/Hub SHAs, not their Git - # directories or upstream checkouts, when stamping the final image. - with patch.dict(STAMP.os.environ, {'CF_SERVER_REF': commit, 'CF_HUB_REF': commit}): - with patch.object(STAMP.subprocess, 'check_output', - side_effect=AssertionError('unnecessary source checkout')): - manifest = STAMP.current_manifest(sources) - self.assertEqual({name: item['ref'] for name, item in manifest['sources'].items()}, sources) - self.assertEqual(locked.read_bytes(), before) - - def test_rejects_mismatched_immutable_ci_plan_and_upstream(self): - config = STAMP.READER.load(STAMP.ROOT / 'release.yaml') - actual = {name: ('a' * 40 if name in ('seafile-server', 'seahub') - else config['upstream.' + name]) - for name in STAMP.SOURCE_NAMES} - with patch.dict(STAMP.os.environ, {'CF_SERVER_REF': 'b' * 40, - 'CF_HUB_REF': 'a' * 40}): - with self.assertRaisesRegex(ValueError, 'release.yaml input: seafile-server'): - STAMP.current_manifest(actual) - altered = dict(actual, libevhtp='c' * 40) - with patch.dict(STAMP.os.environ, {'CF_SERVER_REF': 'a' * 40, - 'CF_HUB_REF': 'a' * 40}): - with self.assertRaisesRegex(ValueError, 'release.yaml input: libevhtp'): - STAMP.current_manifest(altered) - - - def test_dev_manifest_rejects_cached_source_different_from_selected_input(self): - sources = dict.fromkeys(STAMP.SOURCE_NAMES, 'a' * 40) - with patch.object(STAMP.subprocess, 'check_output', return_value='b' * 40 + '\n'): - with self.assertRaisesRegex(ValueError, 'release.yaml input'): - STAMP.current_manifest(sources) + base = '\n'.join(['product: 14.0.8-storage-test', 'cloudfile-docker: ' + commit, + *[name + ': ' + commit for name in STAMP.SOURCE_NAMES]]) + '\n' + with tempfile.TemporaryDirectory() as directory: + info = Path(directory) / 'cloudfile-build-info.txt' + info.write_text(base) + self.assertEqual(STAMP.build_sources(directory, '14.0.8-storage-test')['seahub'], commit) + for invalid in (base + 'seahub: ' + commit + '\n', base.replace('seahub: ' + commit + '\n', '')): + info.write_text(invalid) + with self.assertRaises(ValueError): + STAMP.build_sources(directory, '14.0.8-storage-test') + + def test_moving_local_refs_are_never_used_to_stamp(self): + source = (ROOT / 'build/cloudfile_14.0/stamp-current-package.py').read_text() + self.assertNotIn('rev-parse', source) + self.assertNotIn('current_manifest', source) + self.assertIn('load_receipt', source) diff --git a/tests/test_package_provenance.py b/tests/test_package_provenance.py index 8faa482a..761dba1e 100644 --- a/tests/test_package_provenance.py +++ b/tests/test_package_provenance.py @@ -27,7 +27,7 @@ def setUp(self): self.temporary = tempfile.TemporaryDirectory() self.addCleanup(self.temporary.cleanup) self.root = Path(self.temporary.name) - self.manifest = load_manifest(BUILD / "release.json") + self.manifest = load_manifest(BUILD.parents[1] / "tests/fixtures/source-lock.json") self.manifest_path = self.root / "release.json" self.manifest_path.write_text(json.dumps(self.manifest)) self.package = self.root / "package" @@ -63,7 +63,7 @@ def test_missing_provenance_or_required_binary_cannot_be_labeled(self): def test_changed_manifest_cannot_relabel_old_package(self): self.stamp() - self.manifest["sources"]["seahub"]["ref"] = "a" * 40 + self.manifest["sources"]["seahub"]["ref"] = "b" * 40 self.manifest_path.write_text(json.dumps(self.manifest)) with self.assertRaisesRegex(ValueError, "differs from the release manifest"): verify_package(self.package, self.manifest_path) @@ -129,67 +129,25 @@ def test_source_capture_requires_clean_exact_immutable_checkouts(self): with self.assertRaisesRegex(ValueError, "immutable"): capture_sources(self.manifest_path, sources) - def test_image_assembly_rejects_stale_package_before_docker_is_called(self): - builder = self.root / "build/seafile_14.0" - builder.mkdir(parents=True) - for name in ("build-local-image.sh", "source_manifest.py", "package_provenance.py", - "image_platform.py", "release.json"): - shutil.copyfile(BUILD / name, builder / name) - shutil.copytree(self.package, builder / "seafile-server-14.0.8", symlinks=True) - binaries = self.root / "bin" - binaries.mkdir() - marker = self.root / "docker-called" - docker = binaries / "docker" - docker.write_text('#!/bin/sh\ntouch "' + str(marker) + '"\n') - docker.chmod(0o755) - result = subprocess.run(["bash", str(builder / "build-local-image.sh")], - env={**os.environ, "PATH": str(binaries) + os.pathsep + os.environ["PATH"]}, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) - self.assertNotEqual(result.returncode, 0) - self.assertFalse(marker.exists()) - self.assertIn("Native package validation failed", result.stderr) - - def test_image_assembly_uses_shared_base_and_original_local_tag(self): - native_arch = {"arm64": "arm64", "aarch64": "arm64", - "x86_64": "amd64", "amd64": "amd64"}[platform.machine()] - header = bytearray(20) - header[:6] = b"\x7fELF\x02\x01" - header[18:20] = struct.pack("> "$CF_DOCKER_CALLS" -case "$*" in - *Architecture*) echo ''' + native_arch + ''' ;; - *com.cloudfile.runtime-base.version*) echo 14.0.8 ;; - *'{{.Id}}'*) echo sha256:fixture ;; -esac -''') - docker.chmod(0o755) - environment = {**os.environ, "PATH": str(binary) + os.pathsep + os.environ["PATH"], - "CLOUDFILE_PACKAGE_DIR": str(self.package), "CF_DOCKER_CALLS": str(calls)} - environment.pop("CF_PLATFORM", None) - result = subprocess.run(["bash", str(BUILD / "build-local-image.sh")], - env=environment, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) - self.assertEqual(result.returncode, 0, result.stderr) - recorded = calls.read_text() - self.assertIn("--platform linux/" + native_arch, recorded) - self.assertIn("cloudfile/runtime-base:14.0.8-local", recorded) - self.assertIn("cloudfile/cloudfile:14.0.8-v0.2-local", recorded) - self.assertNotIn("Dockerfile.runtime-base", recorded) - calls.write_text("") - environment["CF_PLATFORM"] = "linux/" + native_arch - result = subprocess.run(["bash", str(BUILD / "build-local-image.sh")], - env=environment, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) - self.assertEqual(result.returncode, 0, result.stderr) - recorded = calls.read_text() - self.assertIn("cloudfile/cloudfile:14.0.8-v0.2-" + native_arch + "-local", recorded) - self.assertIn("cloudfile/runtime-base:14.0.8-" + native_arch + "-local", recorded) + receipt = {"schema": 2, "lock": self.manifest, "tests": {}} + self.manifest_path.write_text(json.dumps(receipt)) + with self.assertRaisesRegex(ValueError, "successful focused test evidence"): + verify_package(self.package, self.manifest_path) + for name in ("local_storage", "objstore"): + receipt["tests"][name] = {"status": "passed", "command": "fixture", "log_sha256": "a" * 64} + self.manifest_path.write_text(json.dumps(receipt)) + verify_package(self.package, self.manifest_path) + receipt["tests"]["objstore"]["status"] = "failed" + self.manifest_path.write_text(json.dumps(receipt)) + with self.assertRaisesRegex(ValueError, "successful focused test evidence"): + verify_package(self.package, self.manifest_path) + + def test_missing_web_console_entry_point_is_rejected(self): + (self.package / "seahub/thirdpart/bin/gunicorn").unlink() + with self.assertRaisesRegex(ValueError, "gunicorn"): + self.stamp() if __name__ == "__main__": diff --git a/tests/test_receipt_cache.py b/tests/test_receipt_cache.py new file mode 100644 index 00000000..5399450b --- /dev/null +++ b/tests/test_receipt_cache.py @@ -0,0 +1,165 @@ +"""Execute production cache/provenance shell functions against disposable packages.""" +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +BUILD = ROOT / "build/seafile_14.0" +SCRIPT = ROOT / "build/cloudfile_14.0/cloudfile-build.sh" +sys.path.insert(0, str(BUILD)) +from package_provenance import (RECORD_NAME, REQUIRED_FILES, capture_sources, + verify_package) + + +def production_function(name): + """Select declarations only, leaving all production commands unchanged.""" + found = re.search(r"^function " + re.escape(name) + r"\(\) \{\n.*?^\}", + SCRIPT.read_text(), re.MULTILINE | re.DOTALL) + if found is None: + raise AssertionError("missing production function: " + name) + return found.group(0) + + +class ReceiptCacheTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.current = self.root / "builder" + self.sources = self.root / "sources" + self.current.mkdir() + self.sources.mkdir() + self.version = "14.0.8-storage-test" + self.package = self.current / ("seafile-server-" + self.version) + self.package.mkdir() + for name in REQUIRED_FILES: + path = self.package / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"installed fixture bytes") + lock = json.loads((ROOT / "tests/fixtures/source-lock.json").read_text()) + for name, source in lock["sources"].items(): + path = self.sources / name + path.mkdir() + subprocess.run(["git", "init", "-q", str(path)], check=True) + subprocess.run(["git", "-C", str(path), "-c", "user.name=Cache Test", + "-c", "user.email=cache-test@example.invalid", "commit", + "--quiet", "--allow-empty", "-m", "fixture source"], check=True) + source["ref"] = subprocess.check_output( + ["git", "-C", str(path), "rev-parse", "HEAD"], text=True).strip() + lock["docker_commit"] = "1" * 40 + self.receipt = self.root / "build-receipt.json" + self.document = {"schema": 2, "version": self.version, "lock": lock, + "image": None, "tests": {}} + for name in ("local_storage", "objstore"): + self.document["tests"][name] = { + "status": "passed", "command": "fixture " + name, + "log_sha256": hashlib.sha256(name.encode()).hexdigest()} + self.save_receipt() + (self.current / "captured-sources.json").write_text( + json.dumps(capture_sources(self.receipt, self.sources))) + for name in ("cloudfile-build.sh", "cloudfile-build.py"): + shutil.copyfile(SCRIPT.parent / name, self.current / name) + + def save_receipt(self): + self.receipt.write_text(json.dumps(self.document)) + + def shell(self, function, jobs="1"): + declarations = "\n".join(production_function(name) for name in + ("source_fingerprint", "dist_fingerprint", "write_build_info")) + return subprocess.run(["bash", "-c", "set -euo pipefail\n" + declarations + + "\n" + function], capture_output=True, text=True, + env={**os.environ, "current_dir": str(self.current), + "repo_root": str(ROOT), "code_path": str(self.sources), + "version": self.version, "receipt": str(self.receipt), + "CF_BUILD_JOBS": jobs}) + + def assert_shell_ok(self, result): + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_matching_pristine_cache_is_verified_then_restamped(self): + first = self.shell("write_build_info") + self.assert_shell_ok(first) + previous = verify_package(self.package, self.receipt)["package_sha256"] + result = self.shell("write_build_info", jobs="2") + self.assert_shell_ok(result) + self.assertIn("jobs: 2\n", (self.package / "cloudfile-build-info.txt").read_text()) + record = verify_package(self.package, self.receipt) + self.assertEqual(record["manifest"], self.document["lock"]) + self.assertNotEqual(record["package_sha256"], previous) + + def test_modified_cache_bytes_are_rejected_before_provenance_is_removed(self): + self.assert_shell_ok(self.shell("write_build_info")) + proof = (self.package / RECORD_NAME).read_bytes() + info = (self.package / "cloudfile-build-info.txt").read_bytes() + (self.package / REQUIRED_FILES[0]).write_bytes(b"tampered cached binary") + result = self.shell("write_build_info", jobs="2") + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual((self.package / RECORD_NAME).read_bytes(), proof) + self.assertEqual((self.package / "cloudfile-build-info.txt").read_bytes(), info) + with self.assertRaisesRegex(ValueError, "content differs"): + verify_package(self.package, self.receipt) + + def test_docker_recipe_change_invalidates_dist_with_identical_sources(self): + sources_before = self.shell("source_fingerprint") + first = self.shell("dist_fingerprint") + repeated = self.shell("dist_fingerprint") + for result in (sources_before, first, repeated): + self.assert_shell_ok(result) + self.assertEqual(first.stdout, repeated.stdout) + self.document["lock"]["docker_commit"] = "2" * 40 + self.save_receipt() + after = self.shell("dist_fingerprint") + self.assert_shell_ok(after) + self.assertEqual(sources_before.stdout, self.shell("source_fingerprint").stdout) + self.assertNotEqual(first.stdout, after.stdout) + self.assertRegex(after.stdout.strip(), r"^[0-9a-f]{64}$") + + def test_real_storage_runner_exports_reachable_shared_fixture(self): + # Keep the actual runner and receipt reader/writer; replace only the + # C/Go test executables at their process boundary, without compiling. + runner_dir = self.root / "recipe/build/cloudfile_14.0" + runner_dir.mkdir(parents=True) + (runner_dir.parent / "seafile_14.0").symlink_to(BUILD, target_is_directory=True) + docs = self.root / "recipe/docs/features" + docs.mkdir(parents=True) + (docs / "automatic-local-storage-cases.json").symlink_to( + ROOT / "docs/features/automatic-local-storage-cases.json") + runner = runner_dir / "run-storage-tests.sh" + shutil.copyfile(SCRIPT.parent / runner.name, runner) + server = self.sources / "seafile-server" + (server / "tests/local_storage").mkdir(parents=True) + (server / "tests/local_storage/run.sh").write_text( + 'test -s "${CF_AUTO_STORAGE_CASES:?shared fixture missing}"\n' + 'printf "C boundary fixture reachable\\n"\n') + (server / "fileserver").mkdir() + binary = self.root / "bin" + binary.mkdir() + go = binary / "go" + go.write_text('#!/bin/bash\nset -euo pipefail\n' + 'test -s "${CF_AUTO_STORAGE_CASES:?shared fixture missing}"\n' + 'test "$1" = test\nprintf "Go boundary fixture reachable\\n"\n') + go.chmod(0o755) + result = subprocess.run(["bash", str(runner), str(self.receipt), str(server)], + capture_output=True, text=True, + env={**os.environ, "PATH": str(binary) + os.pathsep + os.environ["PATH"], + "CF_TEST_BIND_MOUNTS": "0", "CF_AUTO_STORAGE_CASES": "/missing"}) + self.assert_shell_ok(result) + self.assertIn("C boundary fixture reachable", result.stdout) + self.assertIn("Go boundary fixture reachable", result.stdout) + receipt = json.loads(self.receipt.read_text()) + for name in ("local_storage", "objstore"): + self.assertEqual(receipt["tests"][name]["status"], "passed") + log = runner_dir / "test-results" / (name + ".log") + self.assertEqual(receipt["tests"][name]["log_sha256"], + hashlib.sha256(log.read_bytes()).hexdigest()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_source_manifest.py b/tests/test_source_manifest.py index e173b2cc..a1d9d560 100644 --- a/tests/test_source_manifest.py +++ b/tests/test_source_manifest.py @@ -23,21 +23,19 @@ def test_build_script_fails_closed_and_supports_pinned_local_sources(self): script = (BUILD_DIR / "seafile-build.sh").read_text(encoding="utf-8") self.assertIn("set -euo pipefail", script) - self.assertIn("CLOUDFILE_SERVER_SOURCE", script) - self.assertIn("CLOUDFILE_HUB_SOURCE", script) + self.assertIn('cloudfile_14.0/cloudfile-build.sh', script) def test_local_image_builder_is_fail_closed_and_labels_source_pins(self): script = (BUILD_DIR / "build-local-image.sh").read_text(encoding="utf-8") self.assertIn("set -euo pipefail", script) - self.assertIn("com.cloudfile.source.seafile-server", script) - self.assertIn("com.cloudfile.source.seahub", script) + self.assertIn('image/cloudfile_14.0/docker-build.sh', script) self.assertNotIn("docker push", script) def test_release_manifest_is_valid_and_pins_cloudfile_sources(self): - manifest = SOURCE_MANIFEST.load_manifest(BUILD_DIR / "release.json") + manifest = SOURCE_MANIFEST.load_manifest(BUILD_DIR.parents[1] / "tests/fixtures/source-lock.json") - self.assertEqual(manifest["product_version"], "v0.2") + self.assertEqual(manifest["product_version"], "v2") self.assertEqual(manifest["seafile_version"], "14.0.8") self.assertEqual( manifest["sources"]["seahub"]["url"], @@ -50,7 +48,7 @@ def test_release_manifest_is_valid_and_pins_cloudfile_sources(self): ) def test_incomplete_source_set_is_rejected(self): - manifest = json.loads((BUILD_DIR / "release.json").read_text(encoding="utf-8")) + manifest = json.loads((BUILD_DIR.parents[1] / "tests/fixtures/source-lock.json").read_text(encoding="utf-8")) del manifest["sources"]["seahub"] with tempfile.TemporaryDirectory() as directory: diff --git a/tests/test_v2_build_policy.py b/tests/test_v2_build_policy.py deleted file mode 100644 index b1a66b74..00000000 --- a/tests/test_v2_build_policy.py +++ /dev/null @@ -1,61 +0,0 @@ -"""Cheap source and CI policy regressions (no Docker, compiler or network).""" -import unittest -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] - - -def read(path): - return (ROOT / path).read_text(encoding="utf-8") - - -class DevSourcePolicy(unittest.TestCase): - def test_current_dev_is_default_but_historical_snapshot_is_explicit(self): - assembler = read("tools/prepare-v2-source.py") - self.assertIn("if args.source_set else 'dev'", assembler) - self.assertIn("sources['components'][component]['overlay_commit'] = commit", assembler) - self.assertIn("go_seam = 'fileserver/objstore/backend_fs.go'", assembler) - self.assertIn("native_patches.append(go_seam)", assembler) - frozen = read(".github/workflows/v2-clean.yml") - self.assertEqual( - frozen.count("--source-set cloudfile-docker/build/v2/source-set.json"), 2 - ) - - def test_hub_only_revision_does_not_invalidate_native_compiler_layer(self): - dockerfile = read("build/v2/Dockerfile.dev") - server_layer = dockerfile.split("FROM go-dependencies AS native", 1)[1].split( - "FROM ${CF_BASE_IMAGE} AS frontend-dependencies", 1 - )[0] - self.assertNotIn("COPY v2-source-receipt.json", server_layer) - self.assertIn("COPY seafile-server /source/seafile-server", server_layer) - self.assertIn( - "COPY v2-source-receipt.json /opt/cloudfile-v2/source/v2-source-receipt.json", - dockerfile, - ) - preflight = read("build/v2/build-image.sh") - self.assertLess( - preflight.index("Prepared source receipt verified"), - preflight.index('docker tag "$build_id"'), - ) - - def test_current_bundle_includes_the_go_storage_seam(self): - current = read(".github/workflows/v2-dev-bundle.yml") - self.assertIn("native.append('fileserver/objstore/backend_fs.go')", current) - - def test_native_and_release_workflows_never_auto_run(self): - paths = ( - ".github/workflows/v2-clean.yml", - ".github/workflows/v2-dev-bundle.yml", - ".github/workflows/cf-edit-native.yml", - ".github/workflows/prod.yml", - ) - for path in paths: - with self.subTest(workflow=path): - trigger = read(path).split("permissions:", 1)[0] - self.assertIn("workflow_dispatch:", trigger) - self.assertNotIn("\n push:", trigger) - self.assertNotIn("\n pull_request:", trigger) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_v2_source_assembly.py b/tests/test_v2_source_assembly.py deleted file mode 100644 index 1877dfc4..00000000 --- a/tests/test_v2_source_assembly.py +++ /dev/null @@ -1,99 +0,0 @@ -import importlib.util -from pathlib import Path -import subprocess -import tempfile -import unittest - - -spec = importlib.util.spec_from_file_location('v2_sources', Path(__file__).resolve().parents[1] / 'tools/prepare-v2-source.py') -module = importlib.util.module_from_spec(spec) -spec.loader.exec_module(module) - - -class SourceAssemblyTest(unittest.TestCase): - def setUp(self): - self.directory = tempfile.TemporaryDirectory() - self.root = Path(self.directory.name) - self.config = {'format': 1, 'ce_version': '14.0.8', 'components': {}} - self.refs = {} - for name, target, overlay in (('cloudfile-server', 'seafile-server', 'scripts/sql/mysql/cloudfile-v2.sql'), - ('cloudfile-hub', 'seahub', 'cloudfile_ext/v2')): - repo = self.root / name - repo.mkdir() - self.git(repo, 'init', '-q') - self.git(repo, 'config', 'user.name', 'V2 Source Test') - self.git(repo, 'config', 'user.email', 'test@example.invalid') - (repo / 'native.txt').write_text('fixed upstream file') - self.git(repo, 'add', '.') - self.git(repo, 'commit', '-qm', 'native fixture') - base = self.git(repo, 'rev-parse', 'HEAD').strip() - # Pollute the fork HEAD and prove the assembled source excludes it. - (repo / 'native.txt').write_text('unreviewed fork modification') - (repo / 'cloudfile_extensions').mkdir() - (repo / 'cloudfile_extensions/legacy.py').write_text('old_framework = True') - extension = repo / (overlay if overlay.endswith('.sql') else overlay + '/__init__.py') - extension.parent.mkdir(parents=True, exist_ok=True) - extension.write_text('-- reviewed DDL' if extension.suffix == '.sql' else '"""Isolated V2 adapter."""\n') - self.git(repo, 'add', '.') - self.git(repo, 'commit', '-qm', 'fork fixture with unrelated legacy code') - self.config['components'][name] = {'directory': target, 'native_commit': base, 'overlay': [overlay]} - self.refs[name] = 'HEAD' - - def tearDown(self): - self.directory.cleanup() - - @staticmethod - def git(repo, *args): - return subprocess.check_output(['git', '-C', str(repo), *args], text=True) - - def test_head_history_and_legacy_files_cannot_enter_prepared_source(self): - output = self.root / 'prepared' - receipt = module.assemble(self.root, output, self.config, self.refs) - for name, config in self.config['components'].items(): - target = output / config['directory'] - self.assertEqual((target / 'native.txt').read_text(), 'fixed upstream file') - self.assertFalse((target / 'cloudfile_extensions').exists()) - self.assertEqual(receipt['components'][name]['native_commit'], config['native_commit']) - self.assertTrue(receipt['components'][name]['overlay_sha256']) - with self.assertRaises(ValueError): - module.assemble(self.root, output, self.config, self.refs) - - def test_overlay_cannot_replace_a_native_file(self): - self.config['components']['cloudfile-hub']['overlay'] = ['native.txt'] - output = self.root / 'failed' - with self.assertRaises(ValueError): - module.assemble(self.root, output, self.config, self.refs) - self.assertFalse(output.exists()) - - def test_explicit_native_patch_has_before_and_after_receipt(self): - self.config['components']['cloudfile-server']['native_patches'] = ['native.txt'] - output = self.root / 'patched' - receipt = module.assemble(self.root, output, self.config, self.refs) - self.assertEqual((output / 'seafile-server/native.txt').read_text(), 'unreviewed fork modification') - server = receipt['components']['cloudfile-server'] - self.assertNotEqual(server['native_before_sha256']['native.txt'], server['overlay_sha256']['native.txt']) - self.assertEqual((output / 'seahub/native.txt').read_text(), 'fixed upstream file') - self.assertFalse((output / 'seafile-server/cloudfile_extensions').exists()) - - def test_native_patch_cannot_name_directory_or_new_module(self): - for file in ('cloudfile_extensions', 'native.txt/..', 'missing.txt', '/'): - with self.subTest(file=file): - self.config['components']['cloudfile-server']['native_patches'] = [file] - with self.assertRaises(ValueError): - module.assemble(self.root, self.root / 'failed', self.config, self.refs) - self.assertFalse((self.root / 'failed').exists()) - - def test_absolute_legacy_import_is_rejected(self): - self._bad_import('from cloudfile_extensions.jobs.store import JobStore\n') - - def test_relative_import_cannot_escape_isolated_package(self): - self._bad_import('from .. import registry\n') - - def _bad_import(self, source): - repo = self.root / 'cloudfile-hub' - (repo / 'cloudfile_ext/v2/__init__.py').write_text(source) - self.git(repo, 'add', '.') - self.git(repo, 'commit', '-qm', 'bad dependency fixture') - with self.assertRaises(ValueError): - module.assemble(self.root, self.root / 'failed', self.config, self.refs) - self.assertFalse((self.root / 'failed').exists()) diff --git a/tests/tools/test-build-cache-contract.sh b/tests/tools/test-build-cache-contract.sh index d3aef1dd..6472ab6f 100755 --- a/tests/tools/test-build-cache-contract.sh +++ b/tests/tools/test-build-cache-contract.sh @@ -30,7 +30,7 @@ require_text "$container_script" 'all|backend|frontend|package' require_text "$build_script" 'function export_backend_artifact()' require_text "$build_script" 'function export_frontend_artifact()' require_text "$build_script" 'CF_BUILD_TARGET == package' -require_text "$build_script" "manifest_get_optional 'server_commit'" +require_text "$build_script" 'build_receipt.py' fingerprint_body=$(sed -n \ '/^function frontend_fingerprint()/,/^}/p' "$build_script")