diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6113e0590..48413c374 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1824,7 +1824,7 @@ jobs: # (operator does zero logins and is wait-dominated, so it runs # wide; the api buckets stay within the login limiter's budget). # The arm64 job below uses the same steps and per-bucket settings. - bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, restricted-workloads, management-placement] + bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, restricted-workloads, management-placement, backup-preparation] arch: [amd64] include: - bucket: operator @@ -1835,6 +1835,10 @@ jobs: parallel: 1 test_timeout: 15m job_timeout: 60 + - bucket: backup-preparation + parallel: 1 + test_timeout: 15m + job_timeout: 60 - bucket: api-auth parallel: 4 test_timeout: 15m @@ -1896,7 +1900,7 @@ jobs: - name: bring up cluster + helm install if: env.RUN_REGEX != '' || env.TAIL_REGEX != '' env: - GAMEPLANE_E2E_PROFILE: ${{ matrix.bucket == 'management-placement' && 'management' || 'default' }} + GAMEPLANE_E2E_PROFILE: ${{ (matrix.bucket == 'management-placement' || matrix.bucket == 'backup-preparation') && 'management' || 'default' }} run: ./deploy/kind/e2e.sh up "$CLUSTER" e2e "$GAMEPLANE_E2E_PROFILE" - name: run go e2e (${{ matrix.bucket }}) if: env.RUN_REGEX != '' @@ -2292,6 +2296,16 @@ jobs: install_only: true - name: load images uses: ./.github/actions/e2e-images + - name: free runner disk for Garry's Mod image + if: matrix.test == 'TestGameServer_GarrysModBot_Query' + # The pre-baked game image exhausts the standard runner while containerd + # unpacks it. These unused SDKs and the already-loaded archive share + # the node's host filesystem. Keep the real game/protocol test intact. + run: | + df -h / + sudo rm -rf -- /usr/local/lib/android /usr/share/dotnet /opt/ghc + rm -f -- "$RUNNER_TEMP/e2e-images/e2e-images.tar" + df -h / - name: bring up cluster + helm install run: ./deploy/kind/e2e.sh up "$CLUSTER" e2e - name: run go e2e (game bot) @@ -2714,7 +2728,7 @@ jobs: 'go-e2e-unit': (n) => n === 'go (test/e2e)', 'api-postgres': (n) => n === 'api (postgres)', 'e2e-buckets': (n) => n === 'e2e bucket coverage', - 'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|restricted-workloads|management-placement) \//.test(n) && n.includes('/ amd64 '), + 'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|restricted-workloads|management-placement|backup-preparation) \//.test(n) && n.includes('/ amd64 '), 'e2e-go-arm64': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|restricted-workloads) \//.test(n) && n.includes('/ arm64 '), 'e2e-multicluster': (n) => n.startsWith('e2e multicluster /') && n.includes('/ amd64 '), 'e2e-multicluster-arm64': (n) => n.startsWith('e2e multicluster /') && n.includes('/ arm64 '), @@ -2867,7 +2881,7 @@ jobs: const bucketSet = new Set(); for (const j of apiJobs) { const n = j.name || ''; - const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|restricted-workloads|management-placement) \//); + const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|restricted-workloads|management-placement|backup-preparation) \//); if (m) bucketSet.add(m[1]); if (/^e2e multicluster \//.test(n)) bucketSet.add('multicluster'); if (/^e2e upgrade \//.test(n)) bucketSet.add('upgrade'); diff --git a/charts/gameplane/templates/operator.yaml b/charts/gameplane/templates/operator.yaml index 15df3da9c..75ccb20fa 100644 --- a/charts/gameplane/templates/operator.yaml +++ b/charts/gameplane/templates/operator.yaml @@ -83,6 +83,11 @@ rules: - apiGroups: ["snapshot.storage.k8s.io"] resources: [volumesnapshots, volumesnapshotcontents] verbs: [get, list, watch] + # Abort reclamation is restricted in the controller to recorded snapshot + # references and exact content UIDs; VolumeSnapshotContent is cluster-scoped. + - apiGroups: [snapshot.storage.k8s.io] + resources: [volumesnapshotcontents] + verbs: [delete] # The GameServer controller registers Owns(&networkingv1.NetworkPolicy{}) # unconditionally (regardless of networkPolicies.enabled), and the # manager's cache is cluster-scoped (no Cache/DefaultNamespaces option @@ -326,6 +331,18 @@ spec: {{- if .Values.operator.resticImage }} - --restic-image={{ .Values.operator.resticImage }} {{- end }} + {{- with .Values.operator.backupPreparationTimeout }} + - --backup-preparation-timeout={{ . }} + {{- end }} + {{- with .Values.operator.backupQuiesceTimeout }} + - --backup-quiesce-timeout={{ . }} + {{- end }} + {{- with .Values.operator.backupCSICaptureTimeout }} + - --backup-csi-capture-timeout={{ . }} + {{- end }} + {{- if and (hasKey .Values.operator "backupHardSameNode") (ne .Values.operator.backupHardSameNode nil) }} + - --backup-hard-same-node={{ .Values.operator.backupHardSameNode }} + {{- end }} {{- /* Omit the flags when the keys are absent/null (e.g. --reuse-values from a release predating them) so the operator's defaults (2 / 86400) apply. Not `| default`: an explicit 0 backoff is diff --git a/charts/gameplane/values.yaml b/charts/gameplane/values.yaml index 54d297827..d660889ac 100644 --- a/charts/gameplane/values.yaml +++ b/charts/gameplane/values.yaml @@ -134,6 +134,13 @@ operator: # pod cannot hold a Backup/Restore in flight forever. Defaults: 2 and 86400 (24h). backupJobBackoffLimit: 2 backupJobActiveDeadlineSeconds: 86400 + # Preparation (scheduling, image pull, PVC mount, repository/lock probe) + # precedes save-off. Quiesce expiry cancels and drains the copy before save-on. + backupPreparationTimeout: 10m + backupQuiesceTimeout: 5m + backupCSICaptureTimeout: 10m + # Prefer the observed game's node by default; opt in to a hard requirement. + backupHardSameNode: false # Mount a directory of module bundles into the operator so local-type # ModuleSources can index it (homelab: drop module dirs on disk and # they appear in the catalog). Provide exactly one of existingClaim diff --git a/docs/operator-operations.md b/docs/operator-operations.md index fb227e1dd..f24810524 100644 --- a/docs/operator-operations.md +++ b/docs/operator-operations.md @@ -13,8 +13,8 @@ already stopped and its writers drained before capture; the backup did not stop it. Failed or unfinished copies, including completed Jobs still awaiting a snapshot ID, cannot report achieved quiesce or stopped-source consistency. -`status.captureManifest` is recorded once before the first copy worker is -created, after lifecycle ownership and quiesce preparation. It lists the primary +`status.captureManifest` is recorded once before copy authorization or CSI +capture, after lifecycle ownership and quiesce preparation. It lists the primary `-data` claim as included and every other observed persistent claim as uncovered, including extra volumes, nested mod/loader mounts and retained server-owned claims. It records claim UIDs, observed mount paths, source layout @@ -42,12 +42,13 @@ including backups with `quiesce: false` and CSI VolumeSnapshots. A held operatio also defers idle sleep/wake, restart, and pod-template rollout. Requests remain pending and reconciliation resumes after the operation's cleanup completes. -For a quiesced backup of a running workload, an initially absent, non-Running or -terminating source Pod gets a two-minute startup grace period from the Backup's -creation time. If it is still unavailable afterward, the Backup fails and runs -normal cleanup to release its lifecycle owner and workload fence. A source that -is already Running can be admitted after that window; prepared source identity -and outstanding worker or save-on obligations retain their existing safety checks. +Initial source admission waits within the prepared worker's persisted preparation +budget. CSI captures and older Jobs use a separate two-minute startup deadline, +recorded only after lifecycle ownership when the workload or its live source Pod +is unavailable. Queue time does not spend either budget, and operator restarts do +not extend them. Expiry fails the Backup and runs normal cleanup to release its +owner and workload fence. Prepared source identity and outstanding worker or +save-on obligations retain their existing safety checks. The GameServer annotation `operations.gameplane.local/owner` identifies the operation kind, name, UID, admitted generation and target UID. A wipe is identified @@ -60,8 +61,9 @@ deletion; live and terminating worker pods must drain before the owner is releas Quiesced backups also owe a successful save-on call. The existing bounded Backup deletion retry may release its finalizer, but an unconfirmed save-on leaves the durable owner behind for recovery by the next operation. Time alone never releases -ownership. A pending CSI capture retains the current capture-drain behavior; a -separate CSI timeout change is tracked in issue #719. +ownership. CSI cancellation or expiry first invalidates the capture, then allows +save-on independently of driver reclamation. A late-ready aborted capture remains +unrestorable. If an operation waits unexpectedly, inspect the referenced Backup/Restore, its conditions, Jobs and worker pods. Resolve terminating workers, repository failures @@ -78,7 +80,9 @@ Already-running legacy quiesced copies continue being observed until their worke finish and cleanup succeeds. Legacy raw workers also block new admission. Conflicting legacy guards fail closed until their existing operations finish cleanup. Keep a single active operator version during upgrade; older binaries do not honor the -shared owner. No new CRD fields or Kubernetes permissions are required. +shared owner. CSI abort reclamation adds cluster-scoped delete permission for +VolumeSnapshotContents; runtime checks restrict it to recorded, UID-verified +contents with `Delete` policy. CSI restores create a new target and reserve it for the Restore. Initial startup is allowed under that reservation so storage using WaitForFirstConsumer can bind. @@ -92,6 +96,64 @@ writer has drained. Recovery of a force-deleted Restore applies the same check before a successor can proceed, preserving the existing pod template while requesting zero replicas. A successful Restore leaves its initial workload running. -Stopped-source preparation (#716), Restic preparation and quiesce deadlines (#717), -CSI capture deadlines (#719), and staged template adoption (#731) are separate -changes building on this ownership boundary. +Stopped-source proof, bounded copy preparation and CSI invalidation build on this +ownership boundary. Staged template adoption is tracked separately in #731. + +## Backup preparation and capture budgets + +Restic workers now schedule, pull their image, mount the source and validate the +repository before the agent is asked to disable saving. A Ready worker waits for +an expiring authorization bound to its pod UID, Backup UID and observed source. +The preferred node is the game's current node; `operator.backupHardSameNode: true` +makes that placement mandatory. Source tolerations and `operator.backupResources` +apply to preparation and copy containers. Stopped sources leave node selection +to the scheduler and volume affinity. + +When the source StatefulSet requests a running workload, initial preparation +waits for its owned pod to be created and assigned a node before binding the +worker to that source. This wait uses the preparation budget and does not disable +saving. A pod or node replacement after binding still fails the copy. Raw backups +of intentionally stopped workloads remain eligible without a source pod. + +Temporary Kubernetes source-read failures allow retries within the backup's +existing deadline. An unsuccessful source/readiness check revokes the start +token until the operator can verify the source again. Revocation can take time +to reach a worker and does not stop an already running copy, so lifecycle +ownership and save-on obligations remain until worker drain is confirmed. + +The install settings are `operator.backupPreparationTimeout: 10m`, +`operator.backupQuiesceTimeout: 5m`, and `operator.backupCSICaptureTimeout: 10m`. +Equivalent operator flags use kebab-case names. The recorded deadlines remain +fixed across restarts or later configuration edits. Restic's existing Job +backoff/deadline settings remain separate; raw and already-stopped copies use +the full Job deadline. A live quiesce attempt retains the short budget even if +the agent reports the command unsupported. Increase the live quiesce budget +deliberately for worlds that cannot finish within five minutes, or use a +supported CSI capture strategy. + +Preparation failure leaves saving enabled. Quiesce expiry marks the copy failed, +revokes authorization and cancels its Job. Saving is restored only after all +live or terminating workers have stopped. The worker itself also stops Restic +at the token deadline (TERM, then KILL after five seconds), including when the +operator is unavailable. This bounds copying; save-on still requires the operator +and agent to recover. `CaptureReady` describes preparation/authorization or a +pending drain. Resolve stuck mounts, node failures or terminating workers before +retrying; removing owner annotations is not proof that copying stopped. + +Custom Restic images must provide the restic CLI (including `snapshots --retry-lock`), +POSIX `/bin/sh`, `date +%s`, `cat`, `touch`, `sleep`, and `timeout -s TERM -k 5`. +The pinned `restic/restic:0.17.1` release uses Alpine/BusyBox and supports this +contract. The gate runs under the existing non-root, read-only-root-filesystem +security context. Node/operator clocks must be synchronized for absolute token +expiry. No worker receives Kubernetes API credentials. + +CSI expiry and Backup deletion invalidate an incomplete capture before save-on. +The `snapshot-aborted` marker is terminal even if a delayed driver later reports +readiness. Normal restore and new raw GameServer PVC references reject such +captures. `SnapshotCleaned` reports reclamation progress and records snapshot +and content names/UIDs; `Retain` content and unverified replacements require +administrator action. A slow driver can keep Backup deletion pending for storage +cleanup after saving and game ownership have already been restored. Inspect the +condition/event identities before deleting retained storage manually. Existing +PVCs remain usable after source Backup retention; administrator-managed external +snapshots keep their direct-CRD behavior. diff --git a/hack/ci_scope.py b/hack/ci_scope.py index dfdb4709a..492efd102 100644 --- a/hack/ci_scope.py +++ b/hack/ci_scope.py @@ -29,8 +29,8 @@ FORCE_ALL_PREFIXES = (".github/actions/",) # Mirrors the e2e-go matrix buckets in .github/workflows/ci.yaml. E2E_BUCKETS = ["operator", "api-auth", "api-roles", "api-rbac", "api-agent", "api-mods", - "telemetry", "optional-components", "restricted-workloads", "management-placement"] -E2E_AMD64_ONLY = {"management-placement"} + "telemetry", "optional-components", "restricted-workloads", "management-placement", "backup-preparation"] +E2E_AMD64_ONLY = {"management-placement", "backup-preparation"} SUITES_FILE = "test/e2e/suite-components.json" # Paths no Go e2e test reads. Their own jobs keep their paths-filter gates. NO_E2E_FILES = { diff --git a/hack/test_ci_scope.py b/hack/test_ci_scope.py index 3436a0208..b5097a3ec 100644 --- a/hack/test_ci_scope.py +++ b/hack/test_ci_scope.py @@ -426,6 +426,14 @@ def test_audit_management_placement_has_a_run_pattern(self): self.assertIn("TestManagementPlacementAndDisruption", self.select(path)["e2e-run"].get("management-placement", "")) + def test_audit_backup_preparation_has_a_run_pattern(self): + """Worker preparation remains selected for each covered component.""" + for path in ("operator/internal/controller/backup_worker.go", "api/cmd/main.go", + "agent/cmd/main.go", "test/e2e/backup_preparation_e2e_test.go"): + with self.subTest(path=path): + self.assertIn("TestBackup_PreparationAndSameNodeCopy", + self.select(path)["e2e-run"].get("backup-preparation", "")) + def test_audit_staged_rollout_keeps_operator_coverage(self): """Template adoption and its owning fixture select the rollout scenario.""" for path in ("operator/internal/controller/gameserver_template_revision.go", diff --git a/i18n/locales/en/app.json b/i18n/locales/en/app.json index e30f9d5ca..0f2b5f35b 100644 --- a/i18n/locales/en/app.json +++ b/i18n/locales/en/app.json @@ -155,6 +155,22 @@ "defaultMessage": "Requested {requestKind, select, addressAndPool {address ''{address}'' from pool ''{pool}''} address {address ''{address}''} other {pool ''{pool}''}}: waiting for the LoadBalancer Service that carries the request to exist.", "description": "AddressAssignment condition before the load-balancer Service exists." }, + "condition.captureready.snapshotcapturepending": { + "defaultMessage": "Waiting for CSI capture within its persisted deadline.", + "description": "Bounded CSI capture is pending." + }, + "condition.captureready.workerauthorized": { + "defaultMessage": "The prepared worker is authorized to capture data within its deadline.", + "description": "Backup copy authorization granted after preparation." + }, + "condition.captureready.workerdrainpending": { + "defaultMessage": "The copy is invalid. Waiting for workers to stop before restoring saving.", + "description": "Backup cancellation awaiting worker drainage." + }, + "condition.captureready.workerpreparing": { + "defaultMessage": "Waiting for worker scheduling, source mount and repository preparation.", + "description": "Backup preparation before saving is disabled." + }, "condition.completed.failed": { "defaultMessage": "The backup or restore failed.", "description": "Completed condition on a Backup or Restore that failed. The message holds the English detail." @@ -579,6 +595,22 @@ "defaultMessage": "The capture sidecar was told to stop.", "description": "SidecarStopped condition after a stop request." }, + "condition.snapshotcleaned.snapshotcleanupcomplete": { + "defaultMessage": "Aborted snapshot resources are absent.", + "description": "Snapshot reclamation completed." + }, + "condition.snapshotcleaned.snapshotcleanuppending": { + "defaultMessage": "Aborted snapshot reclamation is pending. {detail}", + "description": "Aborted snapshot deletion has not completed; detail includes resource identities." + }, + "condition.snapshotcleaned.snapshotcleanupretained": { + "defaultMessage": "Snapshot content requires administrator reclamation. {detail}", + "description": "Retained or replacement snapshot content is left for an administrator." + }, + "condition.snapshotcleaned.snapshotcleanupunverified": { + "defaultMessage": "A snapshot resource was replaced. It was not deleted. {detail}", + "description": "Snapshot identity verification refused deletion." + }, "condition.snapshotforgotten.forgetabandoned": { "defaultMessage": "The snapshot was not removed from the backup repository when the backup was deleted.", "description": "SnapshotForgotten condition when the forget step was abandoned." diff --git a/operator/cmd/main.go b/operator/cmd/main.go index 9fe5159ba..146b43040 100644 --- a/operator/cmd/main.go +++ b/operator/cmd/main.go @@ -156,6 +156,10 @@ func main() { captureDefaultMaxSizeBytes int64 backupJobBackoffLimit int64 backupJobActiveDeadlineSeconds int64 + backupPreparationTimeout time.Duration + backupQuiesceTimeout time.Duration + backupCSICaptureTimeout time.Duration + backupHardSameNode bool agentResourcesJSON string sentinelResourcesJSON string tunnelResourcesJSON string @@ -187,6 +191,10 @@ func main() { "BackoffLimit for the restic backup and restore Jobs (retries before the Job is marked Failed). Must be >= 0. Defaults to 2.") flag.Int64Var(&backupJobActiveDeadlineSeconds, "backup-job-active-deadline-seconds", controller.DefaultBackupJobActiveDeadlineSeconds, "ActiveDeadlineSeconds for the restic backup and restore Jobs. Must be > 0. Defaults to 86400 (24 hours).") + flag.DurationVar(&backupPreparationTimeout, "backup-preparation-timeout", controller.DefaultBackupPreparationTimeout, "Maximum worker preparation time before copy authorization; must be positive.") + flag.DurationVar(&backupQuiesceTimeout, "backup-quiesce-timeout", controller.DefaultBackupQuiesceTimeout, "Maximum live quiesced Restic copy duration; expiry cancels and drains workers before save-on.") + flag.DurationVar(&backupCSICaptureTimeout, "backup-csi-capture-timeout", controller.DefaultBackupCSICaptureTimeout, "Maximum CSI capture duration before invalidation and save-on; must be positive.") + flag.BoolVar(&backupHardSameNode, "backup-hard-same-node", false, "Require Restic workers on the observed game node instead of the default preferred placement.") flag.StringVar(&sentinelImage, "sentinel-image", controller.DefaultSentinelImage, "Image for the wake sentinel pod that holds advertised ports while a server is asleep. "+ "Point at a private registry mirror for air-gapped installs.") @@ -326,6 +334,10 @@ func main() { setupLog.Error(nil, "invalid --backup-job-active-deadline-seconds value (must be > 0)", "value", backupJobActiveDeadlineSeconds) os.Exit(1) } + if backupPreparationTimeout <= 0 || backupQuiesceTimeout <= 0 || backupCSICaptureTimeout <= 0 { + setupLog.Error(nil, "backup preparation, quiesce and CSI capture timeouts must be positive") + os.Exit(1) + } // Validate capture retention flags while they are still the int64 type // flag.Int64Var naturally produces. The minimum of 60 seconds aligns with @@ -491,6 +503,10 @@ func main() { JobBackoffLimit: &backupJobBackoff32, JobActiveDeadlineSeconds: backupJobActiveDeadlineSeconds, EventRecorder: mgr.GetEventRecorder("backup-controller"), + PreparationTimeout: backupPreparationTimeout, + QuiesceTimeout: backupQuiesceTimeout, + CSICaptureTimeout: backupCSICaptureTimeout, + HardSameNode: backupHardSameNode, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to set up controller", "controller", "Backup") os.Exit(1) diff --git a/operator/config/rbac/role.yaml b/operator/config/rbac/role.yaml index a676b73c6..ca8e31e35 100644 --- a/operator/config/rbac/role.yaml +++ b/operator/config/rbac/role.yaml @@ -151,6 +151,14 @@ rules: - snapshot.storage.k8s.io resources: - volumesnapshotcontents + verbs: + - delete + - get + - list + - watch +- apiGroups: + - snapshot.storage.k8s.io + resources: - volumesnapshots verbs: - get diff --git a/operator/hack/condition-ids/main.go b/operator/hack/condition-ids/main.go index c5ac9ea16..1a33f1700 100644 --- a/operator/hack/condition-ids/main.go +++ b/operator/hack/condition-ids/main.go @@ -52,6 +52,8 @@ var guardCalls = map[string]int{ "markPending": 2, "failWithReason": 2, "setSnapshotForgottenCondition": 3, + "setBackupProgress": 2, + "snapshotCleanupCondition": 3, } // guardReturnFuncs are the functions whose first string result is a reason. @@ -68,6 +70,14 @@ var guardSkipFiles = map[string]bool{ // rows is the English reference for every condition reason. var rows = []row{ + {types: []string{"CaptureReady"}, reason: "WorkerPreparing", english: "Waiting for worker scheduling, source mount and repository preparation.", description: "Backup preparation before saving is disabled."}, + {types: []string{"CaptureReady"}, reason: "WorkerAuthorized", english: "The prepared worker is authorized to capture data within its deadline.", description: "Backup copy authorization granted after preparation."}, + {types: []string{"CaptureReady"}, reason: "WorkerDrainPending", english: "The copy is invalid. Waiting for workers to stop before restoring saving.", description: "Backup cancellation awaiting worker drainage."}, + {types: []string{"CaptureReady"}, reason: "SnapshotCapturePending", english: "Waiting for CSI capture within its persisted deadline.", description: "Bounded CSI capture is pending."}, + {types: []string{"SnapshotCleaned"}, reason: "SnapshotCleanupPending", english: "Aborted snapshot reclamation is pending. {detail}", description: "Aborted snapshot deletion has not completed; detail includes resource identities."}, + {types: []string{"SnapshotCleaned"}, reason: "SnapshotCleanupUnverified", english: "A snapshot resource was replaced. It was not deleted. {detail}", description: "Snapshot identity verification refused deletion."}, + {types: []string{"SnapshotCleaned"}, reason: "SnapshotCleanupRetained", english: "Snapshot content requires administrator reclamation. {detail}", description: "Retained or replacement snapshot content is left for an administrator."}, + {types: []string{"SnapshotCleaned"}, reason: "SnapshotCleanupComplete", english: "Aborted snapshot resources are absent.", description: "Snapshot reclamation completed."}, {types: []string{"TemplateUpdatePending"}, reason: "RolloutCanceled", english: "Template adoption was canceled; the effective pin and power intent are retained. Remove the cancellation annotation before requesting this revision again.", description: "An operator canceled the admitted revision; its token remains blocked until the cancellation annotation is removed."}, {types: []string{"TemplateUpdatePending"}, reason: "AdoptingRevision", english: "The admitted template revision is being adopted.", description: "Template rollout owns the server lifecycle while draining and replacing its workload."}, {types: []string{"TemplateUpdatePending"}, reason: "TemplateUnavailable", english: "The installed template is unavailable; the effective snapshot is retained.", description: "Template availability is unknown but the server retains its immutable effective content."}, diff --git a/operator/internal/controller/backup_budgets.go b/operator/internal/controller/backup_budgets.go new file mode 100644 index 000000000..ca0acf1d2 --- /dev/null +++ b/operator/internal/controller/backup_budgets.go @@ -0,0 +1,122 @@ +package controller + +import ( + "context" + "fmt" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +const ( + DefaultBackupPreparationTimeout = 10 * time.Minute + DefaultBackupQuiesceTimeout = 5 * time.Minute + DefaultBackupCSICaptureTimeout = 10 * time.Minute + backupPreparationDeadlineAnnotation = "backup.gameplane.local/preparation-deadline" + backupQuiesceDeadlineAnnotation = "backup.gameplane.local/quiesce-deadline" + backupCSIDeadlineAnnotation = "backup.gameplane.local/csi-deadline" + backupCopyAbortAnnotation = "backup.gameplane.local/copy-aborted" +) + +func backupTimeout(configured, fallback time.Duration) time.Duration { + if configured > 0 { + return configured + } + return fallback +} + +// Write once before the bounded action. Reconfiguration and restarts cannot +// extend an admitted attempt. Malformed stored state fails closed. +func (r *BackupReconciler) backupDeadline(ctx context.Context, b *gameplanev1alpha1.Backup, key string, duration time.Duration) (time.Time, error) { + if value := b.Annotations[key]; value != "" { + deadline, err := time.Parse(time.RFC3339Nano, value) + if err != nil { + return time.Time{}, fmt.Errorf("invalid persisted backup deadline %s: %w", key, err) + } + return deadline, nil + } + deadline := time.Now().UTC().Add(duration) + patchBackupAnnotations(b, map[string]string{key: deadline.Format(time.RFC3339Nano)}) + return deadline, r.Update(ctx, b) +} + +func (r *BackupReconciler) setBackupProgress(ctx context.Context, b *gameplanev1alpha1.Backup, reason, message string) error { + status := metav1.ConditionFalse + if reason == "WorkerAuthorized" { + status = metav1.ConditionTrue + } + conditions := upsertCondition(b.Status.Conditions, metav1.Condition{Type: "CaptureReady", Status: status, + Reason: reason, Message: message, ObservedGeneration: b.Generation}) + if sameConditions(b.Status.Conditions, conditions) { + return nil + } + b.Status.Conditions = conditions + return r.Status().Update(ctx, b) +} + +// Persist invalidation before revoking authorization or cancelling workers. +// A stale mounted token can still be observed until projection catches up, so +// neither token revocation nor a deadline substitutes for actual worker drain. +func (r *BackupReconciler) abortResticCopy(ctx context.Context, b *gameplanev1alpha1.Backup, message string) (ctrl.Result, error) { + if message == "" { + message = "backup copy invalidated" + } + if b.Annotations[backupCopyAbortAnnotation] == "" { + patchBackupAnnotations(b, map[string]string{backupCopyAbortAnnotation: message}) + if err := r.Update(ctx, b); err != nil { + return ctrl.Result{}, err + } + } + message = b.Annotations[backupCopyAbortAnnotation] + if b.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || b.Status.SnapshotID != "" || b.Status.Message != message { + b.Status.SnapshotID = "" + if _, err := r.fail(ctx, b, message); err != nil { + return ctrl.Result{}, err + } + } + return r.cleanupResticCopy(ctx, b) +} + +func (r *BackupReconciler) cleanupResticCopy(ctx context.Context, b *gameplanev1alpha1.Backup) (ctrl.Result, error) { + if err := r.revokeBackupStart(ctx, b); err != nil { + return ctrl.Result{}, err + } + busy, err := r.backupCleanupWorkersLive(ctx, b, true) + if err != nil { + return ctrl.Result{}, err + } + if busy { + if err := r.setBackupProgress(ctx, b, "WorkerDrainPending", "copy invalidated; waiting for workers to stop before restoring saving"); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + } + if meta.RemoveStatusCondition(&b.Status.Conditions, "CaptureReady") { + if err := r.Status().Update(ctx, b); err != nil { + return ctrl.Result{}, err + } + } + return r.runUnquiesce(ctx, b) +} + +func (r *BackupReconciler) revokeBackupStart(ctx context.Context, b *gameplanev1alpha1.Backup) error { + var gate corev1.ConfigMap + err := r.backupAPIReader().Get(ctx, client.ObjectKey{Namespace: b.Namespace, Name: backupStartConfigMapName(b)}, &gate) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + if err == nil && metav1.IsControlledBy(&gate, b) && gate.Data[backupStartTokenKey] != "" { + gate.Data[backupStartTokenKey] = "" + if err := r.Update(ctx, &gate); err != nil { + return err + } + } + return nil +} diff --git a/operator/internal/controller/backup_budgets_test.go b/operator/internal/controller/backup_budgets_test.go new file mode 100644 index 000000000..738274e96 --- /dev/null +++ b/operator/internal/controller/backup_budgets_test.go @@ -0,0 +1,691 @@ +package controller + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +func budgetPass(t *testing.T, r *BackupReconciler, name string) { + t.Helper() + for range 4 { + if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "ns", Name: name}}); err != nil { + t.Fatal(err) + } + } +} + +// Model the Job controller and kubelet explicitly: no worker may be authorized +// merely because the Job exists or an informer has an old Ready observation. +func budgetReadyWorker(t *testing.T, r *BackupReconciler, name, uid string) *corev1.Pod { + t.Helper() + ctx := context.Background() + var job batchv1.Job + if err := r.Get(ctx, client.ObjectKey{Namespace: "ns", Name: name}, &job); err != nil { + t.Fatal(err) + } + if job.UID == "" { + job.UID = types.UID(name + "-job-uid") + if err := r.Update(ctx, &job); err != nil { + t.Fatal(err) + } + } + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name + "-worker", Namespace: "ns", UID: types.UID(uid), Labels: job.Spec.Template.Labels, + OwnerReferences: []metav1.OwnerReference{{APIVersion: "batch/v1", Kind: "Job", Name: job.Name, UID: job.UID, Controller: ownerBoolPtr(true)}}}, + Spec: job.Spec.Template.Spec, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}}} + pod.Spec.NodeName = "node-a" + if err := r.Create(ctx, pod); err != nil { + t.Fatal(err) + } + return pod +} + +func TestBackupWorkerPreparesBeforeQuiesceAndPinsAuthorization(t *testing.T) { + r, q := guardFixtureWorkers(t, false, guardBackup("prepared")) + budgetPass(t, r, "prepared") + if q.quiesced != 0 { + t.Fatal("save-off preceded worker readiness") + } + b := guardGetBackup(t, r, "prepared") + if b.Status.CaptureManifest != nil { + t.Fatal("capture metadata recorded before worker preparation") + } + var gate corev1.ConfigMap + if err := r.Get(context.Background(), client.ObjectKey{Namespace: "ns", Name: backupStartConfigMapName(b)}, &gate); err != nil { + t.Fatal(err) + } + if gate.Data[backupStartTokenKey] != "" { + t.Fatal("unprepared worker received start authorization") + } + worker := budgetReadyWorker(t, r, b.Name, "prepared-worker-uid") + budgetPass(t, r, b.Name) + b = guardGetBackup(t, r, b.Name) + if q.quiesced != 1 || b.Status.CaptureManifest == nil || b.Status.CaptureManifest.State != "Recorded" || b.Status.CaptureManifest.QuiesceOutcome != "confirmed" { + t.Fatalf("prepared copy not quiesced/recorded: %+v, %+v", q, b.Status) + } + if volumes := b.Status.CaptureManifest.Volumes; len(volumes) != 1 || volumes[0].ClaimName != "server-data" || !volumes[0].Included || volumes[0].Classification != "data" { + t.Fatalf("prepared worker lost capture-time coverage: %+v", volumes) + } + if err := r.Get(context.Background(), client.ObjectKeyFromObject(&gate), &gate); err != nil { + t.Fatal(err) + } + if !strings.Contains(gate.Data[backupStartTokenKey], string(worker.UID)) || !strings.Contains(gate.Data[backupStartTokenKey], string(b.UID)) { + t.Fatalf("unbound authorization: %q", gate.Data[backupStartTokenKey]) + } + deadline := b.Annotations[backupQuiesceDeadlineAnnotation] + r = &BackupReconciler{Client: r.Client, Scheme: r.Scheme, AgentClient: q, QuiesceTimeout: time.Hour} + budgetPass(t, r, b.Name) + if got := guardGetBackup(t, r, b.Name).Annotations[backupQuiesceDeadlineAnnotation]; got != deadline { + t.Fatal("restart extended admitted quiesce deadline") + } +} + +func TestBackupPreparedWorkerUsesTemplateAdmittedBeforeOwnership(t *testing.T) { + for _, adoptBeforeClaim := range []bool{false, true} { + t.Run(fmt.Sprintf("adopt-before-claim=%t", adoptBeforeClaim), func(t *testing.T) { + r, q := guardFixtureWorkers(t, false, guardBackup("prepared")) + base := r.Client.(client.WithWatch) + r.APIReader = base + var gs gameplanev1alpha1.GameServer + if err := base.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server"}, &gs); err != nil { + t.Fatal(err) + } + oldGroup, nextGroup := int64(1000), int64(2000) + installed := &gameplanev1alpha1.GameTemplate{ObjectMeta: metav1.ObjectMeta{Name: "template", UID: "old-catalog-uid"}, + Spec: gameplanev1alpha1.GameTemplateSpec{Security: &gameplanev1alpha1.GameSecuritySpec{FSGroup: &oldGroup}}} + if err := base.Create(t.Context(), installed); err != nil { + t.Fatal(err) + } + revisions := &GameServerReconciler{Client: base, APIReader: base, Scheme: r.Scheme} + oldPin, err := revisions.saveTemplateSnapshot(t.Context(), &gs, installed) + if err != nil { + t.Fatal(err) + } + gs.Annotations = map[string]string{effectiveTemplateAnnotation: oldPin} + if err := base.Update(t.Context(), &gs); err != nil { + t.Fatal(err) + } + // Reinstalling the catalog can change both its UID and filesystem + // settings without admitting that content for this GameServer. + if err := base.Delete(t.Context(), installed); err != nil { + t.Fatal(err) + } + installed = &gameplanev1alpha1.GameTemplate{ObjectMeta: metav1.ObjectMeta{Name: "template", UID: "new-catalog-uid"}, + Spec: gameplanev1alpha1.GameTemplateSpec{Security: &gameplanev1alpha1.GameSecuritySpec{FSGroup: &nextGroup}}} + if err := base.Create(t.Context(), installed); err != nil { + t.Fatal(err) + } + nextPin, err := revisions.saveTemplateSnapshot(t.Context(), &gs, installed) + if err != nil { + t.Fatal(err) + } + adopted := false + r.Client = interceptor.NewClient(base, interceptor.Funcs{Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + if b, ok := obj.(*gameplanev1alpha1.Backup); ok && adoptBeforeClaim && !adopted && b.Annotations[backupTargetAnnotation] != "" { + // Finish a rollout after Reconcile reads the GameServer but + // before the Backup claims its shared lifecycle ownership. + var live gameplanev1alpha1.GameServer + if err := c.Get(ctx, client.ObjectKeyFromObject(&gs), &live); err != nil { + return err + } + live.Annotations[effectiveTemplateAnnotation] = nextPin + if err := c.Update(ctx, &live); err != nil { + return err + } + adopted = true + } + return c.Update(ctx, obj, opts...) + }}) + budgetPass(t, r, "prepared") + var job batchv1.Job + if err := base.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "prepared"}, &job); err != nil { + t.Fatal(err) + } + want := oldGroup + if adoptBeforeClaim { + want = nextGroup + } + security := job.Spec.Template.Spec.SecurityContext + if adopted != adoptBeforeClaim || security == nil || security.FSGroup == nil || *security.FSGroup != want { + t.Fatalf("prepared worker did not use the admitted source: adopted=%t security=%+v want fsGroup=%d", adopted, security, want) + } + if q.quiesced != 0 { + t.Fatal("template resolution bypassed worker readiness") + } + budgetReadyWorker(t, r, "prepared", "ready-worker-uid") + budgetPass(t, r, "prepared") + b := guardGetBackup(t, r, "prepared") + if b.Annotations[backupAuthorizedWorkerAnnotation] == "" || b.Status.CaptureManifest == nil || b.Status.CaptureManifest.State != "Recorded" || q.quiesced != 1 { + t.Fatalf("pinned preparation did not reach authorized capture: %+v %+v", b, q) + } + }) + } +} + +func TestBackupPreparedWorkerRejectsMissingEffectiveTemplate(t *testing.T) { + r, q := guardFixtureWorkers(t, false, guardBackup("prepared")) + var gs gameplanev1alpha1.GameServer + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server"}, &gs); err != nil { + t.Fatal(err) + } + gs.Annotations = map[string]string{effectiveTemplateAnnotation: strings.Repeat("a", 64)} + if err := r.Update(t.Context(), &gs); err != nil { + t.Fatal(err) + } + // A readable catalog must not substitute for the missing admitted content. + if err := r.Create(t.Context(), &gameplanev1alpha1.GameTemplate{ObjectMeta: metav1.ObjectMeta{Name: "template"}}); err != nil { + t.Fatal(err) + } + _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKey{Namespace: "ns", Name: "prepared"}}) + b := guardGetBackup(t, r, "prepared") + if err != nil || b.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || !strings.Contains(b.Status.Message, "read pinned template snapshot") || q.quiesced != 0 { + t.Fatalf("missing pin did not fail visibly before save-off: %v %+v %+v", err, b.Status, q) + } + if err := r.Get(t.Context(), client.ObjectKeyFromObject(&gs), &gs); err != nil { + t.Fatal(err) + } + if gs.Annotations[lifecycleOperationAnnotation] != "" || gs.Annotations[backupGuardAnnotation] != "" { + t.Fatal("missing template retained lifecycle ownership") + } + var job batchv1.Job + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "prepared"}, &job); !apierrors.IsNotFound(err) { + t.Fatalf("missing pin created a worker: %v", err) + } +} + +func TestBackupPreparationDeadlineNeverDisablesSaving(t *testing.T) { + r, q := guardFixtureWorkers(t, false, guardBackup("expired")) + budgetPass(t, r, "expired") + b := guardGetBackup(t, r, "expired") + patchBackupAnnotations(b, map[string]string{backupPreparationDeadlineAnnotation: time.Now().Add(-time.Minute).UTC().Format(time.RFC3339Nano)}) + if err := r.Update(context.Background(), b); err != nil { + t.Fatal(err) + } + budgetPass(t, r, b.Name) + b = guardGetBackup(t, r, b.Name) + if b.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || q.quiesced != 0 { + t.Fatalf("expired preparation disabled saving: %+v %+v", b.Status, q) + } +} + +func TestBackupRawAndStoppedCopiesKeepFullJobBudget(t *testing.T) { + for _, mode := range []string{"raw", "no-agent", "stopped"} { + t.Run(mode, func(t *testing.T) { + b := guardBackup("first") + b.Spec.Quiesce = mode != "raw" + r, q := guardFixture(t, b) + if mode == "no-agent" { + r.AgentClient = nil + } + if mode == "stopped" { + r, q = stoppedBackupFixture(t, true) + } + r.QuiesceTimeout, r.JobActiveDeadlineSeconds = time.Second, 7200 + budgetPass(t, r, "first") + b = guardGetBackup(t, r, "first") + deadline, err := time.Parse(time.RFC3339Nano, b.Annotations[backupQuiesceDeadlineAnnotation]) + if err != nil || time.Until(deadline) < time.Hour || b.Annotations[backupAuthorizedWorkerAnnotation] == "" || q.quiesced != 0 { + t.Fatalf("non-quiesced copy lost its full budget/authorization: %v %+v %+v", err, b.Annotations, q) + } + }) + } +} + +func TestBackupRawSourceWaitsForInitialScheduling(t *testing.T) { + for _, initiallyAbsent := range []bool{false, true} { + t.Run(fmt.Sprintf("absent=%t", initiallyAbsent), func(t *testing.T) { + b := guardBackup("starting") + b.Spec.Quiesce = false + r, q := guardFixtureWorkers(t, false, b) + var pod corev1.Pod + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server-0"}, &pod); err != nil { + t.Fatal(err) + } + pod.Spec.NodeName = "" + if err := r.Update(t.Context(), &pod); err != nil { + t.Fatal(err) + } + if initiallyAbsent { + if err := r.Delete(t.Context(), &pod); err != nil { + t.Fatal(err) + } + } + budgetPass(t, r, b.Name) + b = guardGetBackup(t, r, b.Name) + var job batchv1.Job + if err := r.Get(t.Context(), client.ObjectKeyFromObject(b), &job); !apierrors.IsNotFound(err) || b.Annotations[backupWorkerSourceAnnotation] != "" || q.quiesced != 0 { + t.Fatalf("starting source was bound or quiesced: job=%v annotations=%v calls=%+v", err, b.Annotations, q) + } + pod.Spec.NodeName = "node-a" + if initiallyAbsent { + pod.ResourceVersion = "" + if err := r.Create(t.Context(), &pod); err != nil { + t.Fatal(err) + } + } else if err := r.Update(t.Context(), &pod); err != nil { + t.Fatal(err) + } + budgetPass(t, r, b.Name) + b = guardGetBackup(t, r, b.Name) + if err := r.Get(t.Context(), client.ObjectKeyFromObject(b), &job); err != nil || b.Status.Phase == gameplanev1alpha1.BackupPhaseFailed { + t.Fatalf("normal source scheduling failed backup: job=%v status=%+v", err, b.Status) + } + // Once captured in the worker's immutable environment, the source + // may not silently change to a different pod at the same name. + if err := r.Delete(t.Context(), &pod); err != nil { + t.Fatal(err) + } + pod.UID, pod.ResourceVersion = "replacement-pod", "" + if err := r.Create(t.Context(), &pod); err != nil { + t.Fatal(err) + } + budgetPass(t, r, b.Name) + b = guardGetBackup(t, r, b.Name) + if b.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || b.Annotations[backupAuthorizedWorkerAnnotation] != "" { + t.Fatalf("replacement source escaped immutable binding: %+v", b) + } + }) + } +} + +func TestBackupPreparedWorkerFailurePreservesJobMessage(t *testing.T) { + r, _ := guardFixtureWorkers(t, false, guardBackup("failed-worker")) + budgetPass(t, r, "failed-worker") + var job batchv1.Job + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "failed-worker"}, &job); err != nil { + t.Fatal(err) + } + job.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobFailed, Status: corev1.ConditionTrue}} + if err := r.Status().Update(t.Context(), &job); err != nil { + t.Fatal(err) + } + budgetPass(t, r, job.Name) + b := guardGetBackup(t, r, job.Name) + if b.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || b.Status.Message != "backup job reported Failed" || b.Annotations[backupCopyAbortAnnotation] != b.Status.Message { + t.Fatalf("prepared worker lost failure compatibility/invalidation: %+v", b) + } +} + +func TestBackupQueuedCopyDoesNotSpendPreparationBudget(t *testing.T) { + r, _ := guardFixture(t, guardBackup("first"), guardBackup("second")) + budgetPass(t, r, "first") + budgetPass(t, r, "second") + second := guardGetBackup(t, r, "second") + if second.Annotations[backupPreparationDeadlineAnnotation] != "" || second.Annotations[backupAuthorizedWorkerAnnotation] != "" { + t.Fatal("queued Backup spent preparation budget before owning the source") + } +} + +func TestBackupQuiesceDeadlineDrainsBeforeSaveOn(t *testing.T) { + r, q := guardFixtureWorkers(t, false, guardBackup("expired")) + budgetPass(t, r, "expired") + worker := budgetReadyWorker(t, r, "expired", "worker-uid") + budgetPass(t, r, "expired") + b := guardGetBackup(t, r, "expired") + patchBackupAnnotations(b, map[string]string{backupQuiesceDeadlineAnnotation: time.Now().Add(-time.Minute).UTC().Format(time.RFC3339Nano)}) + if err := r.Update(context.Background(), b); err != nil { + t.Fatal(err) + } + budgetPass(t, r, b.Name) + b = guardGetBackup(t, r, b.Name) + if b.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || q.unquiesced != 0 { + t.Fatalf("deadline bypassed worker drain: %+v %+v", b.Status, q) + } + if err := r.Delete(context.Background(), worker); err != nil { + t.Fatal(err) + } + budgetPass(t, r, b.Name) + if q.unquiesced != 1 { + t.Fatalf("save-on not retried after drain: %+v", q) + } +} + +func TestBackupDeletionRevokesAuthorizationBeforeTerminatingWorkerDrains(t *testing.T) { + r, q := guardFixtureWorkers(t, false, guardBackup("cancel")) + budgetPass(t, r, "cancel") + worker := budgetReadyWorker(t, r, "cancel", "worker-uid") + budgetPass(t, r, "cancel") + worker.Finalizers = []string{"test/hold"} + if err := r.Update(t.Context(), worker); err != nil { + t.Fatal(err) + } + if err := r.Delete(t.Context(), worker); err != nil { + t.Fatal(err) + } + b := guardGetBackup(t, r, "cancel") + if err := r.Delete(t.Context(), b); err != nil { + t.Fatal(err) + } + budgetPass(t, r, "cancel") + var gate corev1.ConfigMap + if err := r.Get(t.Context(), client.ObjectKey{Namespace: b.Namespace, Name: backupStartConfigMapName(b)}, &gate); err != nil { + t.Fatal(err) + } + if gate.Data[backupStartTokenKey] != "" || q.unquiesced != 0 { + t.Fatal("deletion retained authorization or bypassed terminating worker") + } + if err := r.Get(t.Context(), client.ObjectKeyFromObject(worker), worker); err != nil { + t.Fatal(err) + } + worker.Finalizers = nil + if err := r.Update(t.Context(), worker); err != nil { + t.Fatal(err) + } + budgetPass(t, r, "cancel") + if q.unquiesced != 1 { + t.Fatal("deletion did not restore saving after drain") + } +} + +func TestBackupRecoveryRevokesTokenBeforeCancellingWorkers(t *testing.T) { + r, _ := guardFixtureWorkers(t, false, guardBackup("orphan")) + budgetPass(t, r, "orphan") + budgetReadyWorker(t, r, "orphan", "orphan-worker") + budgetPass(t, r, "orphan") + b := guardGetBackup(t, r, "orphan") + deletionObserved := false + r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if _, ok := obj.(*batchv1.Job); ok { + var gate corev1.ConfigMap + if err := c.Get(ctx, client.ObjectKey{Namespace: b.Namespace, Name: backupStartConfigMapName(b)}, &gate); err != nil { + t.Fatal(err) + } + if gate.Data[backupStartTokenKey] != "" { + t.Fatal("recovery cancelled Job before revoking authorization") + } + deletionObserved = true + } + return c.Delete(ctx, obj, opts...) + }}) + busy, err := r.backupWorkersLive(t.Context(), b.Namespace, backupIdentity{Name: b.Name, UID: b.UID}, true) + if err != nil || !busy || !deletionObserved { + t.Fatalf("recovery bypassed worker cancellation/drain: %t %t %v", busy, deletionObserved, err) + } +} + +func TestBackupCSIDeadlineInvalidatesLateReady(t *testing.T) { + for _, status := range []*snapshotv1.VolumeSnapshotStatus{nil, {ReadyToUse: ownerBoolPtr(false)}} { + b := guardBackup("snapshot") + b.Spec.Strategy = "volume-snapshot" + r, q := guardFixture(t, b) + budgetPass(t, r, b.Name) + b = guardGetBackup(t, r, b.Name) + patchBackupAnnotations(b, map[string]string{backupCSIDeadlineAnnotation: time.Now().Add(-time.Minute).UTC().Format(time.RFC3339Nano)}) + if err := r.Update(context.Background(), b); err != nil { + t.Fatal(err) + } + var vs snapshotv1.VolumeSnapshot + if err := r.Get(context.Background(), client.ObjectKeyFromObject(b), &vs); err != nil { + t.Fatal(err) + } + vs.Status = status + if err := r.Status().Update(context.Background(), &vs); err != nil { + t.Fatal(err) + } + budgetPass(t, r, b.Name) + b = guardGetBackup(t, r, b.Name) + if b.Annotations[backupSnapshotAbortAnnotation] != "true" || b.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || b.Status.SnapshotID != "" || q.unquiesced != 1 { + t.Fatalf("CSI abort did not invalidate before cleanup: %+v %+v", b, q) + } + vs.Status = &snapshotv1.VolumeSnapshotStatus{ReadyToUse: ownerBoolPtr(true)} + if _, err := r.observeVolumeSnapshot(context.Background(), b, &vs); err != nil { + t.Fatal(err) + } + if got := guardGetBackup(t, r, b.Name); got.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || got.Status.SnapshotID != "" { + t.Fatal("late readiness revived aborted capture") + } + } +} + +func TestBackupWorkerDoesNotTrustCachedReadinessOrReplacement(t *testing.T) { + for _, replacement := range []bool{false, true} { + t.Run(fmt.Sprintf("replacement-%t", replacement), func(t *testing.T) { + r, q := guardFixtureWorkers(t, false, guardBackup("copy")) + budgetPass(t, r, "copy") + worker := budgetReadyWorker(t, r, "copy", "first-worker") + if replacement { + q.afterQuiesce = func() { + if err := r.Delete(t.Context(), worker); err != nil { + t.Fatal(err) + } + budgetReadyWorker(t, r, "copy", "replacement-worker") + } + } else { + r.APIReader = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if err := cl.List(ctx, list, opts...); err != nil { + return err + } + if pods, ok := list.(*corev1.PodList); ok { + for i := range pods.Items { + if pods.Items[i].UID == worker.UID { + pods.Items[i].Status.Conditions = nil + } + } + } + return nil + }}) + } + if _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKey{Namespace: "ns", Name: "copy"}}); err != nil { + t.Fatal(err) + } + b := guardGetBackup(t, r, "copy") + var gate corev1.ConfigMap + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: backupStartConfigMapName(b)}, &gate); err != nil { + t.Fatal(err) + } + if gate.Data[backupStartTokenKey] != "" { + t.Fatal("stale readiness granted copy authorization") + } + if !replacement && q.quiesced != 0 { + t.Fatal("cached Ready disabled saving") + } + if replacement && q.quiesced != 1 { + t.Fatal("fixture did not replace worker during RPC") + } + }) + } +} + +func TestPreparedBackupPlacementAndResources(t *testing.T) { + for _, hard := range []bool{false, true} { + r := &BackupReconciler{HardSameNode: hard, BackupResources: &corev1.ResourceRequirements{}} + b := guardBackup("copy") + source := &backupWorkerSource{NodeName: "node-a", Tolerations: []corev1.Toleration{{Key: "games", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule}}} + spec := r.buildPreparedBackupPodSpec(b, &gameplanev1alpha1.GameTemplate{}, source, time.Now().Add(time.Minute)) + if spec.AutomountServiceAccountToken == nil || *spec.AutomountServiceAccountToken || len(spec.Tolerations) != 1 || spec.Tolerations[0].Key != "games" { + t.Fatal("worker credentials or tolerations contract lost") + } + if spec.Affinity == nil || len(spec.Affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution) != 1 { + t.Fatal("same-node preference missing") + } + if (spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil) != hard { + t.Fatal("hard placement opt-in not respected") + } + if len(spec.Containers[0].Resources.Requests) != 0 || spec.Containers[0].ReadinessProbe == nil { + t.Fatal("resource clearing or preparation probe missing") + } + source.NodeName = "" + if stopped := r.buildPreparedBackupPodSpec(b, &gameplanev1alpha1.GameTemplate{}, source, time.Now()); stopped.Affinity != nil { + t.Fatal("stopped source invented a game node") + } + } +} + +// Execute the real gate script in CI with only its mount paths redirected to +// temp directories. A fake restic records whether source capture was invoked. +func TestBackupWorkerScriptRejectsExpiredAndReplayedTokens(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("worker image uses a POSIX shell; executed by Linux CI") + } + for _, tokenKind := range []string{"valid", "expired", "other-worker", "other-backup", "other-source", "repo-failure"} { + t.Run(tokenKind, func(t *testing.T) { + dir := t.TempDir() + restic := "#!/bin/sh\nif [ \"$1\" = snapshots ]; then [ \"$REPO_FAIL\" != true ]; exit $?; fi\nprintf copied > \"$COPY_MARKER\"\n" + if err := os.WriteFile(filepath.Join(dir, "restic"), []byte(restic), 0o700); err != nil { + t.Fatal(err) + } + backup, pod, source, expiry := "backup", "worker", "source", time.Now().Add(time.Minute).Unix() + switch tokenKind { + case "expired": + expiry = time.Now().Add(-time.Minute).Unix() + case "other-worker": + pod = "old-worker" + case "other-backup": + backup = "old-backup" + case "other-source": + source = "old-source" + } + tokenFile, prepared, copied := filepath.Join(dir, "token"), filepath.Join(dir, "prepared"), filepath.Join(dir, "copied") + if err := os.WriteFile(tokenFile, fmt.Appendf(nil, "v1 %s %s %s %d\n", backup, pod, source, expiry), 0o600); err != nil { + t.Fatal(err) + } + script := strings.NewReplacer("/data", dir, "/start/start", tokenFile, "/tmp/prepared", prepared).Replace(resticWorkerScript) + ctx, cancel := context.WithTimeout(t.Context(), 4*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, "/bin/sh", "-c", script, "backup-worker", "backup", dir) + cmd.Env = append(os.Environ(), "PATH="+dir+":"+os.Getenv("PATH"), "BACKUP_UID=backup", "POD_UID=worker", "SOURCE_ID=source", fmt.Sprintf("PREPARATION_EXPIRES=%d", time.Now().Add(time.Second).Unix()), "COPY_MARKER="+copied, fmt.Sprintf("REPO_FAIL=%t", tokenKind == "repo-failure")) + output, err := cmd.CombinedOutput() + if ctx.Err() != nil { + t.Fatalf("gate did not terminate: %s", output) + } + _, copiedErr := os.Stat(copied) + if tokenKind == "valid" { + if err != nil || copiedErr != nil { + t.Fatalf("valid token did not start copy: %v %s", err, output) + } + } else if err == nil || copiedErr == nil { + t.Fatalf("invalid token/preparation started capture: %v %s", err, output) + } + if tokenKind == "repo-failure" { + if _, err := os.Stat(prepared); err == nil { + t.Fatal("repository failure became Ready") + } + } + }) + } +} + +func TestBackupCSIAbortReclaimsOnlyVerifiedDeleteContent(t *testing.T) { + for _, mode := range []string{"delete", "retain", "foreign-reference", "replacement-snapshot", "delete-error"} { + t.Run(mode, func(t *testing.T) { + b := guardBackup("snapshot") + b.Spec.Strategy = "volume-snapshot" + r, q := guardFixture(t, b) + budgetPass(t, r, b.Name) + b = guardGetBackup(t, r, b.Name) + var vs snapshotv1.VolumeSnapshot + if err := r.Get(t.Context(), client.ObjectKeyFromObject(b), &vs); err != nil { + t.Fatal(err) + } + contentName := "content" + content := &snapshotv1.VolumeSnapshotContent{ObjectMeta: metav1.ObjectMeta{Name: contentName, UID: "content-uid"}, Spec: snapshotv1.VolumeSnapshotContentSpec{DeletionPolicy: snapshotv1.VolumeSnapshotContentDelete, + VolumeSnapshotRef: corev1.ObjectReference{Name: vs.Name, Namespace: vs.Namespace, UID: vs.UID}}} + if mode == "retain" { + content.Spec.DeletionPolicy = snapshotv1.VolumeSnapshotContentRetain + } + if mode == "foreign-reference" { + content.Spec.VolumeSnapshotRef.UID = "foreign-snapshot" + } + if err := r.Create(t.Context(), content); err != nil { + t.Fatal(err) + } + vs.Status = &snapshotv1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName: &contentName} + if err := r.Status().Update(t.Context(), &vs); err != nil { + t.Fatal(err) + } + if mode == "delete" { + r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + options := &client.DeleteOptions{} + for _, option := range opts { + option.ApplyToDelete(options) + } + if options.Preconditions == nil || options.Preconditions.UID == nil || *options.Preconditions.UID != obj.GetUID() { + t.Fatalf("cleanup deletion lacks exact UID precondition: %+v", options) + } + return c.Delete(ctx, obj, opts...) + }}) + } + if mode == "replacement-snapshot" { + if err := r.Delete(t.Context(), &vs); err != nil { + t.Fatal(err) + } + vs.UID, vs.ResourceVersion = "replacement-uid", "" + if err := r.Create(t.Context(), &vs); err != nil { + t.Fatal(err) + } + } + if mode == "delete-error" { + r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if _, ok := obj.(*snapshotv1.VolumeSnapshot); ok { + return fmt.Errorf("driver deletion unavailable") + } + return c.Delete(ctx, obj, opts...) + }}) + } + patchBackupAnnotations(b, map[string]string{backupCSIDeadlineAnnotation: time.Now().Add(-time.Minute).UTC().Format(time.RFC3339Nano)}) + if err := r.Update(t.Context(), b); err != nil { + t.Fatal(err) + } + // First request deletion, then separately observe its completion. + // A single pass must not claim backend reclamation was observed. + if _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(b)}); err != nil { + t.Fatal(err) + } + b = guardGetBackup(t, r, b.Name) + if q.unquiesced != 1 || b.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || b.Status.SnapshotID != "" { + t.Fatalf("reclamation blocked invalidation/save-on: %+v %+v", b.Status, q) + } + condition := meta.FindStatusCondition(b.Status.Conditions, "SnapshotCleaned") + if condition == nil { + t.Fatal("missing cleanup condition") + } + err := r.Get(t.Context(), client.ObjectKeyFromObject(content), content) + switch mode { + case "delete": + if !apierrors.IsNotFound(err) || condition.Status != metav1.ConditionFalse || condition.Reason != "SnapshotCleanupPending" { + t.Fatalf("deletion request was not recorded before observing absence: %v %+v", err, condition) + } + if err := r.Get(t.Context(), client.ObjectKeyFromObject(&vs), &vs); !apierrors.IsNotFound(err) { + t.Fatalf("snapshot remains after deletion: %v", err) + } + budgetPass(t, r, b.Name) + b = guardGetBackup(t, r, b.Name) + condition = meta.FindStatusCondition(b.Status.Conditions, "SnapshotCleaned") + if condition == nil || condition.Status != metav1.ConditionTrue || condition.Reason != "SnapshotCleanupComplete" { + t.Fatalf("observed resource absence did not complete reclamation: %+v", condition) + } + case "retain": + if err != nil || condition.Reason != "SnapshotCleanupRetained" || !strings.Contains(condition.Message, "content-uid") { + t.Fatalf("Retain identity was lost: %v %+v", err, condition) + } + case "foreign-reference", "replacement-snapshot": + if err != nil || condition.Reason != "SnapshotCleanupUnverified" { + t.Fatalf("foreign resource was touched: %v %+v", err, condition) + } + case "delete-error": + if condition.Reason != "SnapshotCleanupPending" || !strings.Contains(condition.Message, "ns/snapshot") || !strings.Contains(condition.Message, string(vs.UID)) { + t.Fatalf("delete failure not visible: %+v", condition) + } + } + }) + } +} diff --git a/operator/internal/controller/backup_controller.go b/operator/internal/controller/backup_controller.go index 4ba3efe80..0dcb5217a 100644 --- a/operator/internal/controller/backup_controller.go +++ b/operator/internal/controller/backup_controller.go @@ -127,6 +127,10 @@ type BackupReconciler struct { // DefaultBackupJobBackoffLimit / DefaultBackupJobActiveDeadlineSeconds. JobBackoffLimit *int32 JobActiveDeadlineSeconds int64 + PreparationTimeout time.Duration + QuiesceTimeout time.Duration + CSICaptureTimeout time.Duration + HardSameNode bool // SnapshotScrapeGracePeriod bounds how long after the restic Job // completes the reconciler keeps retrying to read the snapshot id from // the pod logs before failing the Backup. Zero uses @@ -235,7 +239,7 @@ func resticImageOrDefault(image string) string { // +kubebuilder:rbac:groups="",resources=events,verbs=create;patch // +kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch // +kubebuilder:rbac:groups=snapshot.storage.k8s.io,resources=volumesnapshots,verbs=get;list;watch -// +kubebuilder:rbac:groups=snapshot.storage.k8s.io,resources=volumesnapshotcontents,verbs=get;list;watch +// +kubebuilder:rbac:groups=snapshot.storage.k8s.io,resources=volumesnapshotcontents,verbs=get;list;watch;delete func (r *BackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { return requeueOnConflict(r.reconcile(ctx, req)) @@ -250,6 +254,12 @@ func (r *BackupReconciler) reconcile(ctx context.Context, req ctrl.Request) (ctr if !b.DeletionTimestamp.IsZero() { return r.finalizeDelete(ctx, &b) } + if b.Annotations[backupCopyAbortAnnotation] != "" { + return r.abortResticCopy(ctx, &b, b.Annotations[backupCopyAbortAnnotation]) + } + if b.Annotations[backupSnapshotAbortAnnotation] == "true" { + return r.abortVolumeSnapshot(ctx, &b, "volume snapshot capture was aborted") + } // Every copy owns the source until its workers drain, even without quiesce. if !controllerutil.ContainsFinalizer(&b, gameplanev1alpha1.BackupFinalizer) { controllerutil.AddFinalizer(&b, gameplanev1alpha1.BackupFinalizer) @@ -292,6 +302,12 @@ func (r *BackupReconciler) reconcile(ctx context.Context, req ctrl.Request) (ctr // is outstanding, so this stays cheap on the common path. if b.Status.Phase == gameplanev1alpha1.BackupPhaseSucceeded || b.Status.Phase == gameplanev1alpha1.BackupPhaseFailed { + if b.Status.Phase == gameplanev1alpha1.BackupPhaseFailed && b.Spec.Strategy == "volume-snapshot" { + return r.abortVolumeSnapshot(ctx, &b, b.Status.Message) + } + if b.Status.Phase == gameplanev1alpha1.BackupPhaseFailed && (b.Annotations[backupPreparationDeadlineAnnotation] != "" || b.Annotations[backupSourceStartupDeadlineAnnotation] != "") { + return r.abortResticCopy(ctx, &b, b.Status.Message) + } return r.runUnquiesce(ctx, &b) } // Observe pre-upgrade workers even when multiple legacy quiesced copies @@ -357,54 +373,7 @@ func (r *BackupReconciler) reconcile(ctx context.Context, req ctrl.Request) (ctr b.Spec.RepoRef.Name, strings.Join(missing, ", "))) } - if res, err := r.prepareBackupQuiesce(ctx, &b); err != nil || res.RequeueAfter > 0 { - return res, err - } - // Admission refreshes the target. Resolve the pin only after owning the - // lifecycle fence, so a concurrent rollout cannot change the copy layout. - bound, err := r.ownedBackupTarget(ctx, &b) - if err != nil { - return ctrl.Result{}, err - } - if bound == nil { - return ctrl.Result{RequeueAfter: 5 * time.Second}, nil - } - // The template is only needed to construct a new worker. An existing - // copy must still reach completion and drain if its snapshot disappears. - worker, err := r.captureWorker(ctx, &b) - if errors.Is(err, errForeignBackupWorker) { - return r.waitForForeignBackupWorker(ctx, &b) - } - if err != nil { - return ctrl.Result{}, err - } - if job, ok := worker.(*batchv1.Job); ok { - return r.mirrorJobStatus(ctx, &b, job) - } - tmpl, err := effectiveGameTemplate(ctx, r.backupAPIReader(), bound) - if err != nil && (bound.Annotations[effectiveTemplateAnnotation] != "" || !apierrors.IsNotFound(err)) { - if _, failErr := r.fail(ctx, &b, "cannot resolve capture template: "+err.Error()); failErr != nil { - return ctrl.Result{}, failErr - } - return r.runUnquiesce(ctx, &b) - } - - backoff, deadline := resolveJobLimits(r.JobBackoffLimit, r.JobActiveDeadlineSeconds) - job := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: b.Name, Namespace: b.Namespace}} - if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, job, func() error { - if job.CreationTimestamp.IsZero() { - job.Spec.BackoffLimit = &backoff - job.Spec.ActiveDeadlineSeconds = &deadline - job.Spec.Template.Labels = map[string]string{backupRestoreJobLabel: backupRestoreJobValue, backupUIDLabel: string(b.UID)} - job.Spec.Template.Spec = r.buildBackupPodSpec(&b, tmpl) - job.Spec.Template.Spec.RestartPolicy = corev1.RestartPolicyNever - } - return controllerutil.SetControllerReference(&b, job, r.Scheme) - }); err != nil { - return ctrl.Result{}, err - } - - return r.mirrorJobStatus(ctx, &b, job) + return r.reconcilePreparedRestic(ctx, &b) } // maybeQuiesce calls the agent's /quiesce endpoint once per Backup. @@ -602,6 +571,9 @@ func (r *BackupReconciler) mirrorJobStatus( cond.Reason = string(phase) } newConds := upsertCondition(b.Status.Conditions, cond) + if phase == gameplanev1alpha1.BackupPhaseSucceeded || phase == gameplanev1alpha1.BackupPhaseFailed { + meta.RemoveStatusCondition(&newConds, "CaptureReady") + } newConds = upsertCondition(newConds, backupConsistencyCondition(b)) if !sameConditions(b.Status.Conditions, newConds) { b.Status.Conditions = newConds @@ -800,6 +772,11 @@ func (r *BackupReconciler) finalizeUnquiesce(ctx context.Context, b *gameplanev1 return ctrl.Result{}, err } } + if b.Annotations[backupSnapshotAbortAnnotation] == "true" { + if res, err := r.cleanupAbortedSnapshot(ctx, b); err != nil || res.RequeueAfter > 0 { + return res, err + } + } controllerutil.RemoveFinalizer(b, gameplanev1alpha1.BackupFinalizer) if err := r.Update(ctx, b); err != nil { return ctrl.Result{}, err @@ -979,6 +956,7 @@ func (r *BackupReconciler) readResticSummary(ctx context.Context, b *gameplanev1 func (r *BackupReconciler) fail(ctx context.Context, b *gameplanev1alpha1.Backup, msg string) (ctrl.Result, error) { now := metav1.Now() b.Status.Phase = gameplanev1alpha1.BackupPhaseFailed + meta.RemoveStatusCondition(&b.Status.Conditions, "CaptureReady") b.Status.ObservedGeneration = b.Generation if b.Status.Message != msg { b.Status.Message = msg diff --git a/operator/internal/controller/backup_csi_deadline_order_test.go b/operator/internal/controller/backup_csi_deadline_order_test.go new file mode 100644 index 000000000..9a7072312 --- /dev/null +++ b/operator/internal/controller/backup_csi_deadline_order_test.go @@ -0,0 +1,45 @@ +package controller + +import ( + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +func TestBackupExpiredCSIDeadlinePrecedesUnavailableWorkloadFence(t *testing.T) { + b := guardBackup("first") + b.Spec.Strategy = "volume-snapshot" + r, q := guardFixture(t, b) + guardPass(t, r, b.Name) + b = guardGetBackup(t, r, b.Name) + if q.quiesced != 1 || b.Annotations[backupCSIDeadlineAnnotation] == "" { + t.Fatalf("fixture never began a bounded CSI capture: %+v %+v", b, q) + } + b.Annotations[backupCSIDeadlineAnnotation] = time.Now().Add(-time.Minute).UTC().Format(time.RFC3339Nano) + if err := r.Update(t.Context(), b); err != nil { + t.Fatal(err) + } + if err := r.Delete(t.Context(), &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "server", Namespace: "ns"}}); err != nil { + t.Fatal(err) + } + if _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(b)}); err != nil { + t.Fatal(err) + } + b = guardGetBackup(t, r, b.Name) + if b.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || b.Annotations[backupSnapshotAbortAnnotation] != "true" || b.Annotations[backupSourceStartupDeadlineAnnotation] != "" || q.unquiesced != 1 { + t.Fatalf("expired CSI capture waited for a new source budget: %+v %+v", b, q) + } + var gs gameplanev1alpha1.GameServer + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server"}, &gs); err != nil { + t.Fatal(err) + } + if gs.Annotations[lifecycleOperationAnnotation] != "" || gs.Annotations[backupGuardAnnotation] != "" { + t.Fatal("expired CSI capture retained its owner after save-on") + } +} diff --git a/operator/internal/controller/backup_envtest_test.go b/operator/internal/controller/backup_envtest_test.go index f577563a6..e67328880 100644 --- a/operator/internal/controller/backup_envtest_test.go +++ b/operator/internal/controller/backup_envtest_test.go @@ -8,11 +8,9 @@ import ( "testing" "time" - appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" ) @@ -56,7 +54,7 @@ func TestBackup_CreatesJobWithExpectedSpec(t *testing.T) { if !strings.HasPrefix(c.Image, "restic/restic:") { t.Errorf("image = %q, want restic/restic:*", c.Image) } - if got, want := c.Args, []string{"backup", "/data", "--json", "--retry-lock", "10m", "--tag", "gameplane"}; !equalStrings(got, want) { + if got, want := c.Args, []string{resticWorkerScript, "backup-worker", "backup", "/data", "--json", "--retry-lock", "10m", "--tag", "gameplane"}; !equalStrings(got, want) { t.Errorf("args = %v, want %v", got, want) } @@ -365,6 +363,7 @@ func TestBackup_PassesTagsToRestic(t *testing.T) { return false, "job not yet created" } want := []string{"backup", "/data", "--json", "--retry-lock", "10m", "--tag", "gameplane", "--tag", "nightly", "--tag", "preupgrade"} + want = append([]string{resticWorkerScript, "backup-worker"}, want...) if !equalStrings(j.Spec.Template.Spec.Containers[0].Args, want) { return false, "args = " + strings.Join(j.Spec.Template.Spec.Containers[0].Args, " ") } @@ -372,29 +371,15 @@ func TestBackup_PassesTagsToRestic(t *testing.T) { }) } -// TestBackup_QuiesceAnnotationSetBeforeJob verifies that with a real -// agent client (faked) and spec.quiesce=true (the default), the -// controller calls Quiesce and stamps the quiesced-at annotation -// before the restic Job appears. -func TestBackup_QuiesceAnnotationSetBeforeJob(t *testing.T) { +// The Job must prepare before saving is disabled; authorization follows the +// durable quiesce annotation and cannot be granted to an unready worker. +func TestBackup_QuiesceAfterWorkerPreparation(t *testing.T) { ns := newNamespace(t) fa := &fakeAgent{} startMgr(t, ns, withBackupReconciler(backupReconcilerOpts{agent: fa})) + // The shared fixture supplies the observed workload and its scheduled, + // running source Pod; envtest has no StatefulSet controller or kubelet. seedGameServer(t, ns, "smp") - // envtest has no StatefulSet controller or kubelet: provide the observed - // running source whose immutable identity is bound before the agent RPC. - var workload appsv1.StatefulSet - if err := k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: "smp"}, &workload); err != nil { - t.Fatal(err) - } - pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "smp-0", Namespace: ns, OwnerReferences: []metav1.OwnerReference{{APIVersion: "apps/v1", Kind: "StatefulSet", Name: workload.Name, UID: workload.UID, Controller: ownerBoolPtr(true)}}}, Spec: workload.Spec.Template.Spec} - if err := k8sClient.Create(context.Background(), pod); err != nil { - t.Fatal(err) - } - pod.Status.Phase = corev1.PodRunning - if err := k8sClient.Status().Update(context.Background(), pod); err != nil { - t.Fatal(err) - } if err := k8sClient.Create(context.Background(), buildResticRepoSecret(ns, "repo")); err != nil { t.Fatalf("create secret: %v", err) @@ -404,6 +389,16 @@ func TestBackup_QuiesceAnnotationSetBeforeJob(t *testing.T) { if err := k8sClient.Create(context.Background(), b); err != nil { t.Fatalf("create backup: %v", err) } + var copyJob *batchv1.Job + eventually(t, func() (bool, string) { + var ok bool + copyJob, ok = getJob(t, ns, "smp-q") + return ok, "job not yet created" + }) + if fa.quiesceCount() != 0 { + t.Fatal("save-off happened before worker preparation") + } + prepareEnvtestBackupWorker(t, copyJob) eventually(t, func() (bool, string) { got := getBackup(t, ns, "smp-q") @@ -416,7 +411,7 @@ func TestBackup_QuiesceAnnotationSetBeforeJob(t *testing.T) { return true, "" }) - // Job exists and was created at-or-after the quiesce call. + // The prepared Job remains the authorized copy. eventually(t, func() (bool, string) { _, ok := getJob(t, ns, "smp-q") return ok, "job not yet created" diff --git a/operator/internal/controller/backup_forget.go b/operator/internal/controller/backup_forget.go index 2d97d9410..5d73b8170 100644 --- a/operator/internal/controller/backup_forget.go +++ b/operator/internal/controller/backup_forget.go @@ -112,6 +112,21 @@ func wantsSnapshotFinalizer(b *gameplanev1alpha1.Backup) bool { // forgets the restic snapshot. Unquiesce goes first because it is time // critical: a world left with auto-save off is worse than an orphaned snapshot. func (r *BackupReconciler) finalizeDelete(ctx context.Context, b *gameplanev1alpha1.Backup) (ctrl.Result, error) { + if b.Annotations[backupPreparationDeadlineAnnotation] != "" { + if err := r.revokeBackupStart(ctx, b); err != nil { + return ctrl.Result{}, err + } + } + if b.Spec.Strategy == "volume-snapshot" && b.Status.Phase != gameplanev1alpha1.BackupPhaseSucceeded { + if err := r.recordSnapshotAbort(ctx, b, "CSI capture cancelled by Backup deletion"); err != nil { + return ctrl.Result{}, err + } + } + if b.Annotations[backupPreparationDeadlineAnnotation] != "" && b.Status.Phase != gameplanev1alpha1.BackupPhaseSucceeded { + if res, err := r.abortResticCopy(ctx, b, "copy cancelled by Backup deletion"); err != nil || res.RequeueAfter > 0 { + return res, err + } + } if controllerutil.ContainsFinalizer(b, gameplanev1alpha1.BackupFinalizer) { res, err := r.finalizeUnquiesce(ctx, b) if err != nil || res.RequeueAfter > 0 { diff --git a/operator/internal/controller/backup_guard.go b/operator/internal/controller/backup_guard.go index 74d5839db..77832d77c 100644 --- a/operator/internal/controller/backup_guard.go +++ b/operator/internal/controller/backup_guard.go @@ -247,7 +247,25 @@ func (r *BackupReconciler) legacyQuiescedBackups(ctx context.Context, b *gamepla return legacy, nil } -func (r *BackupReconciler) prepareBackupQuiesce(ctx context.Context, b *gameplanev1alpha1.Backup) (ctrl.Result, error) { +// A fresh Backup waits for a predecessor's resource. Once admitted, losing its +// worker is terminal; a foreign replacement cannot suspend copy budgets or +// inherit capture authority. Cleanup still waits for this Backup's own workers. +func (r *BackupReconciler) waitForForeignBackupWorker(ctx context.Context, b *gameplanev1alpha1.Backup) (ctrl.Result, error) { + target, err := r.ownedBackupTarget(ctx, b) + if err != nil { + return ctrl.Result{}, err + } + if target == nil { + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + } + const message = "capture worker was replaced after lifecycle admission" + if b.Spec.Strategy == "volume-snapshot" { + return r.abortVolumeSnapshot(ctx, b, message) + } + return r.abortResticCopy(ctx, b, message) +} + +func (r *BackupReconciler) prepareBackupWorkload(ctx context.Context, b *gameplanev1alpha1.Backup) (ctrl.Result, error) { // A deleted Backup's dependent can outlive its finalizer. Wait before // acquiring ownership or pausing saving; never adopt another capture. if _, err := r.captureWorker(ctx, b); err != nil { @@ -275,10 +293,15 @@ func (r *BackupReconciler) prepareBackupQuiesce(ctx context.Context, b *gameplan return ctrl.Result{RequeueAfter: 5 * time.Second}, nil } fenced, err := fenceBackupWorkload(ctx, r.Client, r.backupAPIReader(), gs) - if err != nil || !fenced { + if err != nil { return ctrl.Result{RequeueAfter: 5 * time.Second}, err } - ready, err := r.prepareBackupSource(ctx, b, gs) + ready := false + if fenced { + ready, err = r.prepareBackupSource(ctx, b, gs) + } else { + _, err = r.waitForBackupSource(ctx, b, "source workload is absent or has not observed its fenced rollout") + } if errors.Is(err, errBackupSourceChanged) || errors.Is(err, errBackupSourceStartupTimeout) { _, err = r.fail(ctx, b, err.Error()) return ctrl.Result{RequeueAfter: time.Nanosecond}, err @@ -286,12 +309,23 @@ func (r *BackupReconciler) prepareBackupQuiesce(ctx context.Context, b *gameplan if err != nil || !ready { return ctrl.Result{RequeueAfter: 5 * time.Second}, err } + return ctrl.Result{}, nil +} + +func (r *BackupReconciler) prepareBackupQuiesce(ctx context.Context, b *gameplanev1alpha1.Backup) (ctrl.Result, error) { + if res, err := r.prepareBackupWorkload(ctx, b); err != nil || res.RequeueAfter > 0 { + return res, err + } + gs, err := r.ownedBackupTarget(ctx, b) + if err != nil || gs == nil { + return ctrl.Result{RequeueAfter: 5 * time.Second}, err + } if err := r.maybeQuiesce(ctx, b); err != nil { return ctrl.Result{}, err } // The Service RPC may overlap a pod replacement. Never start a copy on a // different process from the one pinned before the request. - ready, err = r.prepareBackupSource(ctx, b, gs) + ready, err := r.prepareBackupSource(ctx, b, gs) if errors.Is(err, errBackupSourceChanged) { _, err = r.fail(ctx, b, err.Error()) return ctrl.Result{RequeueAfter: time.Nanosecond}, err @@ -311,23 +345,6 @@ func (r *BackupReconciler) prepareBackupQuiesce(ctx context.Context, b *gameplan return ctrl.Result{}, nil } -// Fresh captures wait for a predecessor's dependent to disappear. A worker -// replaced after admission instead invalidates this capture; retrying forever -// would retain its ownership and possibly leave the game with saving paused. -func (r *BackupReconciler) waitForForeignBackupWorker(ctx context.Context, b *gameplanev1alpha1.Backup) (ctrl.Result, error) { - owned, err := r.ownedBackupTarget(ctx, b) - if err != nil { - return ctrl.Result{}, err - } - if owned == nil { - return ctrl.Result{RequeueAfter: 5 * time.Second}, nil - } - if _, err := r.fail(ctx, b, "capture worker was replaced by a foreign same-name worker"); err != nil { - return ctrl.Result{}, err - } - return r.runUnquiesce(ctx, b) -} - // Return only the originally bound GameServer. A foreign owner means this // Backup has no authority to call save-on or clear that owner's guard. func (r *BackupReconciler) ownedBackupTarget(ctx context.Context, b *gameplanev1alpha1.Backup) (*gameplanev1alpha1.GameServer, error) { @@ -379,6 +396,14 @@ func (r *BackupReconciler) releaseBackupTarget(ctx context.Context, b *gameplane // world while a deleting/failed Job still has a restic worker mounted. The // UID label keeps orphaned workers visible even after their Job disappears. func (r *BackupReconciler) backupWorkersLive(ctx context.Context, namespace string, id backupIdentity, cancel bool) (bool, error) { + if cancel { + // Recovery can run after the Backup disappeared. The immutable owner + // UID still identifies its gate, so revoke it before touching workers. + owner := &gameplanev1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Name: id.Name, Namespace: namespace, UID: id.UID}} + if err := r.revokeBackupStart(ctx, owner); err != nil { + return true, err + } + } var jobs batchv1.JobList if err := r.backupAPIReader().List(ctx, &jobs, client.InNamespace(namespace)); err != nil { return false, err @@ -421,13 +446,13 @@ func (r *BackupReconciler) backupWorkersLive(ctx context.Context, namespace stri if busy { return true, nil } - return r.backupSnapshotCopyLive(ctx, namespace, id) + return r.backupSnapshotCopyLive(ctx, namespace, id, cancel) } -// CSI captures have no Job or Pod. On normal deletion retain quiesce until -// capture completion/error instead of deleting the pending VolumeSnapshot; -// owner-reference GC will delete it after the Backup's finalizer clears. -func (r *BackupReconciler) backupSnapshotCopyLive(ctx context.Context, namespace string, id backupIdentity) (bool, error) { +// Normal captures remain live until ready/error. Cancellation first invalidates +// the Backup; orphan recovery marks its UID-bound snapshot invalid before +// save-on, since no surviving Backup can expose it as a restorable capture. +func (r *BackupReconciler) backupSnapshotCopyLive(ctx context.Context, namespace string, id backupIdentity, cancel bool) (bool, error) { scheme := r.Scheme if scheme == nil { scheme = r.Client.Scheme() @@ -455,6 +480,29 @@ func (r *BackupReconciler) backupSnapshotCopyLive(ctx context.Context, namespace if owner == nil || owner.Kind != "Backup" || owner.UID != id.UID { return false, nil } + if cancel { + if b.UID == id.UID && b.Status.Phase != gameplanev1alpha1.BackupPhaseSucceeded { + return false, r.recordSnapshotAbort(ctx, &b, "CSI capture cancelled during lifecycle recovery") + } + if b.UID != id.UID { + if vs.Annotations == nil { + vs.Annotations = map[string]string{} + } + if vs.Annotations[backupSnapshotAbortAnnotation] != "true" { + vs.Annotations[backupSnapshotAbortAnnotation] = "true" + if err := r.Update(ctx, &vs); err != nil { + return true, err + } + } + uid := vs.UID + if uid != "" && vs.DeletionTimestamp.IsZero() { + if err := r.Delete(ctx, &vs, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); err != nil && !apierrors.IsNotFound(err) { + ctrl.LoggerFrom(ctx).Error(err, "orphaned aborted snapshot requires reclamation", "snapshot", client.ObjectKeyFromObject(&vs), "uid", uid) + } + } + return false, nil + } + } if vs.Status != nil && ((vs.Status.ReadyToUse != nil && *vs.Status.ReadyToUse) || (vs.Status.Error != nil && vs.Status.Error.Message != nil)) { return false, nil } diff --git a/operator/internal/controller/backup_guard_test.go b/operator/internal/controller/backup_guard_test.go index 4782cc635..90e6ee14a 100644 --- a/operator/internal/controller/backup_guard_test.go +++ b/operator/internal/controller/backup_guard_test.go @@ -58,6 +58,12 @@ func guardBackup(name string) *gameplanev1alpha1.Backup { } func guardFixture(t *testing.T, backups ...*gameplanev1alpha1.Backup) (*BackupReconciler, *guardQuiescer) { + return guardFixtureWorkers(t, true, backups...) +} + +// Most guard tests model a healthy Job controller/kubelet. Budget tests opt +// out and advance worker scheduling/readiness explicitly. +func guardFixtureWorkers(t *testing.T, automatic bool, backups ...*gameplanev1alpha1.Backup) (*BackupReconciler, *guardQuiescer) { t.Helper() s := scrapeScheme(t) if err := snapshotv1.AddToScheme(s); err != nil { @@ -69,11 +75,36 @@ func guardFixture(t *testing.T, backups ...*gameplanev1alpha1.Backup) (*BackupRe objects := []client.Object{&gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "server", Namespace: "ns", UID: "server-uid"}, Spec: gameplanev1alpha1.GameServerSpec{TemplateRef: gameplanev1alpha1.GameTemplateRef{Name: "template"}}}, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "repo", Namespace: "ns"}, Data: map[string][]byte{"repo": []byte("repo"), "password": []byte("password")}}} one := int32(1) objects = append(objects, &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "server", Namespace: "ns", UID: "workload-uid", OwnerReferences: []metav1.OwnerReference{{APIVersion: gameplanev1alpha1.GroupVersion.String(), Kind: "GameServer", Name: "server", UID: "server-uid", Controller: ownerBoolPtr(true)}}}, Spec: appsv1.StatefulSetSpec{Replicas: &one}}) - objects = append(objects, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "server-0", Namespace: "ns", UID: "game-pod-uid", OwnerReferences: []metav1.OwnerReference{{APIVersion: "apps/v1", Kind: "StatefulSet", Name: "server", UID: "workload-uid", Controller: ownerBoolPtr(true)}}}, Status: corev1.PodStatus{Phase: corev1.PodRunning}}) + objects = append(objects, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "server-0", Namespace: "ns", UID: "game-pod-uid", OwnerReferences: []metav1.OwnerReference{{APIVersion: "apps/v1", Kind: "StatefulSet", Name: "server", UID: "workload-uid", Controller: ownerBoolPtr(true)}}}, Spec: corev1.PodSpec{NodeName: "node-a"}, Status: corev1.PodStatus{Phase: corev1.PodRunning}}) for _, b := range backups { objects = append(objects, b) } cl := fake.NewClientBuilder().WithScheme(s).WithObjects(objects...).WithStatusSubresource(&gameplanev1alpha1.Backup{}, &gameplanev1alpha1.GameServer{}, &batchv1.Job{}, &snapshotv1.VolumeSnapshot{}).Build() + cl = interceptor.NewClient(cl, interceptor.Funcs{Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if obj.GetUID() == "" { + obj.SetUID(types.UID(obj.GetName() + "-generated-uid")) + } + if err := c.Create(ctx, obj, opts...); err != nil { + return err + } + job, ok := obj.(*batchv1.Job) + if !automatic || !ok || job.Annotations[backupWorkerProtocolAnnotation] != "v1" { + return nil + } + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: job.Name + "-automatic", Namespace: job.Namespace, UID: types.UID(job.Name + "-automatic-uid"), Labels: job.Spec.Template.Labels, + OwnerReferences: []metav1.OwnerReference{{APIVersion: "batch/v1", Kind: "Job", Name: job.Name, UID: job.UID, Controller: ownerBoolPtr(true)}}}, Spec: job.Spec.Template.Spec, + Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}}} + pod.Spec.NodeName = "node-a" + return c.Create(ctx, pod) + }, Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if err := c.Delete(ctx, obj, opts...); err != nil { + return err + } + if job, ok := obj.(*batchv1.Job); automatic && ok { + return client.IgnoreNotFound(c.Delete(ctx, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: job.Name + "-automatic", Namespace: job.Namespace}})) + } + return nil + }}) q := &guardQuiescer{} return &BackupReconciler{Client: cl, Scheme: s, AgentClient: q, LogReader: scrapeLogReader{body: `{"message_type":"summary","snapshot_id":"abc123"}`}}, q } @@ -81,6 +112,14 @@ func guardFixture(t *testing.T, backups ...*gameplanev1alpha1.Backup) (*BackupRe func guardPass(t *testing.T, r *BackupReconciler, name string) { t.Helper() for i := 0; i < 3; i++ { + // Model a completed worker after tests advance the Job's terminal + // status. Explicitly created live/terminating pods are never removed. + var job batchv1.Job + if err := r.Get(context.Background(), client.ObjectKey{Namespace: "ns", Name: name}, &job); err == nil && (job.Status.Succeeded > 0 || jobPermanentlyFailed(&job)) { + if err := r.Delete(context.Background(), &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "ns", Name: name + "-automatic"}}); err != nil && !apierrors.IsNotFound(err) { + t.Fatal(err) + } + } if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "ns", Name: name}}); err != nil { t.Fatal(err) } @@ -541,7 +580,7 @@ func TestBackupGuardRecoveryLineageSurvivesRepeatedOwnerDeletion(t *testing.T) { } } -func TestBackupGuardPendingSnapshotCleanupWaitsForCapture(t *testing.T) { +func TestBackupGuardPendingSnapshotCancellationInvalidatesBeforeSaveOn(t *testing.T) { for _, mode := range []string{"delete", "orphan", "legacy"} { t.Run(mode, func(t *testing.T) { a := guardBackup("snapshot") @@ -562,6 +601,19 @@ func TestBackupGuardPendingSnapshotCleanupWaitsForCapture(t *testing.T) { guardPass(t, r, "snapshot") } guardPass(t, r, "next") + if mode != "legacy" { + if q.unquiesced != 1 { + t.Fatalf("cancelled CSI capture did not restore saving: %+v", q) + } + var jobs batchv1.JobList + if err := r.List(context.Background(), &jobs); err != nil { + t.Fatal(err) + } + if len(jobs.Items) != 1 { + t.Fatal("next copy did not start after explicit CSI abort") + } + return + } if q.unquiesced != 0 { t.Fatalf("save-on during pending CSI capture: %+v", q) } @@ -635,11 +687,8 @@ func TestBackupSnapshotAmbiguousCreateStillCleansUp(t *testing.T) { t.Fatalf("ambiguous create lost abort cleanup: %+v", q) } var vs snapshotv1.VolumeSnapshot - if err := r.Get(context.Background(), types.NamespacedName{Namespace: "ns", Name: "snapshot"}, &vs); err != nil { - t.Fatal(err) - } - if vs.Status != nil && vs.Status.ReadyToUse != nil && *vs.Status.ReadyToUse { - t.Fatal("expected still-pending ambiguous capture") + if err := r.Get(context.Background(), types.NamespacedName{Namespace: "ns", Name: "snapshot"}, &vs); !apierrors.IsNotFound(err) { + t.Fatalf("ambiguous aborted capture was not reclaimed: %v", err) } } diff --git a/operator/internal/controller/backup_manifest.go b/operator/internal/controller/backup_manifest.go index bdc97ee99..2ba50f7e6 100644 --- a/operator/internal/controller/backup_manifest.go +++ b/operator/internal/controller/backup_manifest.go @@ -32,8 +32,8 @@ func (r *BackupReconciler) captureWorker(ctx context.Context, b *gameplanev1alph return worker, nil } -// Called after lifecycle ownership and the workload fence, before creating the -// copy worker. Requeues and later template edits never rewrite this manifest. +// Called after lifecycle ownership and the workload fence, before a copy is +// authorized. Requeues and later template edits never rewrite this manifest. func (r *BackupReconciler) recordBackupManifest(ctx context.Context, b *gameplanev1alpha1.Backup, gs *gameplanev1alpha1.GameServer) error { if b.Status.CaptureManifest != nil { return nil @@ -44,10 +44,25 @@ func (r *BackupReconciler) recordBackupManifest(ctx context.Context, b *gameplan return err } if worker != nil { - // Existing workers predate this metadata. Current layout cannot prove - // what they captured, even if their requested quiesce was successful. - b.Status.CaptureManifest = &gameplanev1alpha1.BackupCaptureManifest{State: "LegacyUnknown"} - return r.Status().Update(ctx, b) + // Current prepared workers exist before capture but cannot read data + // without authorization. Its durable annotation is written before the + // projected token and never cleared, including during cancellation. + // All other existing workers retain unknown historical coverage. + waiting := false + if job, ok := worker.(*batchv1.Job); ok && metav1.IsControlledBy(job, b) && job.Annotations[backupWorkerProtocolAnnotation] == "v1" && + b.Annotations[backupPreparationDeadlineAnnotation] != "" && b.Annotations[backupAuthorizedWorkerAnnotation] == "" && + job.Status.Succeeded == 0 && !jobPermanentlyFailed(job) { + var gate corev1.ConfigMap + if err := reader.Get(ctx, client.ObjectKey{Namespace: b.Namespace, Name: backupStartConfigMapName(b)}, &gate); err == nil { + waiting = metav1.IsControlledBy(&gate, b) && gate.Data[backupStartTokenKey] == "" + } else if !apierrors.IsNotFound(err) { + return err + } + } + if !waiting { + b.Status.CaptureManifest = &gameplanev1alpha1.BackupCaptureManifest{State: "LegacyUnknown"} + return r.Status().Update(ctx, b) + } } var ss appsv1.StatefulSet err = reader.Get(ctx, client.ObjectKeyFromObject(gs), &ss) diff --git a/operator/internal/controller/backup_manifest_test.go b/operator/internal/controller/backup_manifest_test.go index cf58d17c6..cb63d493e 100644 --- a/operator/internal/controller/backup_manifest_test.go +++ b/operator/internal/controller/backup_manifest_test.go @@ -142,6 +142,46 @@ func TestBackupManifestDoesNotInventHistoryForExistingWorker(t *testing.T) { } } +func TestBackupManifestDoesNotInventHistoryForAuthorizedPreparedWorker(t *testing.T) { + for _, mode := range []string{"authorized", "projected-token", "missing-gate", "foreign-gate"} { + t.Run(mode, func(t *testing.T) { + b := guardBackup("copy") + b.Annotations = map[string]string{backupPreparationDeadlineAnnotation: "persisted-before-preparation"} + r, _ := guardFixtureWorkers(t, false, b) + job := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: b.Name, Namespace: b.Namespace, Annotations: map[string]string{backupWorkerProtocolAnnotation: "v1"}, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(b, gameplanev1alpha1.GroupVersion.WithKind("Backup"))}}} + if err := r.Create(t.Context(), job); err != nil { + t.Fatal(err) + } + gate := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: backupStartConfigMapName(b), Namespace: b.Namespace, + OwnerReferences: job.OwnerReferences}, Data: map[string]string{backupStartTokenKey: ""}} + if mode == "authorized" { + b.Annotations[backupAuthorizedWorkerAnnotation] = "previous-worker" + if err := r.Update(t.Context(), b); err != nil { + t.Fatal(err) + } + } + if mode == "projected-token" { + gate.Data[backupStartTokenKey] = "previous-authorization" + } + if mode == "foreign-gate" { + gate.OwnerReferences = nil + } + if mode != "missing-gate" { + if err := r.Create(t.Context(), gate); err != nil { + t.Fatal(err) + } + } + if err := r.recordBackupManifest(t.Context(), b, &gameplanev1alpha1.GameServer{}); err != nil { + t.Fatal(err) + } + if b.Status.CaptureManifest == nil || b.Status.CaptureManifest.State != "LegacyUnknown" || len(b.Status.CaptureManifest.Volumes) != 0 { + t.Fatalf("invented history for potentially started copy: %+v", b.Status.CaptureManifest) + } + }) + } +} + func TestBackupConsistencyDescribesAchievedResult(t *testing.T) { for _, tc := range []struct { strategy, outcome, reason string diff --git a/operator/internal/controller/backup_preparation.go b/operator/internal/controller/backup_preparation.go index cd2305c6f..be7cfecc8 100644 --- a/operator/internal/controller/backup_preparation.go +++ b/operator/internal/controller/backup_preparation.go @@ -19,8 +19,9 @@ import ( ) const ( - backupPreparationAnnotation = "backup.gameplane.local/preparation" - backupCleanupOutcomeAnnotation = "backup.gameplane.local/cleanup-outcome" + backupPreparationAnnotation = "backup.gameplane.local/preparation" + backupCleanupOutcomeAnnotation = "backup.gameplane.local/cleanup-outcome" + backupSourceStartupDeadlineAnnotation = "backup.gameplane.local/source-startup-deadline" ) var errBackupSourceChanged = errors.New("backup source changed after preparation") @@ -31,13 +32,26 @@ var errBackupSourceStartupTimeout = errors.New("backup source startup grace peri // indefinitely when an initial live source never becomes Running. const backupSourceStartupGracePeriod = 2 * time.Minute -// waitForBackupSource bounds initial Pod admission using durable API creation -// time, so operator restarts and reconcile retries cannot restart the grace. -func waitForBackupSource(b *gameplanev1alpha1.Backup, reason string) (bool, error) { - if time.Since(b.CreationTimestamp.Time) < backupSourceStartupGracePeriod { +// Call only after lifecycle ownership. Prepared workers share their existing +// preparation budget; CSI and older Jobs persist a separate startup deadline. +// Queue time and operator restarts never spend or reset an admitted budget. +func (r *BackupReconciler) waitForBackupSource(ctx context.Context, b *gameplanev1alpha1.Backup, reason string) (bool, error) { + key := backupSourceStartupDeadlineAnnotation + if b.Annotations[backupPreparationDeadlineAnnotation] != "" { + key = backupPreparationDeadlineAnnotation + } + stored := b.Annotations[key] != "" + deadline, err := r.backupDeadline(ctx, b, key, backupSourceStartupGracePeriod) + if err != nil { + if stored { + return false, fmt.Errorf("%w: cannot read source startup deadline: %w", errBackupSourceStartupTimeout, err) + } + return false, err // A failed first persistence is retryable, not expiry. + } + if time.Now().Before(deadline) { return false, nil } - return false, fmt.Errorf("%w after %s: %s", errBackupSourceStartupTimeout, backupSourceStartupGracePeriod, reason) + return false, fmt.Errorf("%w at %s: %s", errBackupSourceStartupTimeout, deadline.UTC().Format(time.RFC3339), reason) } type podAgentQuiescer interface { @@ -162,7 +176,7 @@ func (r *BackupReconciler) prepareBackupSource(ctx context.Context, b *gameplane if p != nil { return false, fmt.Errorf("%w: original pod absent", errBackupSourceChanged) } - return waitForBackupSource(b, "source pod absent") + return r.waitForBackupSource(ctx, b, "source pod absent") } if err != nil { return false, err @@ -174,7 +188,7 @@ func (r *BackupReconciler) prepareBackupSource(ctx context.Context, b *gameplane return false, fmt.Errorf("backup source pod is not owned by the bound workload") } if !pod.DeletionTimestamp.IsZero() || pod.Status.Phase != corev1.PodRunning { - return waitForBackupSource(b, "source pod is not Running or is terminating") + return r.waitForBackupSource(ctx, b, "source pod is not Running or is terminating") } candidate.Outcome, candidate.PodUID = "live", pod.UID } diff --git a/operator/internal/controller/backup_preparation_test.go b/operator/internal/controller/backup_preparation_test.go index c69ce0d02..1d7548017 100644 --- a/operator/internal/controller/backup_preparation_test.go +++ b/operator/internal/controller/backup_preparation_test.go @@ -1,6 +1,8 @@ package controller import ( + "context" + "errors" "strings" "testing" "time" @@ -12,6 +14,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" ) @@ -27,10 +30,8 @@ func TestBackupInitialSourceStartupGrace(t *testing.T) { } t.Run(name, func(t *testing.T) { first := guardBackup("first") - first.CreationTimestamp = metav1.Now() - if expired { - first.CreationTimestamp = metav1.NewTime(time.Now().Add(-3 * time.Minute)) - } + // Queue age must not spend the preparation budget before ownership. + first.CreationTimestamp = metav1.NewTime(time.Now().Add(-3 * time.Minute)) r, q := guardFixture(t, first, guardBackup("second")) ctx := t.Context() podKey := client.ObjectKey{Namespace: "ns", Name: "server-0"} @@ -61,10 +62,21 @@ func TestBackupInitialSourceStartupGrace(t *testing.T) { if err != nil || result.RequeueAfter <= 0 { t.Fatalf("source admission did not requeue: %+v, %v", result, err) } + first = guardGetBackup(t, r, "first") + deadline := first.Annotations[backupPreparationDeadlineAnnotation] + if first.Status.Phase == gameplanev1alpha1.BackupPhaseFailed || deadline == "" { + t.Fatalf("queued source lost its fresh admission budget: %+v", first) + } + if expired { + first.Annotations[backupPreparationDeadlineAnnotation] = time.Now().Add(-time.Minute).UTC().Format(time.RFC3339Nano) + if err := r.Update(ctx, first); err != nil { + t.Fatal(err) + } + } guardPass(t, r, "first") first = guardGetBackup(t, r, "first") if expired { - if first.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || first.Status.CompletionTime == nil || !strings.Contains(first.Status.Message, "startup grace period expired") { + if first.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || first.Status.CompletionTime == nil || !strings.Contains(first.Status.Message, "preparation deadline exceeded") { t.Fatalf("expired source did not fail visibly: %+v", first.Status) } } else if first.Status.Phase == gameplanev1alpha1.BackupPhaseFailed { @@ -141,6 +153,90 @@ func TestBackupInitialSourceStartupGrace(t *testing.T) { } } +func TestBackupUngatedSourceStartupDeadlineSurvivesRestart(t *testing.T) { + for _, mode := range []string{"csi-pod", "csi-workload", "legacy-job"} { + t.Run(mode, func(t *testing.T) { + b := guardBackup("first") + if mode != "legacy-job" { + b.Spec.Strategy = "volume-snapshot" + } + b.CreationTimestamp = metav1.NewTime(time.Now().Add(-time.Hour)) + r, q := guardFixture(t, b) + if err := r.Delete(t.Context(), &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "server-0", Namespace: "ns"}}); err != nil { + t.Fatal(err) + } + if mode == "csi-workload" { + if err := r.Delete(t.Context(), &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "server", Namespace: "ns"}}); err != nil { + t.Fatal(err) + } + } + if mode == "legacy-job" { + job := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: b.Name, Namespace: b.Namespace, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(b, gameplanev1alpha1.GroupVersion.WithKind("Backup"))}}} + if err := r.Create(t.Context(), job); err != nil { + t.Fatal(err) + } + } + guardPass(t, r, b.Name) + b = guardGetBackup(t, r, b.Name) + deadline := b.Annotations[backupSourceStartupDeadlineAnnotation] + if _, err := time.Parse(time.RFC3339Nano, deadline); err != nil || b.Status.Phase == gameplanev1alpha1.BackupPhaseFailed || q.quiesced != 0 { + t.Fatalf("source did not receive a fresh owned startup budget: %v %+v", err, b) + } + if b.Annotations[backupPreparationDeadlineAnnotation] != "" { + t.Fatal("ungated capture unexpectedly acquired a prepared-worker deadline") + } + r = &BackupReconciler{Client: r.Client, Scheme: r.Scheme, AgentClient: q} + guardPass(t, r, b.Name) + b = guardGetBackup(t, r, b.Name) + if b.Annotations[backupSourceStartupDeadlineAnnotation] != deadline { + t.Fatal("operator restart extended the initial source deadline") + } + var legacyWorker *corev1.Pod + if mode == "legacy-job" { + // Model a dependent that can outlive foreground Job deletion. + legacyWorker = &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "legacy-worker", Namespace: "ns", Labels: map[string]string{backupUIDLabel: string(b.UID)}}, + Status: corev1.PodStatus{Phase: corev1.PodRunning}} + if err := r.Create(t.Context(), legacyWorker); err != nil { + t.Fatal(err) + } + } + b.Annotations[backupSourceStartupDeadlineAnnotation] = time.Now().Add(-time.Minute).UTC().Format(time.RFC3339Nano) + if err := r.Update(t.Context(), b); err != nil { + t.Fatal(err) + } + guardPass(t, r, b.Name) + b = guardGetBackup(t, r, b.Name) + if b.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || !strings.Contains(b.Status.Message, "startup grace period expired") || q.quiesced != 0 || q.unquiesced != 0 { + t.Fatalf("startup deadline did not fail before capture: %+v %+v", b.Status, q) + } + var gs gameplanev1alpha1.GameServer + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server"}, &gs); err != nil { + t.Fatal(err) + } + if legacyWorker != nil { + var job batchv1.Job + if err := r.Get(t.Context(), client.ObjectKeyFromObject(b), &job); !apierrors.IsNotFound(err) { + t.Fatalf("expired legacy startup did not cancel its Job: %v", err) + } + if gs.Annotations[lifecycleOperationAnnotation] == "" || b.Annotations[backupCopyAbortAnnotation] == "" { + t.Fatal("expired legacy startup released ownership before its worker drained") + } + if err := r.Delete(t.Context(), legacyWorker); err != nil { + t.Fatal(err) + } + guardPass(t, r, b.Name) + if err := r.Get(t.Context(), client.ObjectKeyFromObject(&gs), &gs); err != nil { + t.Fatal(err) + } + } + if gs.Annotations[lifecycleOperationAnnotation] != "" || gs.Annotations[backupGuardAnnotation] != "" { + t.Fatal("expired startup retained lifecycle ownership") + } + }) + } +} + func TestBackupHealthySourceAdmittedAfterStartupGrace(t *testing.T) { b := guardBackup("first") b.CreationTimestamp = metav1.NewTime(time.Now().Add(-3 * time.Minute)) @@ -154,3 +250,32 @@ func TestBackupHealthySourceAdmittedAfterStartupGrace(t *testing.T) { t.Fatalf("ready source was not quiesced: %+v", q) } } + +func TestBackupSourceStartupDeadlinePersistenceFailureRetries(t *testing.T) { + b := guardBackup("first") + b.Spec.Strategy = "volume-snapshot" + r, q := guardFixture(t, b) + if err := r.Delete(t.Context(), &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "server-0", Namespace: "ns"}}); err != nil { + t.Fatal(err) + } + base := r.Client.(client.WithWatch) + failure := apierrors.NewServiceUnavailable("deadline persistence unavailable") + r.Client = interceptor.NewClient(base, interceptor.Funcs{Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + if backup, ok := obj.(*gameplanev1alpha1.Backup); ok && backup.Annotations[backupSourceStartupDeadlineAnnotation] != "" { + return failure + } + return c.Update(ctx, obj, opts...) + }}) + if _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(b)}); !errors.Is(err, failure) { + t.Fatalf("failed first deadline persistence was not retried: %v", err) + } + b = guardGetBackup(t, r, b.Name) + if b.Status.Phase == gameplanev1alpha1.BackupPhaseFailed || b.Annotations[backupSourceStartupDeadlineAnnotation] != "" || q.quiesced != 0 { + t.Fatalf("write failure was treated as admitted expiry: %+v %+v", b, q) + } + r.Client = base + guardPass(t, r, b.Name) + if got := guardGetBackup(t, r, b.Name); got.Status.Phase == gameplanev1alpha1.BackupPhaseFailed || got.Annotations[backupSourceStartupDeadlineAnnotation] == "" { + t.Fatalf("healthy retry did not persist its startup budget: %+v", got) + } +} diff --git a/operator/internal/controller/backup_snapshot_cleanup.go b/operator/internal/controller/backup_snapshot_cleanup.go new file mode 100644 index 000000000..6e07b7c0c --- /dev/null +++ b/operator/internal/controller/backup_snapshot_cleanup.go @@ -0,0 +1,215 @@ +package controller + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +const backupSnapshotIdentityAnnotation = "backup.gameplane.local/snapshot-identity" +const backupSnapshotAbortMessageAnnotation = "backup.gameplane.local/snapshot-abort-reason" + +type backupSnapshotIdentity struct { + UID types.UID `json:"uid"` + Contents map[string]types.UID `json:"contents,omitempty"` +} + +func (r *BackupReconciler) rememberBackupSnapshot(ctx context.Context, b *gameplanev1alpha1.Backup, vs *snapshotv1.VolumeSnapshot) error { + var id backupSnapshotIdentity + if value := b.Annotations[backupSnapshotIdentityAnnotation]; value != "" { + if err := json.Unmarshal([]byte(value), &id); err != nil { + return fmt.Errorf("invalid snapshot identity: %w", err) + } + } + if !metav1.IsControlledBy(vs, b) || (id.UID != "" && id.UID != vs.UID) { + return fmt.Errorf("VolumeSnapshot %s/%s identity changed", vs.Namespace, vs.Name) + } + if id.UID == vs.UID { + return nil + } + id.UID = vs.UID + encoded, err := json.Marshal(id) + if err != nil { + return err + } + patchBackupAnnotations(b, map[string]string{backupSnapshotIdentityAnnotation: string(encoded)}) + return r.Update(ctx, b) +} + +func (r *BackupReconciler) recordSnapshotAbort(ctx context.Context, b *gameplanev1alpha1.Backup, message string) error { + if b.Annotations[backupSnapshotAbortAnnotation] != "true" { + patchBackupAnnotations(b, map[string]string{backupSnapshotAbortAnnotation: "true", backupSnapshotAbortMessageAnnotation: message}) + if err := r.Update(ctx, b); err != nil { + return err + } + } + if stored := b.Annotations[backupSnapshotAbortMessageAnnotation]; stored != "" { + message = stored + } + if b.Status.Phase == gameplanev1alpha1.BackupPhaseFailed && b.Status.SnapshotID == "" && b.Status.VolumeSnapshotContentName == "" && b.Status.Message == message { + return nil + } + b.Status.SnapshotID, b.Status.VolumeSnapshotContentName = "", "" + _, err := r.fail(ctx, b, message) + return err +} + +func (r *BackupReconciler) snapshotCleanupCondition(ctx context.Context, b *gameplanev1alpha1.Backup, status metav1.ConditionStatus, reason, message string) error { + conditions := upsertCondition(b.Status.Conditions, metav1.Condition{Type: "SnapshotCleaned", Status: status, Reason: reason, Message: message, ObservedGeneration: b.Generation}) + if sameConditions(b.Status.Conditions, conditions) { + return nil + } + b.Status.Conditions = conditions + if status != metav1.ConditionTrue { + r.recordEvent(b, corev1.EventTypeWarning, reason, message) + } + return r.Status().Update(ctx, b) +} + +// Cleanup is independent of save-on once the capture is durably invalidated. +// The original snapshot UID and each content UID are retained on the Backup; +// resource replacements are reported, never adopted or deleted. +func (r *BackupReconciler) cleanupAbortedSnapshot(ctx context.Context, b *gameplanev1alpha1.Backup) (ctrl.Result, error) { + res, err := r.reclaimAbortedSnapshot(ctx, b) + if err == nil { + return res, nil + } + if conditionErr := r.snapshotCleanupCondition(ctx, b, metav1.ConditionFalse, "SnapshotCleanupPending", err.Error()); conditionErr != nil { + return ctrl.Result{}, conditionErr + } + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil +} + +func (r *BackupReconciler) reclaimAbortedSnapshot(ctx context.Context, b *gameplanev1alpha1.Backup) (ctrl.Result, error) { + var id backupSnapshotIdentity + if value := b.Annotations[backupSnapshotIdentityAnnotation]; value != "" { + if err := json.Unmarshal([]byte(value), &id); err != nil { + return ctrl.Result{}, err + } + } + var vs snapshotv1.VolumeSnapshot + err := r.backupAPIReader().Get(ctx, client.ObjectKeyFromObject(b), &vs) + if meta.IsNoMatchError(err) { + return ctrl.Result{}, fmt.Errorf("snapshot API unavailable; inspect VolumeSnapshot %s/%s and recorded identity %s", b.Namespace, b.Name, b.Annotations[backupSnapshotIdentityAnnotation]) + } + if err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, fmt.Errorf("read VolumeSnapshot %s/%s (recorded UID %s): %w", b.Namespace, b.Name, id.UID, err) + } + present := err == nil + if present { + if !metav1.IsControlledBy(&vs, b) || (id.UID != "" && id.UID != vs.UID) { + return ctrl.Result{}, r.snapshotCleanupCondition(ctx, b, metav1.ConditionFalse, "SnapshotCleanupUnverified", fmt.Sprintf("refusing replacement VolumeSnapshot %s/%s UID %s; original UID %s", vs.Namespace, vs.Name, vs.UID, id.UID)) + } + if id.UID == "" { + if vs.UID == "" { + return ctrl.Result{}, fmt.Errorf("VolumeSnapshot %s/%s has no verifiable UID", vs.Namespace, vs.Name) + } + if err := r.rememberBackupSnapshot(ctx, b, &vs); err != nil { + return ctrl.Result{}, err + } + id.UID = vs.UID + } + if vs.Annotations == nil { + vs.Annotations = map[string]string{} + } + if vs.Annotations[backupSnapshotAbortAnnotation] != "true" { + vs.Annotations[backupSnapshotAbortAnnotation] = "true" + if err := r.Update(ctx, &vs); err != nil { + return ctrl.Result{}, err + } + } + } + var contents snapshotv1.VolumeSnapshotContentList + if err := r.backupAPIReader().List(ctx, &contents); err != nil { + return ctrl.Result{}, fmt.Errorf("list contents for VolumeSnapshot %s/%s UID %s: %w", b.Namespace, b.Name, id.UID, err) + } + if id.Contents == nil { + id.Contents = map[string]types.UID{} + } + var retained []string + unverified := false + boundName := "" + if present && vs.Status != nil && vs.Status.BoundVolumeSnapshotContentName != nil { + boundName = *vs.Status.BoundVolumeSnapshotContentName + } + var owned []*snapshotv1.VolumeSnapshotContent + for i := range contents.Items { + content := &contents.Items[i] + ref := content.Spec.VolumeSnapshotRef + if known, ok := id.Contents[content.Name]; ok && known != content.UID { + unverified = true + retained = append(retained, fmt.Sprintf("replacement content %s UID %s (original %s)", content.Name, content.UID, known)) + continue + } + if id.UID == "" || ref.UID != id.UID || ref.Name != b.Name || ref.Namespace != b.Namespace { + if _, known := id.Contents[content.Name]; known || content.Name == boundName { + // Retain the rejected candidate's UID so deleting the valid + // snapshot cannot erase the administrator's follow-up identity. + id.Contents[content.Name] = content.UID + unverified = true + retained = append(retained, fmt.Sprintf("unverified content %s UID %s references snapshot %s/%s UID %s", content.Name, content.UID, ref.Namespace, ref.Name, ref.UID)) + } + continue + } + id.Contents[content.Name] = content.UID + if content.Spec.DeletionPolicy == snapshotv1.VolumeSnapshotContentRetain { + retained = append(retained, fmt.Sprintf("Retain content %s UID %s for snapshot UID %s", content.Name, content.UID, id.UID)) + } else { + owned = append(owned, content) + } + } + encoded, err := json.Marshal(id) + if err != nil { + return ctrl.Result{}, err + } + if string(encoded) != b.Annotations[backupSnapshotIdentityAnnotation] { + patchBackupAnnotations(b, map[string]string{backupSnapshotIdentityAnnotation: string(encoded)}) + if err := r.Update(ctx, b); err != nil { + return ctrl.Result{}, err + } + } + if present && vs.DeletionTimestamp.IsZero() { + uid := vs.UID + if err := r.Delete(ctx, &vs, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, fmt.Errorf("delete VolumeSnapshot %s/%s UID %s: %w", vs.Namespace, vs.Name, uid, err) + } + } + for _, content := range owned { + if !content.DeletionTimestamp.IsZero() { + continue + } + uid := content.UID + if uid == "" { + return ctrl.Result{}, fmt.Errorf("content %s has no verifiable UID", content.Name) + } + if err := r.Delete(ctx, content, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, fmt.Errorf("delete VolumeSnapshotContent %s UID %s for snapshot UID %s: %w", content.Name, uid, id.UID, err) + } + } + if len(retained) > 0 { + if unverified { + return ctrl.Result{}, r.snapshotCleanupCondition(ctx, b, metav1.ConditionFalse, "SnapshotCleanupUnverified", strings.Join(retained, "; ")+"; administrator verification required") + } + return ctrl.Result{}, r.snapshotCleanupCondition(ctx, b, metav1.ConditionFalse, "SnapshotCleanupRetained", strings.Join(retained, "; ")+"; administrator reclamation required") + } + if present || len(owned) > 0 { + if err := r.snapshotCleanupCondition(ctx, b, metav1.ConditionFalse, "SnapshotCleanupPending", fmt.Sprintf("deletion requested for snapshot %s/%s UID %s and contents %v", b.Namespace, b.Name, id.UID, id.Contents)); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } + return ctrl.Result{}, r.snapshotCleanupCondition(ctx, b, metav1.ConditionTrue, "SnapshotCleanupComplete", "aborted snapshot resources are absent") +} diff --git a/operator/internal/controller/backup_snapshot_source.go b/operator/internal/controller/backup_snapshot_source.go new file mode 100644 index 000000000..160619c22 --- /dev/null +++ b/operator/internal/controller/backup_snapshot_source.go @@ -0,0 +1,49 @@ +package controller + +import ( + "context" + "encoding/json" + "fmt" + + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +// A raw GameServer dataSource must not revive an aborted operator capture while +// a CSI driver still exposes its object. Run only before creating a new PVC: +// existing volumes remain usable after retention deletes their source Backup. +// External snapshots without a Gameplane Backup owner keep the direct-CRD +// administrator contract; API authorization is enforced separately. +func validateBackupSnapshotSource(ctx context.Context, reader client.Reader, namespace, name string) error { + var snapshot snapshotv1.VolumeSnapshot + if err := reader.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, &snapshot); err != nil { + return fmt.Errorf("resolve snapshot source: %w", err) + } + if snapshot.Annotations[backupSnapshotAbortAnnotation] == "true" { + return fmt.Errorf("snapshot %s was aborted and cannot seed a volume", name) + } + owner := metav1.GetControllerOf(&snapshot) + if owner == nil || owner.APIVersion != gameplanev1alpha1.GroupVersion.String() || owner.Kind != "Backup" { + return nil + } + var backup gameplanev1alpha1.Backup + if err := reader.Get(ctx, client.ObjectKey{Namespace: namespace, Name: owner.Name}, &backup); err != nil { + return fmt.Errorf("resolve snapshot Backup owner: %w", err) + } + if owner.UID == "" || owner.UID != backup.UID || !backup.DeletionTimestamp.IsZero() || !snapshot.DeletionTimestamp.IsZero() || + backup.Annotations[backupSnapshotAbortAnnotation] == "true" || backup.Annotations[backupCopyAbortAnnotation] != "" || + backup.Status.Phase != gameplanev1alpha1.BackupPhaseSucceeded || backup.Status.SnapshotID != snapshot.Name || + snapshot.Status == nil || snapshot.Status.ReadyToUse == nil || !*snapshot.Status.ReadyToUse { + return fmt.Errorf("snapshot %s is not a completed, valid capture of its current Backup owner", name) + } + if value := backup.Annotations[backupSnapshotIdentityAnnotation]; value != "" { + var id backupSnapshotIdentity + if err := json.Unmarshal([]byte(value), &id); err != nil || id.UID == "" || id.UID != snapshot.UID { + return fmt.Errorf("snapshot %s differs from the recorded capture UID", name) + } + } + return nil +} diff --git a/operator/internal/controller/backup_snapshot_source_test.go b/operator/internal/controller/backup_snapshot_source_test.go new file mode 100644 index 000000000..2daa8f82d --- /dev/null +++ b/operator/internal/controller/backup_snapshot_source_test.go @@ -0,0 +1,69 @@ +package controller + +import ( + "encoding/json" + "testing" + + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +func TestSnapshotPVCRefusesIncompleteAbortedAndReplacedCapture(t *testing.T) { + for _, mode := range []string{"valid", "pending", "running", "failed", "abort-before-status", "copy-aborted", "snapshot-aborted", "owner-replaced", "snapshot-replaced", "not-ready", "external"} { + t.Run(mode, func(t *testing.T) { + b := guardBackup("snapshot") + b.Spec.Strategy = "volume-snapshot" + b.Status.Phase, b.Status.SnapshotID = gameplanev1alpha1.BackupPhaseSucceeded, b.Name + identity, err := json.Marshal(backupSnapshotIdentity{UID: "snapshot-uid"}) + if err != nil { + t.Fatal(err) + } + b.Annotations = map[string]string{backupSnapshotIdentityAnnotation: string(identity)} + vs := &snapshotv1.VolumeSnapshot{ObjectMeta: metav1.ObjectMeta{Name: b.Name, Namespace: b.Namespace, UID: "snapshot-uid", OwnerReferences: []metav1.OwnerReference{{APIVersion: gameplanev1alpha1.GroupVersion.String(), Kind: "Backup", Name: b.Name, UID: b.UID, Controller: ownerBoolPtr(true)}}}, Status: &snapshotv1.VolumeSnapshotStatus{ReadyToUse: ownerBoolPtr(true)}} + switch mode { + case "pending": + b.Status.Phase = gameplanev1alpha1.BackupPhasePending + case "running": + b.Status.Phase = gameplanev1alpha1.BackupPhaseRunning + case "failed": + b.Status.Phase = gameplanev1alpha1.BackupPhaseFailed + case "abort-before-status": + b.Annotations[backupSnapshotAbortAnnotation] = "true" + case "copy-aborted": + b.Annotations[backupCopyAbortAnnotation] = "cancelled" + case "snapshot-aborted": + vs.Annotations = map[string]string{backupSnapshotAbortAnnotation: "true"} + case "owner-replaced": + vs.OwnerReferences[0].UID = "previous-backup" + case "snapshot-replaced": + vs.UID = "replacement-snapshot" + case "not-ready": + vs.Status = nil + case "external": + vs.OwnerReferences = nil + vs.Status = nil + } + r, _ := guardFixture(t, b) + if err := r.Create(t.Context(), vs); err != nil { + t.Fatal(err) + } + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "restored", Namespace: b.Namespace, UID: "restored-uid"}, Spec: gameplanev1alpha1.GameServerSpec{Storage: &gameplanev1alpha1.GameStorageSpec{DataSource: &gameplanev1alpha1.GameDataSource{Kind: "VolumeSnapshot", Name: b.Name}}}} + gr := &GameServerReconciler{Client: r.Client, APIReader: r.Client, Scheme: r.Scheme} + err = gr.reconcilePVC(t.Context(), gs, &gameplanev1alpha1.GameTemplate{}) + allowed := mode == "valid" || mode == "external" + if (err == nil) != allowed { + t.Fatalf("PVC source validation mode=%s err=%v", mode, err) + } + var pvc corev1.PersistentVolumeClaim + getErr := r.Get(t.Context(), client.ObjectKey{Namespace: b.Namespace, Name: "restored-data"}, &pvc) + if !allowed && !apierrors.IsNotFound(getErr) { + t.Fatalf("invalid source created PVC: %v", getErr) + } + }) + } +} diff --git a/operator/internal/controller/backup_stopped_test.go b/operator/internal/controller/backup_stopped_test.go index 8a54656a2..a61c43712 100644 --- a/operator/internal/controller/backup_stopped_test.go +++ b/operator/internal/controller/backup_stopped_test.go @@ -200,8 +200,12 @@ func TestBackupLiveTimeoutRetainsOwnerUntilOriginalPodGone(t *testing.T) { t.Fatalf("timeout did not fail closed: %+v", first.Status) } var job batchv1.Job - if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "first"}, &job); !apierrors.IsNotFound(err) { - t.Fatal("live timeout created copy") + var gate corev1.ConfigMap + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: backupStartConfigMapName(first)}, &gate); err != nil { + t.Fatal(err) + } + if gate.Data[backupStartTokenKey] != "" { + t.Fatal("live timeout authorized prepared worker") } var gs gameplanev1alpha1.GameServer if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server"}, &gs); err != nil { diff --git a/operator/internal/controller/backup_volumesnapshot.go b/operator/internal/controller/backup_volumesnapshot.go index e90a0b6c9..b8a8dc097 100644 --- a/operator/internal/controller/backup_volumesnapshot.go +++ b/operator/internal/controller/backup_volumesnapshot.go @@ -2,11 +2,13 @@ package controller import ( "context" + "errors" "fmt" "time" snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -29,7 +31,16 @@ func (r *BackupReconciler) reconcileVolumeSnapshot( ctx context.Context, b *gameplanev1alpha1.Backup, gs *gameplanev1alpha1.GameServer, ) (ctrl.Result, error) { if b.Annotations[backupSnapshotAbortAnnotation] == "true" { - return r.failVolumeSnapshot(ctx, b, "volume snapshot aborted after an API or creation failure") + return r.abortVolumeSnapshot(ctx, b, "volume snapshot capture was aborted") + } + // An admitted capture's save-off budget stays authoritative even if its + // source/fence later becomes unavailable. Only a new capture waits for + // admission before creating this deadline. + if value := b.Annotations[backupCSIDeadlineAnnotation]; value != "" { + deadline, err := time.Parse(time.RFC3339Nano, value) + if err != nil || !time.Now().Before(deadline) { + return r.abortVolumeSnapshot(ctx, b, "CSI capture deadline exceeded; incomplete snapshot invalidated") + } } // An uncached read verifies that the snapshot API is served before // disabling autosave. Missing CRDs (including NoMatch) and authorization @@ -38,22 +49,46 @@ func (r *BackupReconciler) reconcileVolumeSnapshot( if err := r.backupAPIReader().Get(ctx, types.NamespacedName{Namespace: b.Namespace, Name: b.Name}, &existing); err != nil && !apierrors.IsNotFound(err) { return r.abortVolumeSnapshot(ctx, b, fmt.Sprintf("volume snapshot API unavailable: %v", err)) } + if existing.Name != "" { + if !metav1.IsControlledBy(&existing, b) { + return r.waitForForeignBackupWorker(ctx, b) + } + if err := r.rememberBackupSnapshot(ctx, b, &existing); err != nil { + return r.abortVolumeSnapshot(ctx, b, err.Error()) + } + } + if !legacyBackupOwesUnquiesce(b) { + if res, err := r.prepareBackupWorkload(ctx, b); err != nil || res.RequeueAfter > 0 { + return res, err + } + } + deadline, err := r.backupDeadline(ctx, b, backupCSIDeadlineAnnotation, backupTimeout(r.CSICaptureTimeout, DefaultBackupCSICaptureTimeout)) + if err != nil { + return r.abortVolumeSnapshot(ctx, b, err.Error()) + } + if !time.Now().Before(deadline) { + return r.abortVolumeSnapshot(ctx, b, "CSI capture deadline exceeded; incomplete snapshot invalidated") + } if legacyBackupOwesUnquiesce(b) && metav1.IsControlledBy(&existing, b) { return r.observeVolumeSnapshot(ctx, b, &existing) } - // Quiesce first (idempotent across requeues). A CSI snapshot is only - // crash-consistent; flushing the game to disk first (RCON save-all, - // etc.) makes it application-consistent. No-op when the backup opts out - // or no agent mTLS is configured. + // The durable capture budget starts before any save-off RPC. Quiesce + // confirmation describes the agent action, not universal game consistency. if res, err := r.prepareBackupQuiesce(ctx, b); err != nil || res.RequeueAfter > 0 { return res, err } + if !time.Now().Before(deadline) { + return r.abortVolumeSnapshot(ctx, b, "CSI capture deadline exceeded during quiesce") + } pvcName := gs.Name + "-data" vs := &snapshotv1.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{Name: b.Name, Namespace: b.Namespace}, } if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, vs, func() error { + if !vs.CreationTimestamp.IsZero() && !metav1.IsControlledBy(vs, b) { + return fmt.Errorf("VolumeSnapshot is not owned by this Backup") + } // Source + class are immutable on a VolumeSnapshot, so only set // them on first creation; SetControllerReference is idempotent. if vs.CreationTimestamp.IsZero() { @@ -71,16 +106,28 @@ func (r *BackupReconciler) reconcileVolumeSnapshot( // independently until save-on is confirmed. return r.abortVolumeSnapshot(ctx, b, fmt.Sprintf("volume snapshot creation failed: %v", err)) } + if err := r.rememberBackupSnapshot(ctx, b, vs); err != nil { + return r.abortVolumeSnapshot(ctx, b, err.Error()) + } return r.observeVolumeSnapshot(ctx, b, vs) } func (r *BackupReconciler) observeVolumeSnapshot(ctx context.Context, b *gameplanev1alpha1.Backup, vs *snapshotv1.VolumeSnapshot) (ctrl.Result, error) { + if b.Annotations[backupSnapshotAbortAnnotation] == "true" { + return r.abortVolumeSnapshot(ctx, b, "volume snapshot capture was aborted") + } + if value := b.Annotations[backupCSIDeadlineAnnotation]; value != "" { + deadline, err := time.Parse(time.RFC3339Nano, value) + if err != nil || !time.Now().Before(deadline) { + return r.abortVolumeSnapshot(ctx, b, "CSI capture deadline exceeded; late readiness is invalid") + } + } switch { case vs.Status != nil && vs.Status.Error != nil && vs.Status.Error.Message != nil: // The CSI driver rejected the snapshot. Surface it and unquiesce so // the game doesn't stay frozen. - return r.failVolumeSnapshot(ctx, b, fmt.Sprintf("volume snapshot failed: %s", *vs.Status.Error.Message)) + return r.abortVolumeSnapshot(ctx, b, fmt.Sprintf("volume snapshot failed: %s", *vs.Status.Error.Message)) case vs.Status != nil && vs.Status.ReadyToUse != nil && *vs.Status.ReadyToUse: return r.completeVolumeSnapshot(ctx, b, vs) default: @@ -93,11 +140,15 @@ func (r *BackupReconciler) observeVolumeSnapshot(ctx context.Context, b *gamepla // created a snapshot. Persist that abort decision before terminal cleanup so // worker-drain checks cannot trap this failed/nonrestorable Backup forever. func (r *BackupReconciler) abortVolumeSnapshot(ctx context.Context, b *gameplanev1alpha1.Backup, msg string) (ctrl.Result, error) { - patchBackupAnnotations(b, map[string]string{backupSnapshotAbortAnnotation: "true"}) - if err := r.Update(ctx, b); err != nil { + if err := r.recordSnapshotAbort(ctx, b, msg); err != nil { return ctrl.Result{}, err } - return r.failVolumeSnapshot(ctx, b, msg) + unquiesce, unquiesceErr := r.runUnquiesce(ctx, b) + cleanup, cleanupErr := r.cleanupAbortedSnapshot(ctx, b) + if unquiesce.RequeueAfter > 0 && (cleanup.RequeueAfter == 0 || unquiesce.RequeueAfter < cleanup.RequeueAfter) { + cleanup = unquiesce + } + return cleanup, errors.Join(unquiesceErr, cleanupErr) } // markVolumeSnapshotRunning flips the Backup to Running on first observation @@ -124,6 +175,9 @@ func (r *BackupReconciler) markVolumeSnapshotRunning( return ctrl.Result{}, err } } + if err := r.setBackupProgress(ctx, b, "SnapshotCapturePending", "waiting for CSI capture; deadline "+b.Annotations[backupCSIDeadlineAnnotation]); err != nil { + return ctrl.Result{}, err + } return ctrl.Result{RequeueAfter: 10 * time.Second}, nil } @@ -132,9 +186,13 @@ func (r *BackupReconciler) markVolumeSnapshotRunning( func (r *BackupReconciler) completeVolumeSnapshot( ctx context.Context, b *gameplanev1alpha1.Backup, vs *snapshotv1.VolumeSnapshot, ) (ctrl.Result, error) { + if b.Annotations[backupSnapshotAbortAnnotation] == "true" { + return r.abortVolumeSnapshot(ctx, b, "volume snapshot capture was aborted") + } if b.Status.Phase != gameplanev1alpha1.BackupPhaseSucceeded { now := metav1.Now() b.Status.Phase = gameplanev1alpha1.BackupPhaseSucceeded + meta.RemoveStatusCondition(&b.Status.Conditions, "CaptureReady") b.Status.ObservedGeneration = b.Generation b.Status.SnapshotID = vs.Name if vs.Status.BoundVolumeSnapshotContentName != nil { @@ -167,14 +225,3 @@ func (r *BackupReconciler) completeVolumeSnapshot( } return r.runUnquiesce(ctx, b) } - -// failVolumeSnapshot marks the Backup Failed and still runs the terminal -// unquiesce so a quiesced game doesn't stay frozen after a snapshot error. -func (r *BackupReconciler) failVolumeSnapshot( - ctx context.Context, b *gameplanev1alpha1.Backup, msg string, -) (ctrl.Result, error) { - if _, err := r.fail(ctx, b, msg); err != nil { - return ctrl.Result{}, err - } - return r.runUnquiesce(ctx, b) -} diff --git a/operator/internal/controller/backup_volumesnapshot_envtest_test.go b/operator/internal/controller/backup_volumesnapshot_envtest_test.go index 6e8bb7e01..d182eb696 100644 --- a/operator/internal/controller/backup_volumesnapshot_envtest_test.go +++ b/operator/internal/controller/backup_volumesnapshot_envtest_test.go @@ -9,8 +9,10 @@ import ( "testing" "time" + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" @@ -210,6 +212,10 @@ func TestRestore_VolumeSnapshotRejectsExistingTarget(t *testing.T) { func TestGameServer_DataSourceSeedsPVC(t *testing.T) { ns := newNamespace(t) startMgr(t, ns, withGameServerReconciler(t, ns)) + sourcePVC := "external-source-data" + if err := k8sClient.Create(context.Background(), &snapshotv1.VolumeSnapshot{ObjectMeta: metav1.ObjectMeta{Name: "snap-1", Namespace: ns}, Spec: snapshotv1.VolumeSnapshotSpec{Source: snapshotv1.VolumeSnapshotSource{PersistentVolumeClaimName: &sourcePVC}}}); err != nil { + t.Fatal(err) + } tmpl := buildGameTemplate(uniqueName("minecraft")) if err := k8sClient.Create(context.Background(), tmpl); err != nil { diff --git a/operator/internal/controller/backup_worker.go b/operator/internal/controller/backup_worker.go new file mode 100644 index 000000000..80cf6c98c --- /dev/null +++ b/operator/internal/controller/backup_worker.go @@ -0,0 +1,380 @@ +package controller + +import ( + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "strconv" + "time" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +const ( + backupWorkerSourceAnnotation = "backup.gameplane.local/worker-source" + backupAuthorizedWorkerAnnotation = "backup.gameplane.local/authorized-worker" + backupStartTokenKey = "start" + backupWorkerProtocolAnnotation = "backup.gameplane.local/worker-protocol" +) + +// The waiting worker never reads source files until the operator authorizes +// this exact Backup/pod/source generation. Repository preparation takes a +// shared restic lock; later lock contention is bounded by the copy timeout. +// Values and tags are arguments/environment, never interpolated shell code. +const resticWorkerScript = `set -eu +for utility in timeout date cat touch sleep restic; do command -v "$utility" >/dev/null; done +timeout -s TERM -k 5 1 /bin/sh -c ':' +test -d /data && test -r /data && test -x /data +restic snapshots --json --retry-lock 10m >/dev/null +touch /tmp/prepared +while :; do + now=$(date +%s) + if [ -r /start/start ]; then + token=$(cat /start/start) + if [ -n "$token" ]; then + read -r version backup pod source expires extra < 0 { + return res, err + } + return r.mirrorJobStatus(ctx, b, &job) + } + // Queueing behind another lifecycle owner is not worker preparation and + // must not consume its budget before this Backup can start preparing. + if b.Annotations[backupPreparationDeadlineAnnotation] == "" { + claimed, err := r.claimBackupTarget(ctx, b) + if err != nil { + if errors.Is(err, errLifecycleGenerationChanged) { + return r.abortResticCopy(ctx, b, err.Error()) + } + return ctrl.Result{}, err + } + if !claimed { + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } + } + preparation, err := r.backupDeadline(ctx, b, backupPreparationDeadlineAnnotation, backupTimeout(r.PreparationTimeout, DefaultBackupPreparationTimeout)) + if err != nil { + return r.abortResticCopy(ctx, b, err.Error()) + } + if b.Annotations[backupAuthorizedWorkerAnnotation] == "" && time.Now().After(preparation) { + return r.abortResticCopy(ctx, b, "worker preparation deadline exceeded before copy authorization") + } + if value := b.Annotations[backupQuiesceDeadlineAnnotation]; value != "" { + deadline, err := time.Parse(time.RFC3339Nano, value) + if err != nil { + return r.abortResticCopy(ctx, b, "invalid persisted quiesce deadline") + } + finishedInTime := job.Status.Succeeded > 0 && job.Status.CompletionTime != nil && !job.Status.CompletionTime.After(deadline) + if time.Now().After(deadline) && !finishedInTime { + return r.abortResticCopy(ctx, b, "maximum quiesce duration exceeded; copy invalidated") + } + } + if res, err := r.prepareBackupWorkload(ctx, b); err != nil || res.RequeueAfter > 0 { + return res, err + } + // Admission refreshes the target. Read its effective content only after + // owning the workload fence, so a rollout completed before admission + // cannot leave the worker using an earlier filesystem layout. + bound, err := r.ownedBackupTarget(ctx, b) + if err != nil { + return ctrl.Result{}, err + } + if bound == nil { + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + } + source, err := r.prepareBackupWorkerSource(ctx, b) + if err != nil { + if errors.Is(err, errBackupSourceChanged) { + return r.abortResticCopy(ctx, b, err.Error()) + } + return ctrl.Result{}, err + } + if source == nil { + if err := r.setBackupProgress(ctx, b, "WorkerPreparing", "waiting for the source workload to be created and scheduled"); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + } + gate := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: backupStartConfigMapName(b), Namespace: b.Namespace}} + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, gate, func() error { + if !gate.CreationTimestamp.IsZero() && !metav1.IsControlledBy(gate, b) { + return fmt.Errorf("copy authorization ConfigMap is not owned by this Backup") + } + if gate.Data == nil { + gate.Data = map[string]string{backupStartTokenKey: ""} + } + return controllerutil.SetControllerReference(b, gate, r.Scheme) + }); err != nil { + return ctrl.Result{}, err + } + if job.Name == "" { + // Existing workers retain their immutable layout and must continue + // through authorization/completion even if the template disappears. + tmpl, err := effectiveGameTemplate(ctx, r.backupAPIReader(), bound) + if err != nil && (bound.Annotations[effectiveTemplateAnnotation] != "" || !apierrors.IsNotFound(err)) { + return r.abortResticCopy(ctx, b, "cannot resolve capture template: "+err.Error()) + } + backoff, deadline := resolveJobLimits(r.JobBackoffLimit, r.JobActiveDeadlineSeconds) + job = batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: b.Name, Namespace: b.Namespace, Annotations: map[string]string{backupWorkerProtocolAnnotation: "v1"}}, Spec: batchv1.JobSpec{BackoffLimit: &backoff, ActiveDeadlineSeconds: &deadline, + Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{backupRestoreJobLabel: backupRestoreJobValue, backupUIDLabel: string(b.UID)}}, Spec: r.buildPreparedBackupPodSpec(b, tmpl, source, preparation)}}} + job.Spec.Template.Spec.RestartPolicy = corev1.RestartPolicyNever + if err := controllerutil.SetControllerReference(b, &job, r.Scheme); err != nil { + return ctrl.Result{}, err + } + if err := r.Create(ctx, &job); err != nil { + return ctrl.Result{}, err + } + } + if jobPermanentlyFailed(&job) { + return r.abortResticCopy(ctx, b, "backup job reported Failed") + } + if job.Status.Succeeded > 0 { + if b.Annotations[backupAuthorizedWorkerAnnotation] == "" { + return r.abortResticCopy(ctx, b, "backup worker completed without copy authorization") + } + return r.mirrorJobStatus(ctx, b, &job) + } + worker, err := r.preparedBackupWorker(ctx, b, &job) + if err != nil { + return ctrl.Result{}, err + } + if worker == nil { + if err := r.setBackupProgress(ctx, b, "WorkerPreparing", "waiting for a scheduled worker with mounted source and validated repository"); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 2 * time.Second}, nil + } + // Raw and already-stopped copies have no save-off window. Their tokens use + // the full Job deadline, preserving long-copy support independently. + _, copySeconds := resolveJobLimits(r.JobBackoffLimit, r.JobActiveDeadlineSeconds) + if job.Spec.ActiveDeadlineSeconds != nil && *job.Spec.ActiveDeadlineSeconds > 0 { + copySeconds = *job.Spec.ActiveDeadlineSeconds + } + const maxDurationSeconds = int64((1<<63 - 1) / time.Second) + if copySeconds > maxDurationSeconds { + copySeconds = maxDurationSeconds + } + copyBudget := time.Duration(copySeconds) * time.Second + protected := b.Spec.Quiesce && agentQuiesceAvailable(r.AgentClient) + if prepared, err := parseBackupPreparation(b); err != nil { + return ctrl.Result{}, err + } else if prepared != nil && prepared.Outcome == "already-stopped" { + protected = false + } + deadlineKey := backupQuiesceDeadlineAnnotation + if protected { + copyBudget = backupTimeout(r.QuiesceTimeout, DefaultBackupQuiesceTimeout) + } + deadline, err := r.backupDeadline(ctx, b, deadlineKey, copyBudget) + if err != nil { + return ctrl.Result{}, err + } + if res, err := r.prepareBackupQuiesce(ctx, b); err != nil || res.RequeueAfter > 0 { + return res, err + } + if _, err := r.prepareBackupWorkerSource(ctx, b); err != nil { + if errors.Is(err, errBackupSourceChanged) { + return r.abortResticCopy(ctx, b, err.Error()) + } + return ctrl.Result{}, err + } + // Re-read readiness after the RPC. A replacement worker cannot inherit + // the first pod's authorization, including during a Job retry. + current, err := r.preparedBackupWorker(ctx, b, &job) + if err != nil { + return ctrl.Result{}, err + } + if current == nil || current.UID != worker.UID { + return ctrl.Result{RequeueAfter: 2 * time.Second}, nil + } + if !time.Now().Before(deadline) { + return r.abortResticCopy(ctx, b, "maximum quiesce duration exceeded before start authorization") + } + if b.Annotations[backupAuthorizedWorkerAnnotation] != string(worker.UID) { + patchBackupAnnotations(b, map[string]string{backupAuthorizedWorkerAnnotation: string(worker.UID)}) + if err := r.Update(ctx, b); err != nil { + return ctrl.Result{}, err + } + } + token := fmt.Sprintf("v1 %s %s %s %d\n", b.UID, worker.UID, backupWorkerSourceID(b), deadline.Unix()) + if gate.Data[backupStartTokenKey] != token { + gate.Data[backupStartTokenKey] = token + if err := r.Update(ctx, gate); err != nil { + return ctrl.Result{}, err + } + } + authorizedThisPass = true + if err := r.setBackupProgress(ctx, b, "WorkerAuthorized", "prepared worker authorized; copy deadline "+deadline.UTC().Format(time.RFC3339)); err != nil { + return ctrl.Result{}, err + } + return r.mirrorJobStatus(ctx, b, &job) +} diff --git a/operator/internal/controller/backup_worker_replacement_test.go b/operator/internal/controller/backup_worker_replacement_test.go new file mode 100644 index 000000000..8352b22a1 --- /dev/null +++ b/operator/internal/controller/backup_worker_replacement_test.go @@ -0,0 +1,91 @@ +package controller + +import ( + "testing" + + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +func TestBackupAdmittedForeignWorkerAbortsAndDrainsOnlyItsOwnWorkers(t *testing.T) { + for _, strategy := range []string{"restic-snapshot", "volume-snapshot"} { + t.Run(strategy, func(t *testing.T) { + b := guardBackup("first") + b.Spec.Strategy = strategy + r, q := guardFixtureWorkers(t, false, b) + guardPass(t, r, b.Name) + var ownPod *corev1.Pod + if strategy == "restic-snapshot" { + ownPod = budgetReadyWorker(t, r, b.Name, "original-worker") + guardPass(t, r, b.Name) + } else { + ownPod = &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "original-worker", Namespace: "ns", Labels: map[string]string{backupUIDLabel: string(b.UID)}}, + Status: corev1.PodStatus{Phase: corev1.PodRunning}} + if err := r.Create(t.Context(), ownPod); err != nil { + t.Fatal(err) + } + } + b = guardGetBackup(t, r, b.Name) + if q.quiesced != 1 || (strategy == "restic-snapshot" && b.Annotations[backupAuthorizedWorkerAnnotation] == "") { + t.Fatalf("fixture never admitted its capture: %+v %+v", b, q) + } + original, err := r.captureWorker(t.Context(), b) + if err != nil || original == nil { + t.Fatalf("fixture worker missing: %v", err) + } + if err := r.Delete(t.Context(), original); err != nil { + t.Fatal(err) + } + foreignBackup := b.DeepCopy() + foreignBackup.UID = "foreign-backup" + meta := metav1.ObjectMeta{Name: b.Name, Namespace: b.Namespace, UID: "foreign-worker", + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(foreignBackup, gameplanev1alpha1.GroupVersion.WithKind("Backup"))}} + var foreign client.Object = &batchv1.Job{ObjectMeta: meta} + if strategy == "volume-snapshot" { + foreign = &snapshotv1.VolumeSnapshot{ObjectMeta: meta} + } + if err := r.Create(t.Context(), foreign); err != nil { + t.Fatal(err) + } + if _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(b)}); err != nil { + t.Fatal(err) + } + b = guardGetBackup(t, r, b.Name) + if b.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || q.unquiesced != 0 { + t.Fatalf("replacement did not abort or released before own worker drain: %+v %+v", b.Status, q) + } + if strategy == "restic-snapshot" && b.Annotations[backupCopyAbortAnnotation] == "" || strategy == "volume-snapshot" && b.Annotations[backupSnapshotAbortAnnotation] != "true" { + t.Fatal("replacement abort was not durable") + } + var gs gameplanev1alpha1.GameServer + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server"}, &gs); err != nil { + t.Fatal(err) + } + if gs.Annotations[lifecycleOperationAnnotation] == "" { + t.Fatal("replacement released the owner before its original worker drained") + } + if err := r.Delete(t.Context(), ownPod); err != nil { + t.Fatal(err) + } + guardPass(t, r, b.Name) + if q.unquiesced != 1 { + t.Fatalf("drained original capture did not restore saving: %+v", q) + } + if err := r.Get(t.Context(), client.ObjectKeyFromObject(&gs), &gs); err != nil { + t.Fatal(err) + } + if gs.Annotations[lifecycleOperationAnnotation] != "" || gs.Annotations[backupGuardAnnotation] != "" { + t.Fatal("drained original capture retained its lifecycle owner") + } + if err := r.Get(t.Context(), client.ObjectKeyFromObject(foreign), foreign); err != nil || foreign.GetUID() != "foreign-worker" || foreign.GetDeletionTimestamp() != nil { + t.Fatalf("replacement cleanup modified the foreign resource: %v %+v", err, foreign) + } + }) + } +} diff --git a/operator/internal/controller/backup_worker_retry_test.go b/operator/internal/controller/backup_worker_retry_test.go new file mode 100644 index 000000000..4da88e7ce --- /dev/null +++ b/operator/internal/controller/backup_worker_retry_test.go @@ -0,0 +1,181 @@ +package controller + +import ( + "context" + "encoding/json" + "errors" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +func TestBackupWorkerSourceReadFailureRetriesWithoutAuthorizing(t *testing.T) { + for _, phase := range []string{"before-authorization", "postcheck", "authorized", "authorized-quiesced"} { + t.Run(phase, func(t *testing.T) { + b := guardBackup("retry") + b.Spec.Quiesce = phase == "authorized-quiesced" + r, q := guardFixtureWorkers(t, false, b) + budgetPass(t, r, b.Name) + budgetReadyWorker(t, r, b.Name, "worker-uid") + if phase == "authorized" || phase == "authorized-quiesced" { + budgetPass(t, r, b.Name) + } + before := guardGetBackup(t, r, b.Name) + readFailure := apierrors.NewServiceUnavailable("temporary source read failure") + failedRead := false + r.APIReader = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, pod := obj.(*corev1.Pod); pod && key.Name == "server-0" { + if phase == "postcheck" { + // Capturing the manifest also reads this Pod. Wait for + // that capture to persist before failing the source + // postcheck immediately before worker authorization. + var current gameplanev1alpha1.Backup + if err := c.Get(ctx, client.ObjectKeyFromObject(b), ¤t); err != nil { + return err + } + if current.Status.CaptureManifest == nil { + return c.Get(ctx, key, obj, opts...) + } + } + failedRead = true + return readFailure + } + return c.Get(ctx, key, obj, opts...) + }}) + request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(b)} + if _, err := r.Reconcile(t.Context(), request); !errors.Is(err, readFailure) { + t.Fatalf("source read failure was not retried: %v", err) + } + if !failedRead { + t.Fatal("reconcile never reached the intended source read") + } + after := guardGetBackup(t, r, b.Name) + if after.Status.Phase == gameplanev1alpha1.BackupPhaseFailed || after.Annotations[backupCopyAbortAnnotation] != "" { + t.Fatalf("transient source read permanently aborted copy: %+v", after) + } + for _, key := range []string{backupPreparationDeadlineAnnotation, backupWorkerSourceAnnotation, backupAuthorizedWorkerAnnotation} { + if after.Annotations[key] != before.Annotations[key] { + t.Fatalf("retry changed durable %s", key) + } + } + var gate corev1.ConfigMap + gateKey := client.ObjectKey{Namespace: b.Namespace, Name: backupStartConfigMapName(b)} + if err := r.Get(t.Context(), gateKey, &gate); err != nil || gate.Data[backupStartTokenKey] != "" { + t.Fatalf("unverified source retained an open gate: %v %+v", err, gate.Data) + } + var server gameplanev1alpha1.GameServer + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server"}, &server); err != nil { + t.Fatal(err) + } + if server.Annotations[lifecycleOperationAnnotation] == "" || q.unquiesced != 0 { + t.Fatal("retry released a possibly running copy's lifecycle/quiesce obligation") + } + deadline := after.Annotations[backupQuiesceDeadlineAnnotation] + r.APIReader = r.Client + budgetPass(t, r, b.Name) + recovered := guardGetBackup(t, r, b.Name) + if err := r.Get(t.Context(), gateKey, &gate); err != nil || gate.Data[backupStartTokenKey] == "" { + t.Fatalf("healthy source did not regain authorization: %v", err) + } + if deadline != "" && recovered.Annotations[backupQuiesceDeadlineAnnotation] != deadline { + t.Fatal("source retry extended the copy deadline") + } + if phase == "authorized-quiesced" && q.quiesced != 1 { + t.Fatalf("source retry repeated save-off: %+v", q) + } + }) + } +} + +func TestBackupWorkerUnverifiedSourceStillExpires(t *testing.T) { + b := guardBackup("expired-read") + b.Spec.Quiesce = false + r, _ := guardFixtureWorkers(t, false, b) + budgetPass(t, r, b.Name) + b = guardGetBackup(t, r, b.Name) + patchBackupAnnotations(b, map[string]string{backupPreparationDeadlineAnnotation: time.Now().Add(-time.Minute).UTC().Format(time.RFC3339Nano)}) + if err := r.Update(t.Context(), b); err != nil { + t.Fatal(err) + } + r.APIReader = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, pod := obj.(*corev1.Pod); pod && key.Name == "server-0" { + return apierrors.NewTooManyRequests("source reads throttled", 1) + } + return c.Get(ctx, key, obj, opts...) + }}) + budgetPass(t, r, b.Name) + if got := guardGetBackup(t, r, b.Name); got.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || got.Annotations[backupCopyAbortAnnotation] == "" { + t.Fatalf("unavailable source escaped the preparation deadline: %+v", got) + } +} + +func TestBackupWorkerInvalidSourceRemainsTerminal(t *testing.T) { + for _, state := range []string{"malformed-binding", "replacement-pod"} { + t.Run(state, func(t *testing.T) { + b := guardBackup("invalid-source") + b.Spec.Quiesce = false + r, _ := guardFixtureWorkers(t, false, b) + budgetPass(t, r, b.Name) + b = guardGetBackup(t, r, b.Name) + if state == "malformed-binding" { + patchBackupAnnotations(b, map[string]string{backupWorkerSourceAnnotation: "{"}) + if err := r.Update(t.Context(), b); err != nil { + t.Fatal(err) + } + _, err := r.prepareBackupWorkerSource(t.Context(), b) + var syntaxError *json.SyntaxError + if !errors.Is(err, errBackupSourceChanged) || !errors.As(err, &syntaxError) { + t.Fatalf("malformed source lost its terminal classification or decode cause: %v", err) + } + } else { + r.APIReader = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if err := c.Get(ctx, key, obj, opts...); err != nil { + return err + } + if pod, ok := obj.(*corev1.Pod); ok && key.Name == "server-0" { + pod.UID = types.UID("replacement") + } + return nil + }}) + } + budgetPass(t, r, b.Name) + if got := guardGetBackup(t, r, b.Name); got.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || got.Annotations[backupCopyAbortAnnotation] == "" { + t.Fatalf("invalid source was treated as retryable: %+v", got) + } + }) + } +} + +func TestBackupWorkerRetrySurfacesGateRevocationFailure(t *testing.T) { + b := guardBackup("revoke-error") + b.Spec.Quiesce = false + r, _ := guardFixture(t, b) + budgetPass(t, r, b.Name) + readFailure := apierrors.NewServiceUnavailable("source unavailable") + revokeFailure := errors.New("gate write unavailable") + base := r.Client.(client.WithWatch) + r.Client = interceptor.NewClient(base, interceptor.Funcs{Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + if _, gate := obj.(*corev1.ConfigMap); gate { + return revokeFailure + } + return c.Update(ctx, obj, opts...) + }}) + r.APIReader = interceptor.NewClient(base, interceptor.Funcs{Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, pod := obj.(*corev1.Pod); pod && key.Name == "server-0" { + return readFailure + } + return c.Get(ctx, key, obj, opts...) + }}) + _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(b)}) + if !errors.Is(err, readFailure) || !errors.Is(err, revokeFailure) { + t.Fatalf("retry lost a source or revocation failure: %v", err) + } +} diff --git a/operator/internal/controller/gameserver_controller.go b/operator/internal/controller/gameserver_controller.go index 7f3115ea0..1ef3a0148 100644 --- a/operator/internal/controller/gameserver_controller.go +++ b/operator/internal/controller/gameserver_controller.go @@ -608,6 +608,13 @@ func (r *GameServerReconciler) reconcilePVC( // size is >= the snapshot's restoreSize by construction (the // snapshot came from a PVC of that size). if gs.Spec.Storage != nil && gs.Spec.Storage.DataSource != nil { + reader := r.APIReader + if reader == nil { + reader = r.Client + } + if err := validateBackupSnapshotSource(ctx, reader, gs.Namespace, gs.Spec.Storage.DataSource.Name); err != nil { + return err + } apiGroup := "snapshot.storage.k8s.io" pvc.Spec.DataSource = &corev1.TypedLocalObjectReference{ APIGroup: &apiGroup, diff --git a/operator/internal/controller/helpers_envtest_test.go b/operator/internal/controller/helpers_envtest_test.go index 8069878a8..0a18108ad 100644 --- a/operator/internal/controller/helpers_envtest_test.go +++ b/operator/internal/controller/helpers_envtest_test.go @@ -468,6 +468,25 @@ func patchJobStatus(t *testing.T, ns, name string, mut func(s *batchv1.JobStatus t.Fatalf("get job %s/%s: %v", ns, name, err) } mut(&job.Status) + if job.Annotations[backupWorkerProtocolAnnotation] == "v1" { + if job.Status.Succeeded > 0 || job.Status.Active > 0 { + prepareEnvtestBackupWorker(t, &job) + } + if job.Status.Succeeded > 0 || job.Status.Failed > 0 { + var worker corev1.Pod + if err := k8sClient.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: name + "-test-worker"}, &worker); err == nil { + worker.Status.Phase = corev1.PodSucceeded + if job.Status.Failed > 0 { + worker.Status.Phase = corev1.PodFailed + } + if err := k8sClient.Status().Update(context.Background(), &worker); err != nil { + t.Fatal(err) + } + } else if !apierrors.IsNotFound(err) { + t.Fatal(err) + } + } + } now := metav1.Now() @@ -494,6 +513,34 @@ func patchJobStatus(t *testing.T, ns, name string, mut func(s *batchv1.JobStatus } } +// envtest has neither Job controller nor kubelet. This models their actual +// readiness contract and waits for operator authorization before reporting a +// successful copy. It never grants a token itself. +func prepareEnvtestBackupWorker(t *testing.T, job *batchv1.Job) { + t.Helper() + ctx := context.Background() + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: job.Name + "-test-worker", Namespace: job.Namespace, Labels: job.Spec.Template.Labels, + OwnerReferences: []metav1.OwnerReference{{APIVersion: "batch/v1", Kind: "Job", Name: job.Name, UID: job.UID, Controller: ownerBoolPtr(true)}}}, Spec: job.Spec.Template.Spec} + pod.Spec.NodeName = "test-node" + if err := k8sClient.Create(ctx, pod); err != nil { + if !apierrors.IsAlreadyExists(err) { + t.Fatal(err) + } + if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(pod), pod); err != nil { + t.Fatal(err) + } + } + pod.Status.Phase = corev1.PodRunning + pod.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + if err := k8sClient.Status().Update(ctx, pod); err != nil { + t.Fatal(err) + } + eventually(t, func() (bool, string) { + backup := getBackup(t, job.Namespace, job.Name) + return backup.Annotations[backupAuthorizedWorkerAnnotation] == string(pod.UID), "prepared worker is not yet authorized: " + describeBackupStatus(backup) + }) +} + // upsertJobCondition sets/replaces the JobCondition of type t to // Status: True in conds, returning the updated slice. Unlike // upsertCondition (helpers.go), which operates on the CRD condition @@ -838,4 +885,15 @@ func seedBackupWorkload(t *testing.T, gs *gameplanev1alpha1.GameServer) { if err := k8sClient.Status().Update(context.Background(), ss); err != nil { t.Fatalf("observe workload: %v", err) } + // Envtest has no StatefulSet controller or scheduler. Model their initial + // pod placement before a worker binds its immutable source identity. + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name + "-0", Namespace: ns, OwnerReferences: []metav1.OwnerReference{{APIVersion: "apps/v1", Kind: "StatefulSet", Name: name, UID: ss.UID, Controller: ownerBoolPtr(true)}}}, + Spec: corev1.PodSpec{NodeName: "fixture-node", Containers: []corev1.Container{{Name: "game", Image: "test/game:fixture"}}}} + if err := k8sClient.Create(context.Background(), pod); err != nil { + t.Fatalf("seed scheduled source pod: %v", err) + } + pod.Status.Phase = corev1.PodRunning + if err := k8sClient.Status().Update(context.Background(), pod); err != nil { + t.Fatalf("observe running source pod: %v", err) + } } diff --git a/operator/internal/controller/restore_controller.go b/operator/internal/controller/restore_controller.go index e979ee712..41839e2a6 100644 --- a/operator/internal/controller/restore_controller.go +++ b/operator/internal/controller/restore_controller.go @@ -80,13 +80,13 @@ func (r *RestoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct // the rug out from under us mid-restore. if rs.Status.SnapshotID == "" { var src gameplanev1alpha1.Backup - if err := r.Get(ctx, types.NamespacedName{Name: rs.Spec.BackupRef.Name, Namespace: rs.Namespace}, &src); err != nil { + if err := r.apiReader().Get(ctx, types.NamespacedName{Name: rs.Spec.BackupRef.Name, Namespace: rs.Namespace}, &src); err != nil { if apierrors.IsNotFound(err) { return r.fail(ctx, &rs, fmt.Sprintf("source backup %q not found", rs.Spec.BackupRef.Name)) } return ctrl.Result{}, err } - if src.Status.Phase == gameplanev1alpha1.BackupPhaseFailed { + if src.Status.Phase == gameplanev1alpha1.BackupPhaseFailed || src.Annotations[backupSnapshotAbortAnnotation] == "true" || src.Annotations[backupCopyAbortAnnotation] != "" { // Terminal: a Failed backup will never grow a snapshotID, so // waiting would leave the Restore Pending forever. return r.fail(ctx, &rs, fmt.Sprintf( @@ -125,7 +125,7 @@ func (r *RestoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct // touch an existing server — they provision a new one seeded from the // CSI snapshot — so they branch off here, before the suspend/Job flow. var src gameplanev1alpha1.Backup - if err := r.Get(ctx, types.NamespacedName{Name: rs.Spec.BackupRef.Name, Namespace: rs.Namespace}, &src); err != nil { + if err := r.apiReader().Get(ctx, types.NamespacedName{Name: rs.Spec.BackupRef.Name, Namespace: rs.Namespace}, &src); err != nil { if apierrors.IsNotFound(err) { return r.fail(ctx, &rs, fmt.Sprintf("source backup %q disappeared", rs.Spec.BackupRef.Name)) } @@ -135,6 +135,9 @@ func (r *RestoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct // preparation path checks that binding before awaiting startup, repairing // references and enforcing the deadline on every subsequent reconcile. if src.Spec.Strategy == "volume-snapshot" { + if src.Annotations[backupSnapshotAbortAnnotation] == "true" || src.Status.Phase != gameplanev1alpha1.BackupPhaseSucceeded || src.Status.SnapshotID != rs.Status.SnapshotID { + return r.fail(ctx, &rs, "source CSI capture is no longer valid") + } return r.reconcileVolumeSnapshotRestore(ctx, &rs, &src) } diff --git a/operator/specs.md b/operator/specs.md index 077fb0f8e..06ca0b645 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -356,18 +356,21 @@ Primary reconcilers register with the manager in `cmd/main.go` and handle CRD li ### BackupReconciler - **Responsibility:** Drive Backup to completion: coordinate quiesce (if agent available), spawn restic Job, track snapshot ID. - **Key functions:** - - Quiesce orchestration: call agent Quiesce before Job, Unquiesce after completion. + - Restic preparation precedes quiesce: claim/fence the source, persist a preparation deadline, create the waiting worker, and require an uncached Running/Ready pod controlled by the exact Job UID. Before binding the first worker, a source StatefulSet requesting replicas must have an owned pod assigned to a node; initial creation/scheduling waits within the preparation budget, while a replacement after binding fails the copy. An intentionally stopped raw source remains eligible without a pod. The worker mounts the PVC, initializes/probes the repository with a shared lock, then waits for an owned ConfigMap start token. Only after readiness does the operator persist the quiesce deadline, perform the pod-bound RPC and source postcheck, record the capture manifest, and grant the Backup UID/worker UID/source-fingerprint/expiry-bound token. Replacement pods cannot reuse it. Workers have no service-account token, use the shared backup resource settings and source tolerations, and prefer the observed game node; hard same-node affinity is opt-in. + - Preparation defaults to 10m and live quiesce attempts to 5m (`--backup-preparation-timeout`, `--backup-quiesce-timeout`; matching Helm `operator.backupPreparationTimeout`, `operator.backupQuiesceTimeout`). Deadlines are annotations written once and survive restart/configuration changes, including when an attempted quiesce is unsupported. Copies requested raw, without an agent, or from an already-stopped source retain the full Job deadline. The waiting shell validates token expiry and bounds Restic with `timeout -s TERM -k 5`. On expiry/deletion, invalidation and token revocation precede foreground cancellation; actual live/terminating workers still block save-on and owner release. A timeout is not evidence of drainage. Existing pre-upgrade Jobs retain their observation protocol. + - Initial source/fence waits share an existing preparation deadline. CSI captures and older Jobs without one persist a separate two-minute source-startup deadline after acquiring lifecycle ownership; queue age and restarts do not spend or reset that budget. Expired unavailable sources fail visibly and enter normal cleanup. A foreign same-name worker is awaited before acquiring ownership or quiescing. Existing owned workers continue through their original gate/completion checks without resolving the template again; template resolution failure when constructing a new worker aborts and releases only after drainage and save-on. - With quiesce and an agent client enabled, preparation is recorded once in `backup.gameplane.local/preparation`, bound to the target UID and StatefulSet UID/generation. `already-stopped` requires uncached proof of desired and observed zero replicas, caught-up observed generation, and no live game/PVC user pods or pending writer Jobs. Current copy workers are excluded by Job ownership. This path sends no agent calls and preserves suspend/idle/wake intent under the shared workload fence. It applies equally to restic and CSI copies. - Live preparation also pins the game Pod UID. The operator checks the agent's read-only `/v1/pods/{uid}/identity` endpoint before recording the first cleanup intent, then uses pod-bound quiesce/unquiesce routes; the agent rejects a different UID before RCON. A missing endpoint or identity mismatch fails closed without a legacy-route fallback. Ambiguous quiesce errors retain cleanup intent and the original Pod UID on both the Backup and shared owner. Only confirmed save-on or uncached evidence that the original pod is absent/replaced/terminal permits cleanup; an HTTP timeout, unready pod, or terminating pod is insufficient. Legacy attempts without a UID retire only on absence/terminal proof and never adopt a new UID. `Unquiesced=True, Reason=SourceProcessGone` distinguishes process exit from an actual save-on RPC. - Source proof is checked before and after quiesce and on each copy reconcile. Observed workload/process replacement fails the Backup before it can report a restorable success. Owner recovery retains the pod UID even if the Backup is force-deleted, drains predecessor workers, and never sends its cleanup to a successor pod. - Imported legacy recovery records and backups without a recorded Pod UID cannot call unbound save-on, even if an agent is available. Their cleanup intent remains durable until an uncached read proves the game Pod absent or terminal and not terminating; a live or unready replacement is not exit proof. - Upgrade the operator and agent image together and allow existing GameServer pods to roll out the new agent and `GAMEPLANE_POD_UID` DownwardAPI env. Older agents fail the identity preflight before cleanup debt, so the workload can continue its upgrade after terminal cleanup releases the owner. Retry failed copies after rollout; preflight never bypasses protection. This does not automatically stop a running game. - Every copy, including `quiesce=false` and CSI snapshots, acquires the shared lifecycle owner before starting. The Backup pins its target in `backup.gameplane.local/target`; `backup.gameplane.local/owner` remains a compatibility marker. The shared owner is described below. Quiesce cleanup intent is recorded on both the Backup and the surviving GameServer before calling the agent, including partial failures. Cleanup waits for live copy workers; deletion cancels the copy Job with foreground propagation first. Every Backup carries the cleanup finalizer. Pre-upgrade quiesced workers are observed through completion even when overlapping legacy copies cannot acquire the new exclusive owner. - - Orphan recovery preserves predecessor operation identities in the shared owner's `recovery` array, atomically with ownership transfer. Existing `backup.gameplane.local/recovering-owners` lineage is imported during migration. Deleting a recovering owner cannot hide an earlier worker. Normal deletion, orphan recovery, and legacy cleanup also wait for pending owned CSI captures to report ready or an explicit error; VolumeSnapshots are reclaimed by owner-reference GC after cleanup. - - VolumeSnapshot API availability is checked before quiescing. Snapshot creation errors fail the Backup and run unquiesce cleanup immediately instead of retrying creation indefinitely with saving disabled. API/creation failures persist `backup.gameplane.local/snapshot-aborted` to allow cleanup without waiting for a failed/nonrestorable capture. Failed unquiesce remains visible and retried. A transport failure can leave a CSI snapshot object despite a Failed Backup; this does not make that Backup restorable. + - Orphan recovery preserves predecessor operation identities in the shared owner's `recovery` array, atomically with ownership transfer. Existing `backup.gameplane.local/recovering-owners` lineage is imported during migration. Deleting a recovering owner cannot hide an earlier worker. Normal CSI observation waits for ready/error, but cancellation durably invalidates incomplete captures before save-on. Force-deleted Backup recovery marks the exact owned snapshot aborted and requests UID-bound deletion; a missing Backup cannot expose that snapshot as a valid Gameplane capture. + - CSI capture has a persisted deadline before quiesce, default 10m (`--backup-csi-capture-timeout`, Helm `operator.backupCSICaptureTimeout`), independent of Restic. API/creation errors, deadline expiry and deletion persist `backup.gameplane.local/snapshot-aborted` as a terminal decision, clear snapshot identity and fail the Backup before save-on. Late readiness cannot revive it. Snapshot/content identity is retained separately for reclamation. Cleanup verifies snapshot ownership and recorded UID, content reference UID/name/namespace and content UID, and deletes with UID preconditions. `Retain` or unverified resources remain for administrator action with names/UIDs in `SnapshotCleaned` conditions/events. Cleanup errors retry independently of save-on; deletion retains the Backup finalizer for pending reclamation but does not retain the game owner after confirmed save-on. Cluster RBAC adds content delete for verified `Delete`-policy resources. + - New PVCs referencing a Gameplane Backup-owned VolumeSnapshot require uncached successful-owner/snapshotID/UID/readiness checks and reject abort markers, including raw GameServer dataSource references. Existing PVCs do not depend on retained source metadata. External snapshots without a Gameplane Backup owner preserve the administrator direct-CRD contract; API authorization is separate. - Restic Job: mount data PVC, run restic backup to configured destination (S3, B2, rest-server, local fs). Job carries BackoffLimit and ActiveDeadlineSeconds from the reconciler's JobBackoffLimit / JobActiveDeadlineSeconds fields (operator flags --backup-job-backoff-limit / --backup-job-active-deadline-seconds, Helm operator.backupJobBackoffLimit / operator.backupJobActiveDeadlineSeconds; defaults 2 and 86400, applied when unset; spec 018 OD-027); only a Failed=True Job condition (set by jobPermanentlyFailed check) marks the Backup Failed. Pod SecurityContext takes FSGroup from the target GameTemplate's spec.security.fsGroup (fallback 65532 if template missing). - Snapshot tracking: parse restic JSON output from container logs to extract snapshot ID. - - Capture metadata: after ownership, workload fencing and quiesce admission, record `status.captureManifest` once before creating the first copy worker. It pins the GameServer UID, strategy, observed Pod/StatefulSet mount layout, PVC names/UIDs, included primary claim and excluded extra/mod/retained claims. `storage.extra[].backupRole` optionally declares `data` or `rebuildable`; absent declarations remain unclassified and excluded. Later template edits never rewrite the manifest. Existing workers and old terminal Backups have unknown historical coverage, never reconstructed from the current template. + - Capture metadata: after ownership, workload fencing and quiesce admission, record `status.captureManifest` once before authorizing the prepared copy worker or starting CSI capture. It pins the GameServer UID, strategy, observed Pod/StatefulSet mount layout, PVC names/UIDs, included primary claim and excluded extra/mod/retained claims. `storage.extra[].backupRole` optionally declares `data` or `rebuildable`; absent declarations remain unclassified and excluded. Later template edits never rewrite the manifest. Legacy workers and old terminal Backups have unknown historical coverage, never reconstructed from the current template. - Achieved consistency: the `Consistency` condition is separate from copy phase and volume coverage. It stays Unknown until a usable snapshot identity is recorded; failed copies report `CaptureFailed`. Agent-confirmed quiesce, already-stopped/drained sources, unsupported/no-agent raw copy and unquiesced CSI snapshots have distinct reasons. Quiesce depends on the module commands and does not imply that excluded volumes were captured. The API exposes the stored status and the backup drawer shows consistency and included/excluded claims. `StoppedSource` records an observation, without adding a stop/copy/restart policy or multi-volume backup. - Snapshot cleanup on delete (`backup_forget.go`): restic-strategy Backups carry `BackupSnapshotFinalizer` (`gameplane.local/backup-snapshot-finalizer`, added on first reconcile, also for pre-existing non-Failed Backups; not added for volume-snapshot or Failed Backups). `finalizeDelete` runs the unquiesce stage (`BackupFinalizer`) first, then `finalizeSnapshot`: it deletes the Backup's own still-running Job (foreground, awaited), waits while a non-terminal Restore pins the Backup or its snapshot id, then runs Job `-forget` (`restic unlock || true; restic forget --prune --retry-lock 10m`, backoff 3, deadline 1800s, 50m/128Mi requests and 512Mi memory limit, no PVC; the snapshot id is passed through env `FORGET_SNAPSHOT_ID` and must match `^[0-9a-f]{8,64}$`). Success sets `SnapshotForgotten=True` and a Normal `SnapshotForgotten` event; a Failed Job, a missing/incomplete repo Secret, a terminating namespace, or `maxSnapshotForgetWait` (45m, from annotation `backup.gameplane.local/forget-started-at`) sets `SnapshotForgotten=False` with a Warning `SnapshotForgetAbandoned` event and releases the Backup, leaving the snapshot in the repository. A Backup with no recorded snapshot id is released with `SnapshotForgetSkipped` when it may have written one. Backup Jobs run `restic backup /data --json --retry-lock 10m ...`. RBAC markers on `backup_controller.go` grant `backups` update/patch, `backups/status`, `backups/finalizers` and `jobs` create/update/patch/delete. Tests: `backup_forget_test.go`, `backup_forget_envtest_test.go`, `retention_terminating_test.go`; e2e `TestBackup_DeleteForgetsSnapshot` and the extended `TestBackupSchedule_RetentionTrimsPast`. Spec: `specs/done_020-backup-snapshot-forget/`. - Status: report phase (Pending/Running/Succeeded/Failed), error details. @@ -568,6 +571,10 @@ different copy under the old owner. See [operator operation recovery](../docs/op | `--agent-image-pull-policy` | string | `` | ImagePullPolicy override (Always/IfNotPresent/Never); empty leaves unset. | | `--config-init-image` | string | `controller.DefaultConfigInitImage` | Init container for rendering config files onto data volume. | | `--restic-image` | string | `controller.DefaultResticImage` | restic backup/restore Job image. | +| `--backup-preparation-timeout` | duration | `10m` | Maximum preparation before copy authorization. | +| `--backup-quiesce-timeout` | duration | `5m` | Live quiesced Restic copy budget; drainage precedes save-on. | +| `--backup-csi-capture-timeout` | duration | `10m` | CSI capture budget before terminal invalidation. | +| `--backup-hard-same-node` | bool | `false` | Require the observed game node instead of preferring it. | | `--backup-job-backoff-limit` | int64 | 2 | BackoffLimit of restic backup/restore Jobs; must be >= 0. | | `--backup-job-active-deadline-seconds` | int64 | 86400 | ActiveDeadlineSeconds of restic backup/restore Jobs; must be > 0. | | `--sentinel-image` | string | `controller.DefaultSentinelImage` | Wake-on-connect sentinel container image. | diff --git a/test/e2e/api_owner_only_ops_e2e_test.go b/test/e2e/api_owner_only_ops_e2e_test.go index 99897640a..f85ef31c6 100644 --- a/test/e2e/api_owner_only_ops_e2e_test.go +++ b/test/e2e/api_owner_only_ops_e2e_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) @@ -110,27 +111,51 @@ func TestAPI_OwnerOnlyServerOperations_RequireOwnerOrAdmin(t *testing.T) { } }) - t.Run("operator may delete a server it owns", func(t *testing.T) { - createGameServerViaAPI(t, op, ns, opServer, tmpl) - resp, body, err := op.Delete("/servers/" + opServer + "?namespace=" + ns) + // Creation starts controller writes that can invalidate the API's + // resourceVersion delete precondition. Only retry an explicit conflict, + // through a fresh request that must pass authorization again. + deleteServer := func(t *testing.T, cli *APIClient, name string) { + t.Helper() + ctx, cancel := context.WithTimeout(t.Context(), time.Minute) + defer cancel() + servers := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns) + created, err := servers.Get(ctx, name, metav1.GetOptions{}) if err != nil { - t.Fatalf("operator DELETE own server: %v", err) - } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusNoContent { - t.Errorf("operator DELETE own server: status=%d want=204 body=%s", resp.StatusCode, string(body)) + t.Fatalf("get server before DELETE: %v", err) } + envInstance.Eventually(t, time.Minute, func() (bool, string) { + current, err := servers.Get(ctx, name, metav1.GetOptions{}) + if err != nil || current.GetUID() != created.GetUID() { + t.Fatalf("server identity changed before DELETE: object=%v err=%v", current, err) + } + resp, body, err := cli.DoContext(ctx, http.MethodDelete, "/servers/"+name+"?namespace="+ns, nil) + if err != nil { + t.Fatalf("DELETE %s: %v", name, err) + } + _ = resp.Body.Close() + if resp.StatusCode == http.StatusConflict { + return false, "DELETE conflicted with a controller update: " + string(body) + } + if resp.StatusCode != http.StatusNoContent { + t.Fatalf("DELETE %s: status=%d want=204 body=%s", name, resp.StatusCode, body) + } + remaining, err := servers.Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return true, "" + } + if err != nil || remaining.GetUID() != created.GetUID() || remaining.GetDeletionTimestamp().IsZero() { + t.Fatalf("DELETE did not remove or mark the original server for deletion: object=%v err=%v", remaining, err) + } + return true, "" + }) + } + t.Run("operator may delete a server it owns", func(t *testing.T) { + createGameServerViaAPI(t, op, ns, opServer, tmpl) + deleteServer(t, op, opServer) }) t.Run("admin may delete a server it does not own", func(t *testing.T) { createGameServerViaAPI(t, op, ns, opServer2, tmpl) - resp, body, err := admin.Delete("/servers/" + opServer2 + "?namespace=" + ns) - if err != nil { - t.Fatalf("admin DELETE operator's server: %v", err) - } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusNoContent { - t.Errorf("admin DELETE operator's server: status=%d want=204 body=%s", resp.StatusCode, string(body)) - } + deleteServer(t, admin, opServer2) }) } diff --git a/test/e2e/backup_preparation_e2e_test.go b/test/e2e/backup_preparation_e2e_test.go new file mode 100644 index 000000000..fd5ec305c --- /dev/null +++ b/test/e2e/backup_preparation_e2e_test.go @@ -0,0 +1,167 @@ +//go:build e2e + +package e2e + +import ( + "context" + "fmt" + "regexp" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// Real scheduler, RWO PVC and repository failure/recovery on two schedulable +// nodes. Kind's local-path volume supplies node affinity; attachable CSI +// drivers still need their own multi-attach qualification. Zero API logins. +func TestBackup_PreparationAndSameNodeCopy(t *testing.T) { + t.Parallel() + ctx := t.Context() + nodes, err := envInstance.K8s.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + t.Fatal(err) + } + eligible := 0 + for _, node := range nodes.Items { + ready, tainted := false, node.Spec.Unschedulable + for _, condition := range node.Status.Conditions { + ready = ready || (condition.Type == corev1.NodeReady && condition.Status == corev1.ConditionTrue) + } + for _, taint := range node.Spec.Taints { + tainted = tainted || taint.Effect == corev1.TaintEffectNoSchedule || taint.Effect == corev1.TaintEffectNoExecute + } + if ready && !tainted { + eligible++ + } + } + if eligible < 2 { + t.Fatal("backup preparation requires the management Kind profile with two schedulable workers") + } + suffix := fmt.Sprintf("%d", time.Now().UnixNano()) + ns, gs, template, name := "gameplane-games", "prepared-source-"+suffix, "prepared-template-"+suffix, "prepared-copy-"+suffix + ensureResticRepo(t) + applyBusyboxTemplate(t, template) + applyBusyboxGameServer(t, ns, gs, template) + waitPVCBound(t, ns, gs+"-data", 90*time.Second) + var sourceNode string + envInstance.Eventually(t, 2*time.Minute, func() (bool, string) { + pod, err := envInstance.K8s.CoreV1().Pods(ns).Get(ctx, gs+"-0", metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + for _, status := range pod.Status.ContainerStatuses { + if status.Name == "game" && status.Ready { + sourceNode = pod.Spec.NodeName + return sourceNode != "", "source has no node" + } + } + return false, "game container not ready" + }) + pvc, err := envInstance.K8s.CoreV1().PersistentVolumeClaims(ns).Get(ctx, gs+"-data", metav1.GetOptions{}) + if err != nil || len(pvc.Spec.AccessModes) != 1 || pvc.Spec.AccessModes[0] != corev1.ReadWriteOnce { + t.Fatalf("fixture is not RWO: %v %+v", err, pvc) + } + if out, err := envInstance.KubectlExec(t, ns, "pod/"+gs+"-0", "sh", "-c", "echo prepared-copy-marker > /data/prepared-marker.txt && sync"); err != nil { + t.Fatalf("write marker: %v %s", err, out) + } + repo, err := envInstance.K8s.CoreV1().Secrets(ns).Get(ctx, "e2e-restic-creds", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + repo = repo.DeepCopy() + repo.ObjectMeta = metav1.ObjectMeta{Name: "prepared-repo-" + suffix, Namespace: ns} + // Authentication failure against the warm repository is deterministic. + // A refused HTTP endpoint exercises Restic's network retry budget and may + // keep the init container running longer than this preparation assertion. + validPassword := append([]byte(nil), repo.Data["password"]...) + repo.Data["password"] = []byte("incorrect-repository-password-" + suffix) + if _, err := envInstance.K8s.CoreV1().Secrets(ns).Create(ctx, repo, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + _ = envInstance.K8s.CoreV1().Secrets(ns).Delete(cleanupCtx, repo.Name, metav1.DeleteOptions{}) + }) + backup := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "gameplane.local/v1alpha1", "kind": "Backup", "metadata": map[string]any{"name": name, "namespace": ns}, + "spec": map[string]any{"serverRef": map[string]any{"name": gs}, "repoRef": map[string]any{"name": repo.Name, "key": "repo"}, "quiesce": true}}} + if _, err := envInstance.Dyn.Resource(backupGVR).Namespace(ns).Create(ctx, backup, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { deleteBackupCR(t, ns, name) }) + var failedWorker *corev1.Pod + envInstance.Eventually(t, 2*time.Minute, func() (bool, string) { + got, err := envInstance.Dyn.Resource(backupGVR).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + if got.GetAnnotations()["backup.gameplane.local/quiesce-attempted"] != "" || got.GetAnnotations()["backup.gameplane.local/authorized-worker"] != "" { + t.Fatal("repository preparation failure disabled saving or authorized copy") + } + pods, err := envInstance.K8s.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{LabelSelector: "job-name=" + name}) + if err != nil { + return false, err.Error() + } + for i := range pods.Items { + pod := &pods.Items[i] + if pod.Spec.NodeName != "" && pod.Spec.NodeName != sourceNode { + t.Fatalf("RWO worker scheduled away from source: %s vs %s", pod.Spec.NodeName, sourceNode) + } + for _, status := range pod.Status.InitContainerStatuses { + if status.Name == "restic-init" && ((status.State.Terminated != nil && status.State.Terminated.ExitCode != 0) || (status.LastTerminationState.Terminated != nil && status.LastTerminationState.Terminated.ExitCode != 0)) { + failedWorker = pod + return true, "" + } + } + } + return false, "waiting for actual repository preparation failure" + }) + repo, err = envInstance.K8s.CoreV1().Secrets(ns).Get(ctx, repo.Name, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + repo.Data["password"] = validPassword + if _, err := envInstance.K8s.CoreV1().Secrets(ns).Update(ctx, repo, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + uid := failedWorker.UID + if err := envInstance.K8s.CoreV1().Pods(ns).Delete(ctx, failedWorker.Name, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); err != nil { + t.Fatal(err) + } + var snapshotID string + envInstance.Eventually(t, 5*time.Minute, func() (bool, string) { + got, err := envInstance.Dyn.Resource(backupGVR).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + phase, _, _ := unstructured.NestedString(got.Object, "status", "phase") + if phase == "Failed" { + t.Fatalf("copy failed after repository recovery: %+v", got.Object["status"]) + } + snapshotID, _, _ = unstructured.NestedString(got.Object, "status", "snapshotID") + return phase == "Succeeded" && snapshotID != "", "phase=" + phase + }) + pods, err := envInstance.K8s.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{LabelSelector: "job-name=" + name}) + if err != nil { + t.Fatal(err) + } + completed := false + for _, pod := range pods.Items { + if pod.Status.Phase == corev1.PodSucceeded { + completed = true + if pod.Spec.NodeName != sourceNode || pod.Spec.AutomountServiceAccountToken == nil || *pod.Spec.AutomountServiceAccountToken { + t.Fatal("completed worker violated placement or credential boundary") + } + } + } + if !completed { + t.Fatal("no completed real restic worker") + } + if !regexp.MustCompile(`^[0-9a-f]{8,64}$`).MatchString(snapshotID) { + t.Fatalf("unexpected snapshot identity %q", snapshotID) + } + runResticJob(t, fmt.Sprintf("restic restore %s --target /tmp/prepared-restore --retry-lock 5m && test \"$(cat /tmp/prepared-restore/data/prepared-marker.txt)\" = prepared-copy-marker", snapshotID)) +} diff --git a/test/e2e/buckets.sh b/test/e2e/buckets.sh index ba9f800ec..eca94e11c 100755 --- a/test/e2e/buckets.sh +++ b/test/e2e/buckets.sh @@ -110,6 +110,7 @@ TestAddressPool_PoolExhausted TestAPI_EventStreamAndRoleEdits_FollowCallerPermissions TestAPI_I18n_LanguagePref TestAPI_I18n_NotifyLanguage +TestAPI_I18n_NotifyTestSendRateLimit TestAPI_I18n_InstanceDefault TestE2E_ResticRepoReady TestCISelection_AgentOnlyChange @@ -428,10 +429,15 @@ TestManagementPlacementAndDisruption EOF } +bucket_backup_preparation() { cat <<'EOF' +TestBackup_PreparationAndSameNodeCopy +EOF +} + unbucketed() { :; } bucket_names() { - printf '%s\n' operator api-auth api-roles api-rbac api-agent api-mods ratelimit bot-fast bot-heavy multicluster upgrade postgres-availability telemetry capture-overhead optional-components restricted-workloads management-placement + printf '%s\n' operator api-auth api-roles api-rbac api-agent api-mods ratelimit bot-fast bot-heavy multicluster upgrade postgres-availability telemetry capture-overhead optional-components restricted-workloads management-placement backup-preparation } list_bucket() { @@ -453,6 +459,7 @@ list_bucket() { optional-components) bucket_optional_components ;; restricted-workloads) bucket_restricted_workloads ;; management-placement) bucket_management_placement ;; + backup-preparation) bucket_backup_preparation ;; *) echo "unknown bucket: $1 (known: $(bucket_names | tr '\n' ' '))" >&2 exit 2 diff --git a/test/e2e/ci_selection_e2e_test.go b/test/e2e/ci_selection_e2e_test.go index 02949e334..de2b09a51 100644 --- a/test/e2e/ci_selection_e2e_test.go +++ b/test/e2e/ci_selection_e2e_test.go @@ -22,7 +22,7 @@ import ( // ciSelectionBuckets mirrors E2E_BUCKETS in hack/ci_scope.py (the e2e-go matrix). var ciSelectionBuckets = []string{ "operator", "api-auth", "api-roles", "api-rbac", "api-agent", "api-mods", - "telemetry", "optional-components", "restricted-workloads", "management-placement", + "telemetry", "optional-components", "restricted-workloads", "management-placement", "backup-preparation", } // ciSelection is the decoded output of one `ci_scope.py e2e` run. diff --git a/test/e2e/failure_paths_e2e_test.go b/test/e2e/failure_paths_e2e_test.go index 6cf00a068..e497340a4 100644 --- a/test/e2e/failure_paths_e2e_test.go +++ b/test/e2e/failure_paths_e2e_test.go @@ -123,7 +123,8 @@ func TestBackup_FailsOnBadCredentials(t *testing.T) { } phase, _, _ := unstructured.NestedString(got.Object, "status", "phase") if phase != "Succeeded" { - return false, "seed phase=" + phase + message, _, _ := unstructured.NestedString(got.Object, "status", "message") + return false, "seed phase=" + phase + ": " + message } return true, "" }) diff --git a/test/e2e/fixtures/notify-sink.yaml b/test/e2e/fixtures/notify-sink.yaml index 110088c24..ed4df0162 100644 --- a/test/e2e/fixtures/notify-sink.yaml +++ b/test/e2e/fixtures/notify-sink.yaml @@ -11,8 +11,13 @@ metadata: data: hook: | #!/bin/sh - { printf '%s\t' "$QUERY_STRING"; head -c "${CONTENT_LENGTH:-0}"; echo; } >> /tmp/hooks.log + set -eu + printf '%s\t%s\tstart\t%s\n' "$(date -u +%FT%TZ)" "$QUERY_STRING" "${CONTENT_LENGTH:-0}" >> /tmp/requests.log + body=$(head -c "${CONTENT_LENGTH:-0}") + printf '%s\t%s\n' "$QUERY_STRING" "$body" >> /tmp/hooks.log + printf '%s\t%s\tbody-read\n' "$(date -u +%FT%TZ)" "$QUERY_STRING" >> /tmp/requests.log printf 'Status: 204 No Content\r\n\r\n' + printf '%s\t%s\tresponse-written\n' "$(date -u +%FT%TZ)" "$QUERY_STRING" >> /tmp/requests.log --- apiVersion: apps/v1 kind: Deployment diff --git a/test/e2e/garrysmod_bot_e2e_test.go b/test/e2e/garrysmod_bot_e2e_test.go index 384bba92e..f2b1809e1 100644 --- a/test/e2e/garrysmod_bot_e2e_test.go +++ b/test/e2e/garrysmod_bot_e2e_test.go @@ -15,7 +15,7 @@ import ( // // Garry's Mod is part of the fast set because it uses the Source engine (A2S queries), // which is shared with CS2. The server boots from a pre-baked image (no steamcmd download) -// and is lightweight. +// so the CI job reserves disk space for both downloaded and extracted layers. // // The test is deliberately NOT t.Parallel(): concurrent real-game boots OOM-starve a // single kind node. diff --git a/test/e2e/i18n_notify_language_e2e_test.go b/test/e2e/i18n_notify_language_e2e_test.go index 22a569503..f854c3a99 100644 --- a/test/e2e/i18n_notify_language_e2e_test.go +++ b/test/e2e/i18n_notify_language_e2e_test.go @@ -5,12 +5,17 @@ package e2e import ( "context" "encoding/json" + "errors" "fmt" "io" + "net" "net/http" + "net/http/httptest" "path/filepath" + "strconv" "strings" "sync" + "sync/atomic" "testing" "time" ) @@ -22,6 +27,151 @@ import ( // restore the default before releasing it. var i18nInstanceDefaultMu sync.Mutex +// i18nNotifyTestSend retries only requests rejected before delivery by the +// notification test limiter. Its delta-seconds Retry-After must elapse before +// another send; the caller's context bounds both requests and the wait. +func i18nNotifyTestSend(ctx context.Context, admin *APIClient, sink string) (*http.Response, []byte, error) { + for { + if err := ctx.Err(); err != nil { + return nil, nil, err + } + resp, body, err := admin.DoContext(ctx, http.MethodPost, + "/admin/notifications/sinks/"+sink+"/test", nil) + if err != nil || resp.StatusCode != http.StatusTooManyRequests { + return resp, body, err + } + seconds, err := strconv.ParseInt(resp.Header.Get("Retry-After"), 10, 32) + if err != nil { + return resp, body, fmt.Errorf("notify test %s: parse Retry-After %q: %w", sink, resp.Header.Get("Retry-After"), err) + } + if seconds <= 0 { + return resp, body, fmt.Errorf("notify test %s: Retry-After must be positive, got %q", sink, resp.Header.Get("Retry-After")) + } + timer := time.NewTimer(time.Duration(seconds) * time.Second) + select { + case <-ctx.Done(): + timer.Stop() + return resp, body, ctx.Err() + case <-timer.C: + } + } +} + +func TestAPI_I18n_NotifyTestSendRateLimit(t *testing.T) { + t.Parallel() + t.Run("honors retry after", func(t *testing.T) { + var calls atomic.Int32 + var rejectedAt atomic.Int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if calls.Add(1) == 1 { + rejectedAt.Store(time.Now().UnixNano()) + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + if time.Since(time.Unix(0, rejectedAt.Load())) < time.Second { + t.Error("retried before Retry-After elapsed") + } + _, _ = io.WriteString(w, `{"delivered":true}`) + })) + defer server.Close() + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + resp, body, err := i18nNotifyTestSend(ctx, &APIClient{BaseURL: server.URL, HTTP: server.Client()}, "test") + if resp != nil { + _ = resp.Body.Close() + } + if err != nil || resp.StatusCode != http.StatusOK || string(body) != `{"delivered":true}` || calls.Load() != 2 { + t.Fatalf("send: response=%v body=%s calls=%d err=%v", resp, body, calls.Load(), err) + } + }) + t.Run("cancels retry wait", func(t *testing.T) { + var calls atomic.Int32 + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls.Add(1) + w.Header().Set("Retry-After", "30") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + // APIClient closes the consumed response before returning it. Cancel + // there so the retry wait, rather than the HTTP transport, sees it. + client := server.Client() + client.Transport = i18nNotifyCancelTransport{base: client.Transport, cancel: cancel} + type sendResult struct { + resp *http.Response + err error + } + done := make(chan sendResult, 1) + go func() { + resp, _, err := i18nNotifyTestSend(ctx, &APIClient{BaseURL: server.URL, HTTP: client}, "test") + if resp != nil { + _ = resp.Body.Close() + } + done <- sendResult{resp: resp, err: err} + }() + deadline := time.NewTimer(5 * time.Second) + defer deadline.Stop() + var result sendResult + select { + case result = <-done: + case <-deadline.C: + t.Fatal("canceled retry wait did not finish within 5 seconds") + } + if result.resp == nil || result.resp.StatusCode != http.StatusTooManyRequests || !errors.Is(result.err, context.Canceled) || calls.Load() != 1 { + t.Fatalf("canceled send: response=%v calls=%d err=%v", result.resp, calls.Load(), result.err) + } + }) + for _, status := range []int{http.StatusBadGateway, http.StatusTooManyRequests} { + t.Run(strconv.Itoa(status), func(t *testing.T) { + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls.Add(1) + w.WriteHeader(status) + _, _ = io.WriteString(w, "rejected") + })) + defer server.Close() + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() + resp, body, err := i18nNotifyTestSend(ctx, &APIClient{BaseURL: server.URL, HTTP: server.Client()}, "test") + if resp != nil { + _ = resp.Body.Close() + } + if resp == nil || resp.StatusCode != status || string(body) != "rejected" || calls.Load() != 1 { + t.Fatalf("send: response=%v body=%s calls=%d err=%v", resp, body, calls.Load(), err) + } + if (err != nil) != (status == http.StatusTooManyRequests) { + t.Fatalf("unexpected error for status %d: %v", status, err) + } + }) + } +} + +type i18nNotifyCancelTransport struct { + base http.RoundTripper + cancel context.CancelFunc +} + +func (c i18nNotifyCancelTransport) RoundTrip(req *http.Request) (*http.Response, error) { + resp, err := c.base.RoundTrip(req) + if err == nil { + resp.Body = i18nNotifyCancelBody{ReadCloser: resp.Body, cancel: c.cancel} + } + return resp, err +} + +type i18nNotifyCancelBody struct { + io.ReadCloser + cancel context.CancelFunc +} + +func (c i18nNotifyCancelBody) Close() error { + err := c.ReadCloser.Close() + c.cancel() + return err +} + // i18nAuthLocale reads the unauthenticated GET /auth/locale through a // port-forward to the API and returns its defaultLanguage. The endpoint // returns exactly one key, so any other shape fails the test. @@ -160,8 +310,8 @@ func TestAPI_I18n_NotifyLanguage(t *testing.T) { i18nInstanceDefaultMu.Lock() defer i18nInstanceDefaultMu.Unlock() - // Receiver: a hard gate. The rollout waits for the TCP readiness probe, so - // the test-send cannot race the receiver's startup. + // The rollout proves the receiver is listening. The API-to-Service CGI + // readiness send below also gates DNS, endpoint routing, and body handling. fixture, err := filepath.Abs(filepath.Join("fixtures", "notify-sink.yaml")) if err != nil { t.Fatalf("resolve notify-sink fixture: %v", err) @@ -172,6 +322,29 @@ func TestAPI_I18n_NotifyLanguage(t *testing.T) { defer cancel() _, _ = envInstance.Kubectl(ctx, "delete", "-f", fixture, "--ignore-not-found") }) + // Cleanup runs in reverse registration order, so retain receiver evidence + // before deleting it. t.Context() is already canceled when cleanups run. + t.Cleanup(func() { + if !t.Failed() { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + defer cancel() + commands := [][]string{ + {"get", "pods", "-o", "wide"}, + {"get", "service", "e2e-notify-sink", "-o", "yaml"}, + {"get", "endpointslices", "-l", "kubernetes.io/service-name=e2e-notify-sink", "-o", "yaml"}, + {"describe", "pods", "-l", "app.kubernetes.io/name=e2e-notify-sink"}, + {"logs", "deploy/e2e-notify-sink", "--all-containers=true", "--tail=100"}, + {"exec", "deploy/e2e-notify-sink", "--", "sh", "-c", `for file in /tmp/requests.log /tmp/hooks.log; do printf '\n%s\n' "$file"; cat "$file"; done`}, + } + for _, args := range commands { + commandCtx, stop := context.WithTimeout(ctx, 7*time.Second) + out, err := envInstance.Kubectl(commandCtx, append([]string{"-n", "gameplane-system"}, args...)...) + stop() + t.Logf("notify sink diagnostics %v: %v\n%s", args, err, out) + } + }) if out, err := envInstance.Kubectl(t.Context(), "rollout", "status", "-n", "gameplane-system", "deploy/e2e-notify-sink", "--timeout=2m"); err != nil { t.Fatalf("notify sink receiver not ready: %v\n%s", err, out) @@ -190,11 +363,14 @@ func TestAPI_I18n_NotifyLanguage(t *testing.T) { } }() - sinks := []struct{ name, query string }{ + type sinkTarget struct{ name, query string } + sinks := []sinkTarget{ {"e2e-notify-fr", "sink=fr"}, {"e2e-notify-default", "sink=default"}, } - for _, s := range sinks { + readinessSink := sinkTarget{"e2e-notify-readiness", "probe=readiness"} + configuredSinks := append([]sinkTarget{readinessSink}, sinks...) + for _, s := range configuredSinks { name := s.name defer func() { r, _, _ := admin.Delete("/admin/notifications/sinks/" + name + "/secret") @@ -205,7 +381,7 @@ func TestAPI_I18n_NotifyLanguage(t *testing.T) { } // Secrets hold each sink's URL; configRef must name the returned Secret. - for _, s := range sinks { + for _, s := range configuredSinks { url := fmt.Sprintf("http://e2e-notify-sink.gameplane-system.svc.cluster.local:8080/cgi-bin/hook?%s", s.query) resp, body, err := admin.Do(http.MethodPut, "/admin/notifications/sinks/"+s.name+"/secret", map[string]string{"kind": "webhook", "url": url, "authorization": ""}) @@ -231,9 +407,12 @@ func TestAPI_I18n_NotifyLanguage(t *testing.T) { } // The fr sink names French explicitly; the default sink omits language. - // Both stay disabled so the notifier never delivers real events here. + // All stay disabled so the notifier never delivers real events here. The + // readiness sink uses French too, exercising the same UTF-8 body path. resp, body, err := admin.Do(http.MethodPut, "/admin/config/notifications", map[string]any{ "sinks": []map[string]any{ + {"name": readinessSink.name, "kind": "webhook", "enabled": false, + "configRef": "gameplane-notify-" + readinessSink.name, "language": "fr"}, {"name": "e2e-notify-fr", "kind": "webhook", "enabled": false, "configRef": "gameplane-notify-e2e-notify-fr", "language": "fr"}, {"name": "e2e-notify-default", "kind": "webhook", "enabled": false, @@ -259,14 +438,57 @@ func TestAPI_I18n_NotifyLanguage(t *testing.T) { t.Fatalf("instance default language = %q, want en before the test-send", defaultTag) } + // Establish the exact API -> Service -> CGI POST path before measuring the + // language contract. A probe may be delivered despite a lost response, so + // its distinct query cannot satisfy either language assertion below. + // Failed readiness deliveries also consume the shared test-send bucket. + // Allow its advertised 30-second cooldown within each bounded send budget. + const sendBudget = time.Minute + readinessCtx, cancelReadiness := context.WithTimeout(t.Context(), sendBudget) + defer cancelReadiness() + readinessAttempts := 0 + envInstance.Eventually(t, sendBudget, func() (bool, string) { + readinessAttempts++ + resp, b, err := i18nNotifyTestSend(readinessCtx, admin, readinessSink.name) + if resp != nil { + _ = resp.Body.Close() + } + if err != nil { + var networkErr net.Error + if readinessCtx.Err() != nil || !errors.As(err, &networkErr) { + t.Fatalf("notify sink readiness request: %v", err) + } + t.Logf("notify sink readiness attempt %d: %v", readinessAttempts, err) + return false, fmt.Sprintf("notify sink readiness request: %v", err) + } + if resp.StatusCode == http.StatusBadGateway && + (strings.Contains(string(b), "connection refused") || strings.Contains(string(b), "context deadline exceeded")) { + t.Logf("notify sink readiness attempt %d: %s", readinessAttempts, b) + return false, fmt.Sprintf("notify sink readiness: %s", b) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("notify sink readiness: status=%d body=%s", resp.StatusCode, b) + } + var delivered struct { + Delivered bool `json:"delivered"` + } + if err := json.Unmarshal(b, &delivered); err != nil || !delivered.Delivered { + t.Fatalf("notify sink readiness: delivered not true; body=%s", b) + } + return true, "" + }) + cancelReadiness() + for _, s := range sinks { // A ready pod can still refuse connections on its Service IP for a - // moment while kube-proxy programs the new endpoint, so only that - // refusal is retried. A refused send reaches no receiver, so it adds - // no log line. + // moment while kube-proxy programs the new endpoint. Apart from the + // limiter's pre-delivery 429, only that refusal is retried. A refused + // send reaches no receiver, so it adds no log line. + sendCtx, cancelSend := context.WithTimeout(t.Context(), sendBudget) + defer cancelSend() var body []byte - envInstance.Eventually(t, 30*time.Second, func() (bool, string) { - resp, b, err := admin.Post("/admin/notifications/sinks/"+s.name+"/test", nil) + envInstance.Eventually(t, sendBudget, func() (bool, string) { + resp, b, err := i18nNotifyTestSend(sendCtx, admin, s.name) if resp != nil { _ = resp.Body.Close() } @@ -282,6 +504,7 @@ func TestAPI_I18n_NotifyLanguage(t *testing.T) { body = b return true, "" }) + cancelSend() var delivered struct { Delivered bool `json:"delivered"` } diff --git a/test/e2e/specs.md b/test/e2e/specs.md index 9cd5bd94d..0fdf94e2c 100644 --- a/test/e2e/specs.md +++ b/test/e2e/specs.md @@ -84,6 +84,17 @@ a cordon, readiness of the surviving replica, and singleton API eviction. Node labels and cordons are restored in cleanup. This validates scheduler/PDB behavior, not application HA or storage migration. +### Backup preparation acceptance + +The `backup-preparation` bucket also uses the `management` Kind profile and zero +API logins. It verifies a real RWO copy on the game's node across two schedulable +workers, repository authentication failure before quiesce/authorization, retry after +repository recovery, token-gated Restic execution and restoration of a marker. +Kind local-path PV affinity supplies the storage placement constraint; this is +not an attachable CSI driver or multi-attach qualification. Unit/envtest regressions +cover stale worker/source identity, persisted budgets, delayed drainage, CSI +abort/reclamation, retained/replaced content and raw snapshot-source rejection. + ## Non-goals / boundaries - Does **not** replace the unit or envtest tiers for any other module — it exercises integration behavior, not the exhaustive branch coverage those tiers own (this module carries no `.testcoverage.yml` coverage gate).