From 77bb5b19039da7b81beb62f75c982d52a5af3bba Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 21 Oct 2026 11:51:33 -0400 Subject: [PATCH 01/20] fix: back up stopped servers and bind quiesce to pod identity Signed-off-by: Baylem --- agent/cmd/main.go | 7 + agent/internal/quiesce/quiesce_test.go | 28 ++ agent/internal/target/target.go | 11 +- agent/specs.md | 3 + i18n/locales/en/app.json | 4 + operator/hack/condition-ids/main.go | 1 + operator/internal/agent/client.go | 39 ++ operator/internal/agent/client_test.go | 66 +++ .../internal/controller/backup_controller.go | 117 ++++-- .../controller/backup_envtest_test.go | 16 + operator/internal/controller/backup_guard.go | 39 +- .../internal/controller/backup_guard_test.go | 20 +- .../internal/controller/backup_preparation.go | 234 +++++++++++ .../controller/backup_stopped_test.go | 382 ++++++++++++++++++ .../controller/gameserver_controller.go | 1 + .../controller/helpers_envtest_test.go | 8 + .../controller/lifecycle_operation.go | 34 +- operator/specs.md | 6 +- test/e2e/backup_stopped_e2e_test.go | 112 +++++ test/e2e/buckets.sh | 1 + 20 files changed, 1074 insertions(+), 55 deletions(-) create mode 100644 operator/internal/controller/backup_preparation.go create mode 100644 operator/internal/controller/backup_stopped_test.go create mode 100644 test/e2e/backup_stopped_e2e_test.go diff --git a/agent/cmd/main.go b/agent/cmd/main.go index 8ce7f15f8..0b5c86d44 100644 --- a/agent/cmd/main.go +++ b/agent/cmd/main.go @@ -209,6 +209,13 @@ func main() { protected.Use(authCheck.Middleware, target.Guard(serverUID)) mountProtected(protected) }) + // Backups bind save-off/save-on to the process they prepared. The service + // may already route to a replacement pod, which must reject the old UID. + r.Route("/v1/pods/{uid}", func(protected chi.Router) { + protected.Use(authCheck.Middleware, target.Guard(envOr("GAMEPLANE_POD_UID", "")), middleware.Timeout(30*time.Second)) + protected.Get("/identity", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) }) + quiesce.Mount(protected, rconClient, gameName, quiesceSpec) + }) srv := &http.Server{ Addr: addr, diff --git a/agent/internal/quiesce/quiesce_test.go b/agent/internal/quiesce/quiesce_test.go index 777ba9ade..62aca43a4 100644 --- a/agent/internal/quiesce/quiesce_test.go +++ b/agent/internal/quiesce/quiesce_test.go @@ -12,6 +12,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/GameplanePanel/gameplane/agent/internal/caps" + "github.com/GameplanePanel/gameplane/agent/internal/target" ) type fakeRcon struct { @@ -86,6 +87,33 @@ func newTestRouter(rc Rcon, spec *caps.Quiesce) *chi.Mux { return r } +func TestPodBoundRoutesRejectReplacementBeforeRCON(t *testing.T) { + for _, actual := range []string{"original", "replacement", ""} { + t.Run("pod="+actual, func(t *testing.T) { + rc := &fakeRcon{} + r := chi.NewRouter() + r.Route("/v1/pods/{uid}", func(bound chi.Router) { + bound.Use(target.Guard(actual)) + Mount(bound, rc, "test", minecraftQuiesceSpec()) + }) + for _, path := range []string{"quiesce", "unquiesce"} { + response := httptest.NewRecorder() + r.ServeHTTP(response, httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/v1/pods/original/"+path, nil)) + if actual == "original" { + if response.Code != http.StatusOK { + t.Fatalf("matching process rejected: %d", response.Code) + } + } else if response.Code != http.StatusNotFound { + t.Fatalf("wrong process admitted: %d", response.Code) + } + } + if (len(rc.calls()) > 0) != (actual == "original") { + t.Fatalf("commands reached wrong process: %v", rc.calls()) + } + }) + } +} + func decodeResponse(t *testing.T, body []byte) response { t.Helper() var got response diff --git a/agent/internal/target/target.go b/agent/internal/target/target.go index 6a1300631..3e79a3944 100644 --- a/agent/internal/target/target.go +++ b/agent/internal/target/target.go @@ -1,4 +1,4 @@ -// Package target binds versioned agent routes to an immutable GameServer UID. +// Package target binds versioned agent routes to an immutable resource UID. package target import ( @@ -7,12 +7,13 @@ import ( "github.com/go-chi/chi/v5" ) -// Guard rejects a route for any server other than the one that started this -// agent. Legacy routes stay available for local callers during rolling upgrades. -func Guard(serverUID string) func(http.Handler) http.Handler { +// Guard rejects a route for any resource other than the one that started this +// agent (GameServer or Pod, according to the route). Legacy routes stay available +// for local callers during rolling upgrades. +func Guard(resourceUID string) func(http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - if serverUID == "" || chi.URLParam(req, "uid") != serverUID { + if resourceUID == "" || chi.URLParam(req, "uid") != resourceUID { http.NotFound(w, req) return } diff --git a/agent/specs.md b/agent/specs.md index af1f57529..8c716bb9c 100644 --- a/agent/specs.md +++ b/agent/specs.md @@ -177,6 +177,8 @@ Resource usage env vars (set by the operator): | `/players/banned` | GET | Currently banned players; response: `[ { name, reason?, source? } ]`; 502 if RCON unavailable | | `/quiesce` | POST | Pause auto-saves before snapshot; response: `{ quiesced, reason? }` (boolean, reason optional); unsupported games return `quiesced: false` with a reason | | `/unquiesce` | POST | Resume auto-saves after snapshot; response: `{ quiesced, reason? }` | +| `/v1/pods/{uid}/identity` | GET | Operator preflight: 204 only when `{uid}` matches `GAMEPLANE_POD_UID`; never mutates game state | +| `/v1/pods/{uid}/quiesce`, `/v1/pods/{uid}/unquiesce` | POST | Pod-bound backup RPCs with the same response contract as the legacy routes; reject missing/mismatched Pod UID with 404 before RCON | | `/lifecycle/stop` | POST | Run stop sequence before scale-to-zero; RCON connection drop is the expected outcome | | `/actions/run` | POST | Execute module-declared operator action; request: `{ id, params }`; response: `{ ok, raw? }` | | `/status` | GET | Live game metrics (module-declared); response: bare JSON array `[ { id, displayName?, value, unit? } ]` | @@ -209,6 +211,7 @@ All endpoints on the `--addr` control mux, except `/healthz`, return `401 Unauth - **Extracted mod files stay world-readable**: `mods.moduleFileMode` is `0o644`, not a tighter `0o600`, because the mods volume is shared with the game container, which runs as whatever uid its image needs — a different uid than the agent's own. At `0o600` the game process couldn't read its own mods. This is the rationale behind the `.golangci.yml` gosec G302 exclusion scoped to `agent/internal/mods/mods.go`. - **Resource usage is in-pod**: The `usage` package reads from `/proc` or cgroups; no external metrics pipeline required. Cgroup mode is a fallback for older clusters; proc mode (default in production) requires the operator to set `ShareProcessNamespace: true`. - **Module capabilities drive behavior**: Every game-specific handler (players, quiesce, lifecycle, status, actions) reads its config from `--capabilities` (JSON unmarshaled into `caps.Spec`). New games require no agent code change. +- **Backup process identity**: The operator injects `GAMEPLANE_POD_UID` from `metadata.uid` through the DownwardAPI. Pod-bound backup routes require the existing mTLS authentication and check this UID at the final network hop, including when the Service resolves to a replacement pod. Missing UID fails closed. Deploy the updated agent image and allow pod rollout when upgrading the operator; older agents return 404 and the operator does not fall back to unbound routes. Legacy endpoints remain available to existing callers during rollout. - **RCON connection errors are graceful**: A lost RCON connection does not crash the agent. `console`, `players`, `quiesce`, and `lifecycle` handlers catch connection errors and return appropriate HTTP status (e.g., `502 Bad Gateway`). - **Source RCON sends complete frames**: Each AUTH or command frame is submitted in one socket write, with its existing byte-size limit, request ID, type, body, and two NUL terminators preserved. This avoids separate header-only writes rejected by vanilla Minecraft's RCON reader. A short write is an error and drops the connection without replaying the command; response grace and healthy connection reuse remain unchanged. - **Log streams are tail-only**: The `logs` package does not support random-access reads. It streams from the current end (live mode) or from file start (backlog mode); clients must handle partial output and reconnection. diff --git a/i18n/locales/en/app.json b/i18n/locales/en/app.json index 059470fd2..f27588870 100644 --- a/i18n/locales/en/app.json +++ b/i18n/locales/en/app.json @@ -599,6 +599,10 @@ "defaultMessage": "The server could not be resumed after the backup. Its world may still have auto-save turned off.", "description": "Unquiesced condition when the unfreeze failed." }, + "condition.unquiesced.sourceprocessgone": { + "defaultMessage": "The original game process has exited. No resume command was needed.", + "description": "Unquiesced condition when the original source pod no longer runs." + }, "condition.unquiesced.unquiesced": { "defaultMessage": "The server was resumed after the backup.", "description": "Unquiesced condition when the world was unfrozen." diff --git a/operator/hack/condition-ids/main.go b/operator/hack/condition-ids/main.go index e644cf041..cc764ac06 100644 --- a/operator/hack/condition-ids/main.go +++ b/operator/hack/condition-ids/main.go @@ -140,6 +140,7 @@ var rows = []row{ {types: []string{"Completed"}, reason: "Pending", english: "The backup is waiting to start.", description: "Completed condition on a Backup that has not started (dynamic: reason is the phase).", dynamic: true}, {types: []string{"Completed"}, reason: "Running", english: "The backup is running.", description: "Completed condition on a Backup in progress (dynamic: reason is the phase).", dynamic: true}, {types: []string{"Unquiesced"}, reason: "Unquiesced", english: "The server was resumed after the backup.", description: "Unquiesced condition when the world was unfrozen."}, + {types: []string{"Unquiesced"}, reason: "SourceProcessGone", english: "The original game process has exited. No resume command was needed.", description: "Unquiesced condition when the original source pod no longer runs."}, {types: []string{"Unquiesced"}, reason: "Failed", english: "The server could not be resumed after the backup. Its world may still have auto-save turned off.", description: "Unquiesced condition when the unfreeze failed."}, {types: []string{"SnapshotForgotten"}, reason: "Forgotten", english: "The snapshot was removed from the backup repository.", description: "SnapshotForgotten condition after a deleted Backup's snapshot was forgotten."}, {types: []string{"SnapshotForgotten"}, reason: "ForgetAbandoned", english: "The snapshot was not removed from the backup repository when the backup was deleted.", description: "SnapshotForgotten condition when the forget step was abandoned."}, diff --git a/operator/internal/agent/client.go b/operator/internal/agent/client.go index d606c3c75..d5a573c7c 100644 --- a/operator/internal/agent/client.go +++ b/operator/internal/agent/client.go @@ -14,6 +14,7 @@ import ( "fmt" "io" "net/http" + "net/url" "os" "time" ) @@ -119,6 +120,44 @@ func (c *Client) Unquiesce(ctx context.Context, namespace, server string) error return c.callQuiesce(ctx, namespace, server, "/unquiesce", false) } +// Pod-bound routes reject a replacement process at the final network hop. +// Older agents return 404 without executing a mutation; never fall back. +func (c *Client) QuiesceEnabled() bool { return c != nil && !c.Disabled } + +func (c *Client) CheckPod(ctx context.Context, namespace, server, podUID string) error { + if c.Disabled || podUID == "" { + return fmt.Errorf("agent: pod identity check requires an enabled client and pod UID") + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, agentURL(namespace, server, "/v1/pods/"+url.PathEscape(podUID)+"/identity"), nil) + if err != nil { + return err + } + resp, err := c.http.Do(req) + if err != nil { + return fmt.Errorf("agent: pod identity check: %w", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusNoContent { + return fmt.Errorf("agent: pod identity check: status %d", resp.StatusCode) + } + return nil +} + +func (c *Client) QuiescePod(ctx context.Context, namespace, server, podUID string) error { + return c.callPodQuiesce(ctx, namespace, server, podUID, "/quiesce", true) +} + +func (c *Client) UnquiescePod(ctx context.Context, namespace, server, podUID string) error { + return c.callPodQuiesce(ctx, namespace, server, podUID, "/unquiesce", false) +} + +func (c *Client) callPodQuiesce(ctx context.Context, namespace, server, podUID, path string, expectQuiesced bool) error { + if c.Disabled || podUID == "" { + return fmt.Errorf("agent: pod-bound quiesce requires an enabled client and pod UID") + } + return c.callQuiesce(ctx, namespace, server, "/v1/pods/"+url.PathEscape(podUID)+path, expectQuiesced) +} + // expectQuiesced is what the response should report on a successful // completion: true after /quiesce, false after /unquiesce. A mismatch // is a protocol bug worth surfacing. diff --git a/operator/internal/agent/client_test.go b/operator/internal/agent/client_test.go index c3866488e..7d04dc9d1 100644 --- a/operator/internal/agent/client_test.go +++ b/operator/internal/agent/client_test.go @@ -122,6 +122,46 @@ func TestAgentURL(t *testing.T) { } } +func TestPodBoundQuiesceNeverFallsBackToLegacyRoute(t *testing.T) { + for _, status := range []int{http.StatusOK, http.StatusNotFound} { + t.Run(http.StatusText(status), func(t *testing.T) { + paths := []string{} + c, closeServer := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + paths = append(paths, r.URL.Path) + w.WriteHeader(status) + _, _ = w.Write([]byte(`{"quiesced":true}`)) + }) + defer closeServer() + err := c.QuiescePod(t.Context(), "ns", "server", "original-pod") + if (err == nil) != (status == http.StatusOK) { + t.Fatalf("status=%d error=%v", status, err) + } + if len(paths) != 1 || paths[0] != "/v1/pods/original-pod/quiesce" { + t.Fatalf("unbound mutation/fallback: %v", paths) + } + }) + } +} + +func TestPodBoundUnquiesceAndDisabledClient(t *testing.T) { + c, closeServer := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/pods/original-pod/unquiesce" { + t.Errorf("wrong cleanup target: %s", r.URL.Path) + } + _, _ = w.Write([]byte(`{"quiesced":false}`)) + }) + defer closeServer() + if err := c.UnquiescePod(t.Context(), "ns", "server", "original-pod"); err != nil { + t.Fatal(err) + } + if err := c.QuiescePod(t.Context(), "ns", "server", ""); err == nil { + t.Fatal("empty pod UID admitted") + } + if err := (&Client{Disabled: true}).QuiescePod(t.Context(), "ns", "server", "pod"); err == nil { + t.Fatal("disabled client reported identity-bound quiesce") + } +} + func TestQuiesce_DisabledNoOp(t *testing.T) { c := &Client{Disabled: true} if err := c.Quiesce(context.Background(), "ns", "srv"); err != nil { @@ -257,6 +297,32 @@ func TestUnquiesce_HappyPath(t *testing.T) { } } +func TestCheckPodRequiresIdentityEndpoint(t *testing.T) { + for _, status := range []int{http.StatusNoContent, http.StatusNotFound, http.StatusOK, http.StatusServiceUnavailable} { + t.Run(http.StatusText(status), func(t *testing.T) { + calls := 0 + c, cleanup := newMockClient(t, func(w http.ResponseWriter, req *http.Request) { + calls++ + if req.Method != http.MethodGet || req.URL.Path != "/v1/pods/original/identity" { + t.Errorf("unexpected preflight request: %s %s", req.Method, req.URL.Path) + } + w.WriteHeader(status) + }) + defer cleanup() + err := c.CheckPod(t.Context(), "ns", "server", "original") + if (err == nil) != (status == http.StatusNoContent) || calls != 1 { + t.Fatalf("identity result = %v, calls = %d", err, calls) + } + }) + } + if err := (&Client{Disabled: true}).CheckPod(t.Context(), "ns", "server", "original"); err == nil { + t.Fatal("disabled client confirmed identity") + } + if err := (&Client{}).CheckPod(t.Context(), "ns", "server", ""); err == nil { + t.Fatal("empty pod UID confirmed identity") + } +} + func TestQuiesce_TransportError(t *testing.T) { // Use a base transport that always errors. c := &Client{ diff --git a/operator/internal/controller/backup_controller.go b/operator/internal/controller/backup_controller.go index c579713d9..d707d12f5 100644 --- a/operator/internal/controller/backup_controller.go +++ b/operator/internal/controller/backup_controller.go @@ -383,15 +383,47 @@ func (r *BackupReconciler) reconcile(ctx context.Context, req ctrl.Request) (ctr // requeues; ErrUnsupported is recorded as "attempted but unsupported" // so the matching unquiesce step is skipped on terminal phase. func (r *BackupReconciler) maybeQuiesce(ctx context.Context, b *gameplanev1alpha1.Backup) error { + preparation, err := parseBackupPreparation(b) + if err != nil { + return err + } + if preparation != nil && preparation.Outcome == "already-stopped" { + return nil + } if !b.Spec.Quiesce { return nil } - if r.AgentClient == nil { + if !agentQuiesceAvailable(r.AgentClient) { + if preparation != nil && preparation.Outcome == "live" && b.Annotations[annoQuiesceAttempted] != "true" && b.Annotations[annoQuiesceAttempted] != "unsupported" { + err := fmt.Errorf("agent quiesce client was disabled after live preparation") + if _, ferr := r.fail(ctx, b, err.Error()); ferr != nil { + return ferr + } + return err + } return nil } if state := b.Annotations[annoQuiesceAttempted]; state != "" && state != "pending" { return nil } + // Verify the versioned endpoint before creating the first cleanup debt. + // Old agents cannot mutate through it and must not pin an owner that + // would prevent their own upgrade. The POST still checks the UID again. + if preparation != nil && b.Annotations[annoQuiesceAttempted] == "" { + bound, ok := r.AgentClient.(podAgentQuiescer) + if !ok { + err = fmt.Errorf("agent client does not support pod-bound quiesce") + } else { + err = bound.CheckPod(ctx, b.Namespace, b.Spec.ServerRef.Name, string(preparation.PodUID)) + } + if err != nil { + _, ferr := r.fail(ctx, b, fmt.Sprintf("quiesce identity check failed: %v", err)) + if ferr != nil { + return ferr + } + return err + } + } // Persist cleanup intent BEFORE save-off: a crash or a partial agent failure // must still owe save-on even when quiesced-at was never recorded. if err := r.setLifecycleUnquiesceIntent(ctx, b, true); err != nil { @@ -402,7 +434,11 @@ func (r *BackupReconciler) maybeQuiesce(ctx context.Context, b *gameplanev1alpha return err } - err := r.AgentClient.Quiesce(ctx, b.Namespace, b.Spec.ServerRef.Name) + var podUID types.UID + if preparation != nil { + podUID = preparation.PodUID + } + err = callBackupQuiesce(ctx, r.AgentClient, b.Namespace, b.Spec.ServerRef.Name, podUID, false) switch { case err == nil: now := time.Now().UTC().Format(time.RFC3339) @@ -626,8 +662,8 @@ func (r *BackupReconciler) runUnquiesce(ctx context.Context, b *gameplanev1alpha } return ctrl.Result{RequeueAfter: 30 * time.Second}, nil } - // When an unquiesce was actually performed, record success so the - // "world might be frozen" signal clears. + // Confirmed save-on or confirmed original-process exit clears the + // "world might be frozen" signal, with distinct condition reasons. if _, ok := b.Annotations[annoUnquiescedAt]; ok { if cerr := r.setUnquiescedCondition(ctx, b, true, ""); cerr != nil { return ctrl.Result{}, cerr @@ -646,6 +682,10 @@ func (r *BackupReconciler) setUnquiescedCondition( if ok { cond.Status = metav1.ConditionTrue cond.Reason = "Unquiesced" + if b.Annotations[backupCleanupOutcomeAnnotation] == "source-gone" { + cond.Reason = "SourceProcessGone" + cond.Message = "original process no longer exists; no unquiesce RPC was sent" + } } else { cond.Status = metav1.ConditionFalse cond.Reason = "Failed" @@ -737,49 +777,29 @@ func (r *BackupReconciler) finalizeUnquiesce(ctx context.Context, b *gameplanev1 return ctrl.Result{}, nil } -// unquiesceTargetGone reports whether there is nothing left to unquiesce: -// the target GameServer no longer exists or is itself being deleted, or its -// pod no longer exists. A GameServer can be gone-together-with-its-Backup -// under an auto BackupSchedule's owner chain, and a pod alone can be gone if -// it was deleted independently of the GameServer; either way a fresh pod -// starts with auto-save on, so retrying the unquiesce would never succeed -// and would only block this Backup's deletion. +// unquiesceTargetGone requires evidence that the original process has exited. +// A deleted/deleting GameServer alone is insufficient: its pod can still run. func (r *BackupReconciler) unquiesceTargetGone(ctx context.Context, b *gameplanev1alpha1.Backup) (bool, error) { targetName := b.Spec.ServerRef.Name - var targetUID types.UID if value := b.Annotations[backupTargetAnnotation]; value != "" { identity, err := parseBackupIdentity(value) if err != nil { return false, err } - targetName, targetUID = identity.Name, identity.UID + targetName = identity.Name } if targetName == "" { return true, nil } - gs := &gameplanev1alpha1.GameServer{} - key := types.NamespacedName{Namespace: b.Namespace, Name: targetName} - switch err := r.Get(ctx, key, gs); { - case apierrors.IsNotFound(err): - return true, nil - case err != nil: - return false, fmt.Errorf("get gameserver %s to check unquiesce target: %w", key.Name, err) - case gs.DeletionTimestamp != nil: - return true, nil - } - if targetUID != "" && gs.UID != targetUID { - return true, nil + preparation, err := parseBackupPreparation(b) + if err != nil { + return false, err } - - var pod corev1.Pod - podKey := types.NamespacedName{Namespace: b.Namespace, Name: gs.Name + "-0"} - switch err := r.Get(ctx, podKey, &pod); { - case apierrors.IsNotFound(err): - return true, nil - case err != nil: - return false, fmt.Errorf("get pod %s to check unquiesce target: %w", podKey.Name, err) + var podUID types.UID + if preparation != nil { + podUID = preparation.PodUID } - return false, nil + return backupQuiesceProcessGone(ctx, r.backupAPIReader(), b.Namespace, targetName, podUID) } // unquiesceRetryExpired records (on first call) when finalizeDelete's @@ -867,10 +887,35 @@ func (r *BackupReconciler) maybeUnquiesce(ctx context.Context, b *gameplanev1alp return fmt.Errorf("legacy quiesce cleanup is blocked by a protected backup") } } - if r.AgentClient == nil { + if targetName != "" { + gone, err := r.unquiesceTargetGone(ctx, b) + if err != nil { + return err + } + if gone { + patchBackupAnnotations(b, map[string]string{backupCleanupOutcomeAnnotation: "source-gone"}) + return r.recordBackupUnquiesced(ctx, b) + } + } + if !agentQuiesceAvailable(r.AgentClient) { return fmt.Errorf("cannot release quiesced world without an agent client") } - if err := r.AgentClient.Unquiesce(ctx, b.Namespace, targetName); err != nil { + preparation, err := parseBackupPreparation(b) + if err != nil { + return err + } + var podUID types.UID + if preparation != nil { + podUID = preparation.PodUID + } + if err := callBackupQuiesce(ctx, r.AgentClient, b.Namespace, targetName, podUID, true); err != nil { + return err + } + return r.recordBackupUnquiesced(ctx, b) +} + +func (r *BackupReconciler) recordBackupUnquiesced(ctx context.Context, b *gameplanev1alpha1.Backup) error { + if err := r.setLifecycleUnquiesceIntent(ctx, b, false); err != nil { return err } patchBackupAnnotations(b, map[string]string{ diff --git a/operator/internal/controller/backup_envtest_test.go b/operator/internal/controller/backup_envtest_test.go index 87ce6cc7e..b4d6db2e2 100644 --- a/operator/internal/controller/backup_envtest_test.go +++ b/operator/internal/controller/backup_envtest_test.go @@ -8,9 +8,11 @@ import ( "testing" "time" + appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" ) @@ -379,6 +381,20 @@ func TestBackup_QuiesceAnnotationSetBeforeJob(t *testing.T) { fa := &fakeAgent{} startMgr(t, ns, withBackupReconciler(backupReconcilerOpts{agent: fa})) seedGameServer(t, ns, "smp") + // envtest has no StatefulSet controller or kubelet: provide the observed + // running source whose immutable identity is bound before the agent RPC. + var workload appsv1.StatefulSet + if err := k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: "smp"}, &workload); err != nil { + t.Fatal(err) + } + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "smp-0", Namespace: ns, OwnerReferences: []metav1.OwnerReference{{APIVersion: "apps/v1", Kind: "StatefulSet", Name: workload.Name, UID: workload.UID, Controller: ownerBoolPtr(true)}}}, Spec: workload.Spec.Template.Spec} + if err := k8sClient.Create(context.Background(), pod); err != nil { + t.Fatal(err) + } + pod.Status.Phase = corev1.PodRunning + if err := k8sClient.Status().Update(context.Background(), pod); err != nil { + t.Fatal(err) + } if err := k8sClient.Create(context.Background(), buildResticRepoSecret(ns, "repo")); err != nil { t.Fatalf("create secret: %v", err) diff --git a/operator/internal/controller/backup_guard.go b/operator/internal/controller/backup_guard.go index cee37734e..2b6e4d4f2 100644 --- a/operator/internal/controller/backup_guard.go +++ b/operator/internal/controller/backup_guard.go @@ -119,11 +119,19 @@ func (r *BackupReconciler) recoverBackupTarget(ctx context.Context, b *gameplane return false, err } } - if r.AgentClient == nil { - return false, fmt.Errorf("cannot recover orphaned quiesce without an agent client") + // Legacy owners have no pod UID. Retire them only on positive absence or + // terminal-pod proof; never bind an old RPC to a newly observed identity. + gone, err := backupQuiesceProcessGone(ctx, r.backupAPIReader(), b.Namespace, b.Spec.ServerRef.Name, "") + if err != nil { + return false, err } - if err := r.AgentClient.Unquiesce(ctx, b.Namespace, b.Spec.ServerRef.Name); err != nil { - return false, fmt.Errorf("recover orphaned quiesce: %w", err) + if !gone { + if !agentQuiesceAvailable(r.AgentClient) { + return false, fmt.Errorf("cannot recover orphaned quiesce without an agent client") + } + if err := r.AgentClient.Unquiesce(ctx, b.Namespace, b.Spec.ServerRef.Name); err != nil { + return false, fmt.Errorf("recover orphaned quiesce: %w", err) + } } delete(b.Annotations, backupRecoveryAnnotation) delete(b.Annotations, annoQuiesceAttempted) @@ -267,7 +275,28 @@ func (r *BackupReconciler) prepareBackupQuiesce(ctx context.Context, b *gameplan if err != nil || !fenced { return ctrl.Result{RequeueAfter: 5 * time.Second}, err } - return ctrl.Result{}, r.maybeQuiesce(ctx, b) + ready, err := r.prepareBackupSource(ctx, b, gs) + if errors.Is(err, errBackupSourceChanged) { + _, err = r.fail(ctx, b, err.Error()) + return ctrl.Result{RequeueAfter: time.Nanosecond}, err + } + if err != nil || !ready { + return ctrl.Result{RequeueAfter: 5 * time.Second}, err + } + if err := r.maybeQuiesce(ctx, b); err != nil { + return ctrl.Result{}, err + } + // The Service RPC may overlap a pod replacement. Never start a copy on a + // different process from the one pinned before the request. + ready, err = r.prepareBackupSource(ctx, b, gs) + if errors.Is(err, errBackupSourceChanged) { + _, err = r.fail(ctx, b, err.Error()) + return ctrl.Result{RequeueAfter: time.Nanosecond}, err + } + if err != nil || !ready { + return ctrl.Result{RequeueAfter: 5 * time.Second}, err + } + return ctrl.Result{}, nil } // Return only the originally bound GameServer. A foreign owner means this diff --git a/operator/internal/controller/backup_guard_test.go b/operator/internal/controller/backup_guard_test.go index 8b40e1104..8009ae300 100644 --- a/operator/internal/controller/backup_guard_test.go +++ b/operator/internal/controller/backup_guard_test.go @@ -28,13 +28,30 @@ import ( type guardQuiescer struct { quiesced, unquiesced int quiesceErr error + unquiesceErr error + checkErr error + afterQuiesce func() } func (q *guardQuiescer) Quiesce(context.Context, string, string) error { q.quiesced++ return q.quiesceErr } -func (q *guardQuiescer) Unquiesce(context.Context, string, string) error { q.unquiesced++; return nil } +func (q *guardQuiescer) Unquiesce(context.Context, string, string) error { + q.unquiesced++ + return q.unquiesceErr +} +func (q *guardQuiescer) CheckPod(context.Context, string, string, string) error { return q.checkErr } +func (q *guardQuiescer) QuiescePod(ctx context.Context, namespace, server, podUID string) error { + err := q.Quiesce(ctx, namespace, server) + if q.afterQuiesce != nil { + q.afterQuiesce() + } + return err +} +func (q *guardQuiescer) UnquiescePod(ctx context.Context, namespace, server, podUID string) error { + return q.Unquiesce(ctx, namespace, server) +} func guardBackup(name string) *gameplanev1alpha1.Backup { return &gameplanev1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "ns", UID: types.UID(name + "-uid"), Finalizers: []string{gameplanev1alpha1.BackupFinalizer, gameplanev1alpha1.BackupSnapshotFinalizer}}, Spec: gameplanev1alpha1.BackupSpec{ServerRef: gameplanev1alpha1.LocalObjectRef{Name: "server"}, RepoRef: &gameplanev1alpha1.SecretKeySelector{Name: "repo"}, Quiesce: true}} @@ -52,6 +69,7 @@ func guardFixture(t *testing.T, backups ...*gameplanev1alpha1.Backup) (*BackupRe objects := []client.Object{&gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "server", Namespace: "ns", UID: "server-uid"}, Spec: gameplanev1alpha1.GameServerSpec{TemplateRef: gameplanev1alpha1.GameTemplateRef{Name: "template"}}}, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "repo", Namespace: "ns"}, Data: map[string][]byte{"repo": []byte("repo"), "password": []byte("password")}}} one := int32(1) objects = append(objects, &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "server", Namespace: "ns", UID: "workload-uid", OwnerReferences: []metav1.OwnerReference{{APIVersion: gameplanev1alpha1.GroupVersion.String(), Kind: "GameServer", Name: "server", UID: "server-uid", Controller: ownerBoolPtr(true)}}}, Spec: appsv1.StatefulSetSpec{Replicas: &one}}) + objects = append(objects, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "server-0", Namespace: "ns", UID: "game-pod-uid", OwnerReferences: []metav1.OwnerReference{{APIVersion: "apps/v1", Kind: "StatefulSet", Name: "server", UID: "workload-uid", Controller: ownerBoolPtr(true)}}}, Status: corev1.PodStatus{Phase: corev1.PodRunning}}) for _, b := range backups { objects = append(objects, b) } diff --git a/operator/internal/controller/backup_preparation.go b/operator/internal/controller/backup_preparation.go new file mode 100644 index 000000000..4f682ab53 --- /dev/null +++ b/operator/internal/controller/backup_preparation.go @@ -0,0 +1,234 @@ +package controller + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +const ( + backupPreparationAnnotation = "backup.gameplane.local/preparation" + backupCleanupOutcomeAnnotation = "backup.gameplane.local/cleanup-outcome" +) + +var errBackupSourceChanged = errors.New("backup source changed after preparation") + +type podAgentQuiescer interface { + CheckPod(context.Context, string, string, string) error + QuiescePod(context.Context, string, string, string) error + UnquiescePod(context.Context, string, string, string) error +} + +func agentQuiesceAvailable(q AgentQuiescer) bool { + if q == nil { + return false + } + if configured, ok := q.(interface{ QuiesceEnabled() bool }); ok { + return configured.QuiesceEnabled() + } + return true +} + +func callBackupQuiesce(ctx context.Context, q AgentQuiescer, namespace, server string, podUID types.UID, undo bool) error { + if !agentQuiesceAvailable(q) { + return fmt.Errorf("agent quiesce client is not configured") + } + if podUID != "" { + bound, ok := q.(podAgentQuiescer) + if !ok { + return fmt.Errorf("agent client does not support pod-bound quiesce") + } + if undo { + return bound.UnquiescePod(ctx, namespace, server, string(podUID)) + } + return bound.QuiescePod(ctx, namespace, server, string(podUID)) + } + if undo { + return q.Unquiesce(ctx, namespace, server) + } + return q.Quiesce(ctx, namespace, server) +} + +// Source identity is immutable for one copy. RPC outcome remains in the +// quiesce annotations, so a stopped source is never described as quiesced. +type backupPreparation struct { + Outcome string `json:"outcome"` + TargetUID types.UID `json:"targetUID"` + WorkloadUID types.UID `json:"workloadUID"` + WorkloadGeneration int64 `json:"workloadGeneration"` + PodUID types.UID `json:"podUID,omitempty"` +} + +func parseBackupPreparation(b *gameplanev1alpha1.Backup) (*backupPreparation, error) { + value := b.Annotations[backupPreparationAnnotation] + if value == "" { + return nil, nil + } + var p backupPreparation + if err := json.Unmarshal([]byte(value), &p); err != nil { + return nil, fmt.Errorf("decode backup preparation: %w", err) + } + if p.TargetUID == "" || p.WorkloadUID == "" || (p.Outcome != "live" && p.Outcome != "already-stopped") || (p.Outcome == "live" && p.PodUID == "") || (p.Outcome == "already-stopped" && p.PodUID != "") { + return nil, fmt.Errorf("invalid backup preparation identity") + } + return &p, nil +} + +// Call only after the shared owner and StatefulSet fence are held. Readiness +// and desired suspend/idle state are not evidence that writers have exited. +func (r *BackupReconciler) prepareBackupSource(ctx context.Context, b *gameplanev1alpha1.Backup, gs *gameplanev1alpha1.GameServer) (bool, error) { + if !b.Spec.Quiesce { + return true, nil + } + p, err := parseBackupPreparation(b) + if err != nil { + return false, err + } + if p == nil && !agentQuiesceAvailable(r.AgentClient) { + return true, nil + } + // Pre-upgrade attempts have no source identity. Do not invent a new one + // after their RPC; preserve their existing cleanup contract. + if p == nil && b.Annotations[annoQuiesceAttempted] != "" { + return true, nil + } + var ss appsv1.StatefulSet + if err := r.backupAPIReader().Get(ctx, client.ObjectKeyFromObject(gs), &ss); err != nil { + if apierrors.IsNotFound(err) && p != nil { + return false, fmt.Errorf("%w: workload absent", errBackupSourceChanged) + } + return false, client.IgnoreNotFound(err) + } + if p != nil && (p.TargetUID != gs.UID || p.WorkloadUID != ss.UID || p.WorkloadGeneration != ss.Generation) { + return false, fmt.Errorf("%w: workload identity", errBackupSourceChanged) + } + if !metav1.IsControlledBy(&ss, gs) { + return false, fmt.Errorf("backup source workload is not owned by the bound GameServer") + } + if ss.Spec.Replicas == nil || ss.Status.ObservedGeneration < ss.Generation { + return false, nil + } + if p != nil && ((p.Outcome == "already-stopped") != (*ss.Spec.Replicas == 0)) { + return false, fmt.Errorf("%w: desired replicas", errBackupSourceChanged) + } + var candidate backupPreparation + candidate.TargetUID, candidate.WorkloadUID, candidate.WorkloadGeneration = gs.UID, ss.UID, ss.Generation + if *ss.Spec.Replicas == 0 { + if ss.Status.Replicas != 0 { + if p != nil { + return false, fmt.Errorf("%w: stopped workload has replicas", errBackupSourceChanged) + } + return false, nil + } + drained, err := r.backupSourceDrained(ctx, b, gs, &ss) + if err != nil || !drained { + if err == nil && p != nil { + return false, fmt.Errorf("%w: stopped source acquired a writer", errBackupSourceChanged) + } + return false, err + } + candidate.Outcome = "already-stopped" + } else { + var pod corev1.Pod + err := r.backupAPIReader().Get(ctx, client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-0"}, &pod) + if apierrors.IsNotFound(err) { + if p != nil { + return false, fmt.Errorf("%w: original pod absent", errBackupSourceChanged) + } + return false, nil + } + if err != nil { + return false, err + } + if p != nil && (p.PodUID != pod.UID || !pod.DeletionTimestamp.IsZero() || pod.Status.Phase != corev1.PodRunning) { + return false, fmt.Errorf("%w: original process no longer running", errBackupSourceChanged) + } + if !metav1.IsControlledBy(&pod, &ss) || pod.UID == "" { + return false, fmt.Errorf("backup source pod is not owned by the bound workload") + } + if !pod.DeletionTimestamp.IsZero() || pod.Status.Phase != corev1.PodRunning { + return false, nil + } + candidate.Outcome, candidate.PodUID = "live", pod.UID + } + if p != nil { + if p.Outcome != candidate.Outcome || p.PodUID != candidate.PodUID { + return false, fmt.Errorf("%w: process identity", errBackupSourceChanged) + } + return true, nil + } + encoded, err := json.Marshal(candidate) + if err != nil { + return false, err + } + patchBackupAnnotations(b, map[string]string{backupPreparationAnnotation: string(encoded)}) + return true, r.Update(ctx, b) +} + +// Current-copy workers are readers and are already protected by the owner; +// ignore only those, so requeues can observe their completion. Any other PVC +// user or unscheduled writer Job prevents stopped-source admission. +func (r *BackupReconciler) backupSourceDrained(ctx context.Context, b *gameplanev1alpha1.Backup, gs *gameplanev1alpha1.GameServer, ss *appsv1.StatefulSet) (bool, error) { + var jobs batchv1.JobList + if err := r.backupAPIReader().List(ctx, &jobs, client.InNamespace(gs.Namespace)); err != nil { + return false, err + } + copyJobs := map[types.UID]bool{} + for i := range jobs.Items { + job := &jobs.Items[i] + if job.Name == b.Name && metav1.IsControlledBy(job, b) { + copyJobs[job.UID] = true + continue + } + if ((job.Status.Succeeded == 0 && !jobPermanentlyFailed(job)) || job.Status.Active > 0 || !job.DeletionTimestamp.IsZero()) && podMountsRestoreTarget(&corev1.Pod{Spec: job.Spec.Template.Spec}, gs.Name) { + return false, nil + } + } + var pods corev1.PodList + if err := r.backupAPIReader().List(ctx, &pods, client.InNamespace(gs.Namespace)); err != nil { + return false, err + } + for i := range pods.Items { + pod := &pods.Items[i] + if !liveRestorePod(pod) { + continue + } + owner := metav1.GetControllerOf(pod) + if owner != nil && owner.Kind == "Job" && copyJobs[owner.UID] { + continue + } + if metav1.IsControlledBy(pod, ss) || metav1.IsControlledBy(pod, gs) || pod.Name == gs.Name+"-0" || podMountsRestoreTarget(pod, gs.Name) { + return false, nil + } + } + return true, nil +} + +// An absent/replaced original pod cannot retain save-off state. A timeout, +// unready pod, or terminating pod with the same UID is never proof of exit. +// Empty UID is legacy state: only absence or an observed terminal pod suffices. +func backupQuiesceProcessGone(ctx context.Context, reader client.Reader, namespace, server string, uid types.UID) (bool, error) { + var pod corev1.Pod + err := reader.Get(ctx, client.ObjectKey{Namespace: namespace, Name: server + "-0"}, &pod) + if apierrors.IsNotFound(err) { + return true, nil + } + if err != nil { + return false, err + } + if uid != "" && pod.UID != uid { + return true, nil + } + return pod.DeletionTimestamp.IsZero() && (pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed), nil +} diff --git a/operator/internal/controller/backup_stopped_test.go b/operator/internal/controller/backup_stopped_test.go new file mode 100644 index 000000000..3d0f30f39 --- /dev/null +++ b/operator/internal/controller/backup_stopped_test.go @@ -0,0 +1,382 @@ +package controller + +import ( + "context" + "encoding/json" + "errors" + "testing" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" + "github.com/GameplanePanel/gameplane/operator/internal/agent" +) + +func stoppedBackupFixture(t *testing.T, suspended bool) (*BackupReconciler, *guardQuiescer) { + t.Helper() + r, q := guardFixture(t, guardBackup("first"), guardBackup("second")) + ctx := t.Context() + var gs gameplanev1alpha1.GameServer + key := client.ObjectKey{Namespace: "ns", Name: "server"} + if err := r.Get(ctx, key, &gs); err != nil { + t.Fatal(err) + } + gs.Spec.Suspend = suspended + if !suspended { + gs.Annotations = map[string]string{IdleAsleepSinceAnnotation: "2026-10-21T00:00:00Z"} + } + if err := r.Update(ctx, &gs); err != nil { + t.Fatal(err) + } + var ss appsv1.StatefulSet + if err := r.Get(ctx, key, &ss); err != nil { + t.Fatal(err) + } + zero := int32(0) + ss.Spec.Replicas = &zero + if err := r.Update(ctx, &ss); err != nil { + t.Fatal(err) + } + if err := r.Delete(ctx, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "server-0", Namespace: "ns"}}); err != nil { + t.Fatal(err) + } + return r, q +} + +// A regression to unconditional agent quiesce would fail the first admission; +// releasing before worker drain would let the second Job appear prematurely. +func TestBackupStoppedCopiesWithoutWakingAndReleasesForNextCopy(t *testing.T) { + for _, suspended := range []bool{false, true} { + t.Run(map[bool]string{false: "idle", true: "suspended"}[suspended], func(t *testing.T) { + r, q := stoppedBackupFixture(t, suspended) + q.quiesceErr = errors.New("absent agent") + guardPass(t, r, "first") + first := guardGetBackup(t, r, "first") + var prep backupPreparation + if err := json.Unmarshal([]byte(first.Annotations[backupPreparationAnnotation]), &prep); err != nil { + t.Fatal(err) + } + if prep.Outcome != "already-stopped" || prep.TargetUID != "server-uid" || prep.WorkloadUID != "workload-uid" || prep.PodUID != "" { + t.Fatalf("wrong source proof: %+v", prep) + } + var job batchv1.Job + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "first"}, &job); err != nil { + t.Fatal(err) + } + // A restart and concurrent wake request must preserve the zero-replica fence. + r = &BackupReconciler{Client: r.Client, APIReader: r.Client, Scheme: r.Scheme, AgentClient: q, LogReader: r.LogReader} + var gs gameplanev1alpha1.GameServer + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server"}, &gs); err != nil { + t.Fatal(err) + } + if gs.Annotations == nil { + gs.Annotations = map[string]string{} + } + gs.Annotations[IdleWakeRequestedAnnotation] = "wake" + if err := r.Update(t.Context(), &gs); err != nil { + t.Fatal(err) + } + gr := &GameServerReconciler{Client: r.Client, APIReader: r.Client, Scheme: r.Scheme} + if err := gr.reconcileStatefulSet(t.Context(), &gs, singlePortTemplate(true, "minecraft"), nil, nil, 1); err != nil { + t.Fatal(err) + } + var ss appsv1.StatefulSet + if err := r.Get(t.Context(), client.ObjectKeyFromObject(&gs), &ss); err != nil { + t.Fatal(err) + } + if ss.Spec.Replicas == nil || *ss.Spec.Replicas != 0 { + t.Fatal("wake crossed the backup fence") + } + guardPass(t, r, "second") + var second batchv1.Job + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "second"}, &second); !apierrors.IsNotFound(err) { + t.Fatalf("second copy admitted before drain: %v", err) + } + job.Status.Succeeded = 1 + if err := r.Status().Update(t.Context(), &job); err != nil { + t.Fatal(err) + } + guardPass(t, r, "first") + if got := guardGetBackup(t, r, "first"); got.Status.Phase != gameplanev1alpha1.BackupPhaseSucceeded || got.Status.SnapshotID == "" { + t.Fatalf("not restorable: %+v", got.Status) + } + guardPass(t, r, "second") + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "second"}, &second); err != nil { + t.Fatal(err) + } + if q.quiesced != 0 || q.unquiesced != 0 { + t.Fatalf("stopped copy contacted absent agent: %+v", q) + } + if err := r.Get(t.Context(), client.ObjectKeyFromObject(&gs), &gs); err != nil { + t.Fatal(err) + } + if gs.Spec.Suspend != suspended || gs.Annotations[IdleWakeRequestedAnnotation] != "wake" || (!suspended && gs.Annotations[IdleAsleepSinceAnnotation] != "2026-10-21T00:00:00Z") { + t.Fatal("copy changed power intent") + } + }) + } +} + +func TestBackupStoppedRequiresLiveDrainProof(t *testing.T) { + for _, tc := range []string{"desired-one", "observed-replicas", "stale-generation", "terminating-game", "foreign-writer", "pending-writer-job", "read-error"} { + t.Run(tc, func(t *testing.T) { + r, q := stoppedBackupFixture(t, true) + base := r.Client + r.APIReader = interceptor.NewClient(base.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if err := c.Get(ctx, key, obj, opts...); err != nil { + return err + } + if ss, ok := obj.(*appsv1.StatefulSet); ok { + switch tc { + case "desired-one": + one := int32(1) + ss.Spec.Replicas = &one + case "observed-replicas": + ss.Status.Replicas = 1 + case "stale-generation": + ss.Generation = 2 + ss.Status.ObservedGeneration = 1 + } + } + return nil + }, + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if err := c.List(ctx, list, opts...); err != nil { + return err + } + if pods, ok := list.(*corev1.PodList); ok { + if tc == "read-error" { + return errors.New("live list failed") + } + if tc == "terminating-game" { + now := metav1.Now() + pods.Items = append(pods.Items, corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "server-0", Namespace: "ns", DeletionTimestamp: &now}, Status: corev1.PodStatus{Phase: corev1.PodSucceeded}}) + } + if tc == "foreign-writer" { + pods.Items = append(pods.Items, corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "writer", Namespace: "ns"}, Spec: corev1.PodSpec{Volumes: []corev1.Volume{{Name: "data", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "server-data"}}}}}}) + } + } + if jobs, ok := list.(*batchv1.JobList); ok && tc == "pending-writer-job" { + jobs.Items = append(jobs.Items, batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "writer", Namespace: "ns"}, Spec: batchv1.JobSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Volumes: []corev1.Volume{{Name: "data", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "server-data"}}}}}}}}) + } + return nil + }, + }) + _, _ = r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKey{Namespace: "ns", Name: "first"}}) + var job batchv1.Job + if err := base.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "first"}, &job); !apierrors.IsNotFound(err) { + t.Fatalf("copy admitted without drain proof: %v", err) + } + if q.quiesced != 0 { + t.Fatal("no proven live process to quiesce") + } + }) + } +} + +func TestBackupLiveTimeoutRetainsOwnerUntilOriginalPodGone(t *testing.T) { + r, q := guardFixture(t, guardBackup("first"), guardBackup("second")) + q.quiesceErr = errors.New("live agent unreachable") + q.unquiesceErr = errors.New("live agent still unreachable") + _, _ = r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKey{Namespace: "ns", Name: "first"}}) + first := guardGetBackup(t, r, "first") + if first.Status.Phase != gameplanev1alpha1.BackupPhaseFailed { + t.Fatalf("timeout did not fail closed: %+v", first.Status) + } + var job batchv1.Job + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "first"}, &job); !apierrors.IsNotFound(err) { + t.Fatal("live timeout created copy") + } + var gs gameplanev1alpha1.GameServer + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server"}, &gs); err != nil { + t.Fatal(err) + } + op, _, err := lifecycleOwner(&gs) + if err != nil || op.QuiescePodUID != "game-pod-uid" || !op.NeedsUnquiesce { + t.Fatalf("cleanup identity not durable: %+v %v", op, err) + } + guardPass(t, r, "first") + guardPass(t, r, "second") + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "second"}, &job); !apierrors.IsNotFound(err) { + t.Fatal("timeout released the live process owner") + } + cleanupAttempts := q.unquiesced + // The replacement must not receive its predecessor's delayed save-on. + if err := r.Delete(t.Context(), &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "server-0", Namespace: "ns"}}); err != nil { + t.Fatal(err) + } + if err := r.Create(t.Context(), &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "server-0", Namespace: "ns", UID: "replacement"}, Status: corev1.PodStatus{Phase: corev1.PodRunning}}); err != nil { + t.Fatal(err) + } + guardPass(t, r, "first") + if q.unquiesced != cleanupAttempts { + t.Fatal("replacement inherited original process cleanup") + } + if err := r.Get(t.Context(), client.ObjectKeyFromObject(&gs), &gs); err != nil { + t.Fatal(err) + } + if gs.Annotations[lifecycleOperationAnnotation] != "" { + t.Fatal("absent original process kept owner forever") + } +} + +func TestBackupOldAgentFailsBeforeCleanupIntent(t *testing.T) { + r, q := guardFixture(t, guardBackup("first")) + q.checkErr = errors.New("identity endpoint: status 404") + _, _ = r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKey{Namespace: "ns", Name: "first"}}) + b := guardGetBackup(t, r, "first") + if b.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || b.Annotations[annoQuiesceAttempted] != "" || q.quiesced != 0 { + t.Fatalf("unsupported identity created cleanup debt: backup=%+v agent=%+v", b, q) + } + guardPass(t, r, "first") + var gs gameplanev1alpha1.GameServer + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server"}, &gs); err != nil { + t.Fatal(err) + } + if gs.Annotations[lifecycleOperationAnnotation] != "" { + t.Fatal("old agent failure blocked the workload upgrade") + } +} + +func TestBackupDisabledAgentPreservesRawCopyWithoutClaimingQuiesce(t *testing.T) { + r, _ := guardFixture(t, guardBackup("first")) + r.AgentClient = &agent.Client{Disabled: true} + guardPass(t, r, "first") + b := guardGetBackup(t, r, "first") + if b.Annotations[backupPreparationAnnotation] != "" || b.Annotations[annoQuiesceAttempted] != "" { + t.Fatal("disabled agent claimed preparation or quiesce") + } + var job batchv1.Job + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "first"}, &job); err != nil { + t.Fatalf("disabled client prevented raw copy: %v", err) + } + if err := callBackupQuiesce(t.Context(), r.AgentClient, "ns", "server", "", true); err == nil { + t.Fatal("disabled client cleared an existing cleanup obligation") + } +} + +func TestBackupDisabledAgentCannotBypassPendingLiveQuiesce(t *testing.T) { + r, _ := guardFixture(t, guardBackup("first")) + b := guardGetBackup(t, r, "first") + if ok, err := r.claimBackupTarget(t.Context(), b); err != nil || !ok { + t.Fatalf("claim: %t %v", ok, err) + } + gs, err := r.ownedBackupTarget(t.Context(), b) + if err != nil { + t.Fatal(err) + } + if ready, err := r.prepareBackupSource(t.Context(), b, gs); err != nil || !ready { + t.Fatalf("prepare: %t %v", ready, err) + } + patchBackupAnnotations(b, map[string]string{annoQuiesceAttempted: "pending"}) + if err := r.Update(t.Context(), b); err != nil { + t.Fatal(err) + } + r.AgentClient = &agent.Client{Disabled: true} + if err := r.maybeQuiesce(t.Context(), b); err == nil || b.Status.Phase != gameplanev1alpha1.BackupPhaseFailed { + t.Fatalf("disabled client bypassed pending quiesce: %v %+v", err, b.Status) + } +} + +func TestBackupSourceReplacementFailsBeforeCopyOrSuccess(t *testing.T) { + for _, duringRPC := range []bool{true, false} { + t.Run(map[bool]string{true: "during-quiesce", false: "during-copy"}[duringRPC], func(t *testing.T) { + r, q := guardFixture(t, guardBackup("first")) + replace := func() { + var old corev1.Pod + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server-0"}, &old); err != nil { + t.Fatal(err) + } + if err := r.Delete(t.Context(), &old); err != nil { + t.Fatal(err) + } + old.UID, old.ResourceVersion = "replacement", "" + if err := r.Create(t.Context(), &old); err != nil { + t.Fatal(err) + } + } + if duringRPC { + q.afterQuiesce = replace + } else { + guardPass(t, r, "first") + var job batchv1.Job + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "first"}, &job); err != nil { + t.Fatal(err) + } + job.Status.Succeeded = 1 + if err := r.Status().Update(t.Context(), &job); err != nil { + t.Fatal(err) + } + replace() + } + guardPass(t, r, "first") + if got := guardGetBackup(t, r, "first"); got.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || got.Status.SnapshotID != "" { + t.Fatalf("replacement allowed a restorable copy: %+v", got.Status) + } + if duringRPC { + var job batchv1.Job + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "first"}, &job); !apierrors.IsNotFound(err) { + t.Fatal("replacement during RPC created a copy") + } + } + if q.unquiesced != 0 { + t.Fatal("replacement received the old process cleanup") + } + }) + } +} + +func TestBackupCleanupRequiresOriginalProcessExit(t *testing.T) { + for _, state := range []string{"absent", "replacement", "terminal", "terminating", "unready", "legacy-running", "read-error"} { + t.Run(state, func(t *testing.T) { + r, _ := guardFixture(t) + uid := types.UID("game-pod-uid") + var pod corev1.Pod + key := client.ObjectKey{Namespace: "ns", Name: "server-0"} + if err := r.Get(t.Context(), key, &pod); err != nil { + t.Fatal(err) + } + reader := interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if state == "read-error" { + return errors.New("apiserver unavailable") + } + if state == "absent" { + return apierrors.NewNotFound(corev1.Resource("pods"), key.Name) + } + if err := c.Get(ctx, key, obj, opts...); err != nil { + return err + } + p := obj.(*corev1.Pod) + switch state { + case "replacement": + p.UID = "successor" + case "terminal": + p.Status.Phase = corev1.PodFailed + case "terminating": + now := metav1.Now() + p.DeletionTimestamp = &now + case "unready": + p.Status.Phase = corev1.PodPending + } + return nil + }}) + if state == "legacy-running" { + uid = "" + } + gone, err := backupQuiesceProcessGone(t.Context(), reader, "ns", "server", uid) + wantGone := state == "absent" || state == "replacement" || state == "terminal" + if gone != wantGone || (err != nil) != (state == "read-error") { + t.Fatalf("exit proof = %t, %v; want gone %t", gone, err, wantGone) + } + }) + } +} diff --git a/operator/internal/controller/gameserver_controller.go b/operator/internal/controller/gameserver_controller.go index c5a70a9d9..4c76c16d5 100644 --- a/operator/internal/controller/gameserver_controller.go +++ b/operator/internal/controller/gameserver_controller.go @@ -2358,6 +2358,7 @@ func buildAgentContainer( env := []corev1.EnvVar{ {Name: "GAMEPLANE_SERVER_NAME", Value: gs.Name}, {Name: "GAMEPLANE_SERVER_UID", Value: string(gs.UID)}, + {Name: "GAMEPLANE_POD_UID", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.uid"}}}, {Name: "GAMEPLANE_TEMPLATE", Value: tmpl.Name}, {Name: "GAMEPLANE_GAME", Value: tmpl.Spec.Game}, // Games without RCON (consoleMode pty/none) must not have the diff --git a/operator/internal/controller/helpers_envtest_test.go b/operator/internal/controller/helpers_envtest_test.go index 8f10b7053..502be6c55 100644 --- a/operator/internal/controller/helpers_envtest_test.go +++ b/operator/internal/controller/helpers_envtest_test.go @@ -774,6 +774,14 @@ func (f *fakeAgent) Unquiesce(_ context.Context, _, _ string) error { return f.unquiesceErr } +func (f *fakeAgent) CheckPod(context.Context, string, string, string) error { return nil } +func (f *fakeAgent) QuiescePod(ctx context.Context, namespace, server, _ string) error { + return f.Quiesce(ctx, namespace, server) +} +func (f *fakeAgent) UnquiescePod(ctx context.Context, namespace, server, _ string) error { + return f.Unquiesce(ctx, namespace, server) +} + func (f *fakeAgent) quiesceCount() int { f.mu.Lock() defer f.mu.Unlock() diff --git a/operator/internal/controller/lifecycle_operation.go b/operator/internal/controller/lifecycle_operation.go index e63e75597..555a85054 100644 --- a/operator/internal/controller/lifecycle_operation.go +++ b/operator/internal/controller/lifecycle_operation.go @@ -31,6 +31,7 @@ type lifecycleIdentity struct { Generation int64 `json:"generation"` Token string `json:"token,omitempty"` NeedsUnquiesce bool `json:"needsUnquiesce,omitempty"` + QuiescePodUID types.UID `json:"quiescePodUID,omitempty"` // InitialStartup permits only the first workload of a CSI restore's new // target (including WaitForFirstConsumer PVCs), while other operations wait. InitialStartup bool `json:"initialStartup,omitempty"` @@ -279,15 +280,21 @@ func recoverLifecycleOperation(ctx context.Context, c client.Client, reader clie return false, err } } - needsUnquiesce := false for _, predecessor := range op.Recovery { - needsUnquiesce = needsUnquiesce || predecessor.NeedsUnquiesce - } - if needsUnquiesce { - if quiescer == nil { + if !predecessor.NeedsUnquiesce { + continue + } + gone, err := backupQuiesceProcessGone(ctx, reader, gs.Namespace, gs.Name, predecessor.QuiescePodUID) + if err != nil { + return false, err + } + if gone { + continue + } + if !agentQuiesceAvailable(quiescer) { return false, fmt.Errorf("cannot recover orphaned quiesce without an agent client") } - if err := quiescer.Unquiesce(ctx, gs.Namespace, gs.Name); err != nil { + if err := callBackupQuiesce(ctx, quiescer, gs.Namespace, gs.Name, predecessor.QuiescePodUID, true); err != nil { return false, fmt.Errorf("recover orphaned lifecycle quiesce: %w", err) } } @@ -339,10 +346,23 @@ func (r *BackupReconciler) setLifecycleUnquiesceIntent(ctx context.Context, b *g if err != nil { return err } - if op.NeedsUnquiesce == needed { + var podUID types.UID + if needed { + preparation, err := parseBackupPreparation(b) + if err != nil { + return err + } + if preparation != nil { + podUID = preparation.PodUID + } + } + if op.NeedsUnquiesce == needed && (!needed || op.QuiescePodUID == podUID) { return nil } op.NeedsUnquiesce = needed + if needed { + op.QuiescePodUID = podUID + } gs.Annotations[lifecycleOperationAnnotation] = lifecycleOperationJSON(op) return r.Update(ctx, gs) } diff --git a/operator/specs.md b/operator/specs.md index d0e1fceea..5000b0276 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -218,6 +218,10 @@ Primary reconcilers register with the manager in `cmd/main.go` and handle CRD li - **Responsibility:** Drive Backup to completion: coordinate quiesce (if agent available), spawn restic Job, track snapshot ID. - **Key functions:** - Quiesce orchestration: call agent Quiesce before Job, Unquiesce after completion. + - With quiesce and an agent client enabled, preparation is recorded once in `backup.gameplane.local/preparation`, bound to the target UID and StatefulSet UID/generation. `already-stopped` requires uncached proof of desired and observed zero replicas, caught-up observed generation, and no live game/PVC user pods or pending writer Jobs. Current copy workers are excluded by Job ownership. This path sends no agent calls and preserves suspend/idle/wake intent under the shared workload fence. It applies equally to restic and CSI copies. + - Live preparation also pins the game Pod UID. The operator checks the agent's read-only `/v1/pods/{uid}/identity` endpoint before recording the first cleanup intent, then uses pod-bound quiesce/unquiesce routes; the agent rejects a different UID before RCON. A missing endpoint or identity mismatch fails closed without a legacy-route fallback. Ambiguous quiesce errors retain cleanup intent and the original Pod UID on both the Backup and shared owner. Only confirmed save-on or uncached evidence that the original pod is absent/replaced/terminal permits cleanup; an HTTP timeout, unready pod, or terminating pod is insufficient. Legacy attempts without a UID retire only on absence/terminal proof and never adopt a new UID. `Unquiesced=True, Reason=SourceProcessGone` distinguishes process exit from an actual save-on RPC. + - Source proof is checked before and after quiesce and on each copy reconcile. Observed workload/process replacement fails the Backup before it can report a restorable success. Owner recovery retains the pod UID even if the Backup is force-deleted, drains predecessor workers, and never sends its cleanup to a successor pod. + - Upgrade the operator and agent image together and allow existing GameServer pods to roll out the new agent and `GAMEPLANE_POD_UID` DownwardAPI env. Older agents fail the identity preflight before cleanup debt, so the workload can continue its upgrade after terminal cleanup releases the owner. Retry failed copies after rollout; preflight never bypasses protection. This does not automatically stop a running game. - Every copy, including `quiesce=false` and CSI snapshots, acquires the shared lifecycle owner before starting. The Backup pins its target in `backup.gameplane.local/target`; `backup.gameplane.local/owner` remains a compatibility marker. The shared owner is described below. Quiesce cleanup intent is recorded on both the Backup and the surviving GameServer before calling the agent, including partial failures. Cleanup waits for live copy workers; deletion cancels the copy Job with foreground propagation first. Every Backup carries the cleanup finalizer. Pre-upgrade quiesced workers are observed through completion even when overlapping legacy copies cannot acquire the new exclusive owner. - Orphan recovery preserves predecessor operation identities in the shared owner's `recovery` array, atomically with ownership transfer. Existing `backup.gameplane.local/recovering-owners` lineage is imported during migration. Deleting a recovering owner cannot hide an earlier worker. Normal deletion, orphan recovery, and legacy cleanup also wait for pending owned CSI captures to report ready or an explicit error; VolumeSnapshots are reclaimed by owner-reference GC after cleanup. - VolumeSnapshot API availability is checked before quiescing. Snapshot creation errors fail the Backup and run unquiesce cleanup immediately instead of retrying creation indefinitely with saving disabled. API/creation failures persist `backup.gameplane.local/snapshot-aborted` to allow cleanup without waiting for a failed/nonrestorable capture. Failed unquiesce remains visible and retried. A transport failure can leave a CSI snapshot object despite a Failed Backup; this does not make that Backup restorable. @@ -478,7 +482,7 @@ Forgetting codegen leaves the YAML out of sync with types — CI's `make manifes 6. **Module sources are immutable once pulled.** Digest pinning (spec.digest) defeats tag moves; version pinning (spec.version) tracks a specific semver. Floating (unset version/digest) tracks latest and re-pulls on ModuleSource refresh. -7. **Backup quiesce is best-effort.** If agent unavailable, backup proceeds raw (no pause). If quiesce unsupported (agent returns ErrUnsupported), backup continues degraded (success-with-note). +7. **Backup quiesce fails closed on live transport errors.** With no configured agent client or `quiesce=false`, a copy proceeds raw. With a client enabled, a drained stopped source skips RPC, but an unreachable live agent prevents the copy. A game that explicitly reports quiesce unsupported can continue without claiming it was quiesced. 8. **Remote cluster health checks are non-blocking.** A Cluster with health Unhealthy does not prevent GameServer creation on the local cluster; it surfaces the issue so operators can intervene. diff --git a/test/e2e/backup_stopped_e2e_test.go b/test/e2e/backup_stopped_e2e_test.go new file mode 100644 index 000000000..d469a635d --- /dev/null +++ b/test/e2e/backup_stopped_e2e_test.go @@ -0,0 +1,112 @@ +//go:build e2e + +package e2e + +import ( + "context" + "encoding/json" + "fmt" + "regexp" + "testing" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" +) + +// Exercise real PVC data and restic for two consecutive protected copies while +// the agent is absent. This test needs no API login and runs in the operator bucket. +func TestBackup_StoppedCopiesRemainRestorable(t *testing.T) { + t.Parallel() + ctx := context.Background() + ns, gs, tmpl := "gameplane-games", "e2e-stopped-backup", "e2e-stopped-backup-template" + ensureResticRepo(t) + applyBusyboxTemplate(t, tmpl) + applyBusyboxGameServer(t, ns, gs, tmpl) + waitPVCBound(t, ns, gs+"-data", 90*time.Second) + envInstance.Eventually(t, 2*time.Minute, func() (bool, string) { + pod, err := envInstance.K8s.CoreV1().Pods(ns).Get(ctx, gs+"-0", metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + for _, status := range pod.Status.ContainerStatuses { + if status.Name == "game" && status.Ready { + return true, "" + } + } + return false, "game container not ready" + }) + if out, err := envInstance.KubectlExec(t, ns, "pod/"+gs+"-0", "sh", "-c", "echo stopped-backup-marker > /data/marker.txt && sync"); err != nil { + t.Fatalf("write marker: %v\n%s", err, out) + } + if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Patch(ctx, gs, types.MergePatchType, []byte(`{"spec":{"suspend":true}}`), metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + envInstance.Eventually(t, 3*time.Minute, func() (bool, string) { + ss, err := envInstance.K8s.AppsV1().StatefulSets(ns).Get(ctx, gs, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + if ss.Spec.Replicas == nil || *ss.Spec.Replicas != 0 || ss.Status.Replicas != 0 || ss.Status.ObservedGeneration < ss.Generation { + return false, "workload not drained" + } + _, err = envInstance.K8s.CoreV1().Pods(ns).Get(ctx, gs+"-0", metav1.GetOptions{}) + return apierrors.IsNotFound(err), fmt.Sprintf("pod absence: %v", err) + }) + for i := 1; i <= 2; i++ { + name := fmt.Sprintf("e2e-stopped-copy-%d", i) + backup := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "gameplane.local/v1alpha1", "kind": "Backup", + "metadata": map[string]any{"name": name, "namespace": ns}, + "spec": map[string]any{"serverRef": map[string]any{"name": gs}, "repoRef": map[string]any{"name": "e2e-restic-creds", "key": "repo"}, "strategy": "restic-snapshot", "quiesce": true}, + }} + if _, err := envInstance.Dyn.Resource(backupGVR).Namespace(ns).Create(ctx, backup, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { deleteBackupCR(t, ns, name) }) + var snapshotID string + envInstance.Eventually(t, 5*time.Minute, func() (bool, string) { + // Check throughout the copy, not only after its terminal result. + _, err := envInstance.K8s.CoreV1().Pods(ns).Get(ctx, gs+"-0", metav1.GetOptions{}) + if !apierrors.IsNotFound(err) { + t.Fatalf("backup woke game or absence could not be confirmed: %v", err) + } + got, err := envInstance.Dyn.Resource(backupGVR).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + phase, _, _ := unstructured.NestedString(got.Object, "status", "phase") + if phase == "Failed" { + t.Fatalf("stopped backup failed: %+v", got.Object["status"]) + } + if phase != "Succeeded" { + return false, "phase=" + phase + } + var preparation struct { + Outcome string `json:"outcome"` + } + if err := json.Unmarshal([]byte(got.GetAnnotations()["backup.gameplane.local/preparation"]), &preparation); err != nil || preparation.Outcome != "already-stopped" { + t.Fatalf("missing stopped-source proof: %+v, %v", preparation, err) + } + if got.GetAnnotations()["backup.gameplane.local/quiesce-attempted"] != "" { + t.Fatal("stopped source contacted absent agent") + } + snapshotID, _, _ = unstructured.NestedString(got.Object, "status", "snapshotID") + return snapshotID != "", "snapshot ID pending" + }) + if !regexp.MustCompile(`^[0-9a-f]{8,64}$`).MatchString(snapshotID) { + t.Fatalf("unexpected snapshot ID %q", snapshotID) + } + runResticJob(t, "restic restore "+snapshotID+" --target /tmp/stopped-restore --retry-lock 5m && test \"$(cat /tmp/stopped-restore/data/marker.txt)\" = stopped-backup-marker") + } + got, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Get(ctx, gs, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + suspended, _, _ := unstructured.NestedBool(got.Object, "spec", "suspend") + if !suspended { + t.Fatal("backup changed manual suspend intent") + } +} diff --git a/test/e2e/buckets.sh b/test/e2e/buckets.sh index d60a26f14..b743bd484 100755 --- a/test/e2e/buckets.sh +++ b/test/e2e/buckets.sh @@ -83,6 +83,7 @@ TestModuleSourceUpload TestModule_ScaffoldAndPackage TestModule_SteamcmdScaffoldKeepsNonRootSecurity TestBackup_OperatorMaterializesJob +TestBackup_StoppedCopiesRemainRestorable TestBackup_FailsOnMissingPVC TestBackup_FailsOnBadCredentials TestBackup_DeleteForgetsSnapshot From fe3b965cbb522c545b8daddf7173acd5ed126171 Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 21 Oct 2026 12:36:56 -0400 Subject: [PATCH 02/20] docs: describe pod-bound agent operations and clarify test fakes Signed-off-by: Baylem --- operator/internal/agent/client.go | 7 +++++-- operator/internal/controller/backup_guard_test.go | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/operator/internal/agent/client.go b/operator/internal/agent/client.go index d5a573c7c..2b1c06923 100644 --- a/operator/internal/agent/client.go +++ b/operator/internal/agent/client.go @@ -120,10 +120,11 @@ func (c *Client) Unquiesce(ctx context.Context, namespace, server string) error return c.callQuiesce(ctx, namespace, server, "/unquiesce", false) } -// Pod-bound routes reject a replacement process at the final network hop. -// Older agents return 404 without executing a mutation; never fall back. +// QuiesceEnabled reports whether this client can perform quiesce operations. func (c *Client) QuiesceEnabled() bool { return c != nil && !c.Disabled } +// CheckPod verifies that the responding agent belongs to the selected Pod. +// Older agents return 404; callers must not fall back to unbound mutations. func (c *Client) CheckPod(ctx context.Context, namespace, server, podUID string) error { if c.Disabled || podUID == "" { return fmt.Errorf("agent: pod identity check requires an enabled client and pod UID") @@ -143,10 +144,12 @@ func (c *Client) CheckPod(ctx context.Context, namespace, server, podUID string) return nil } +// QuiescePod disables saving only on the selected Pod's agent. func (c *Client) QuiescePod(ctx context.Context, namespace, server, podUID string) error { return c.callPodQuiesce(ctx, namespace, server, podUID, "/quiesce", true) } +// UnquiescePod restores saving only on the selected Pod's agent. func (c *Client) UnquiescePod(ctx context.Context, namespace, server, podUID string) error { return c.callPodQuiesce(ctx, namespace, server, podUID, "/unquiesce", false) } diff --git a/operator/internal/controller/backup_guard_test.go b/operator/internal/controller/backup_guard_test.go index 8009ae300..b6d2ccb2d 100644 --- a/operator/internal/controller/backup_guard_test.go +++ b/operator/internal/controller/backup_guard_test.go @@ -42,14 +42,14 @@ func (q *guardQuiescer) Unquiesce(context.Context, string, string) error { return q.unquiesceErr } func (q *guardQuiescer) CheckPod(context.Context, string, string, string) error { return q.checkErr } -func (q *guardQuiescer) QuiescePod(ctx context.Context, namespace, server, podUID string) error { +func (q *guardQuiescer) QuiescePod(ctx context.Context, namespace, server, _ string) error { err := q.Quiesce(ctx, namespace, server) if q.afterQuiesce != nil { q.afterQuiesce() } return err } -func (q *guardQuiescer) UnquiescePod(ctx context.Context, namespace, server, podUID string) error { +func (q *guardQuiescer) UnquiescePod(ctx context.Context, namespace, server, _ string) error { return q.Unquiesce(ctx, namespace, server) } From eb7db1c4ca7a3efde4851a5e2b451f04c574586f Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 21 Oct 2026 22:49:07 -0400 Subject: [PATCH 03/20] fix(operator): preserve condition changes before status comparison Signed-off-by: Baylem --- operator/internal/controller/helpers.go | 4 +++- operator/internal/controller/helpers_extra_test.go | 13 +++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/operator/internal/controller/helpers.go b/operator/internal/controller/helpers.go index be11001c0..8bd3890d6 100644 --- a/operator/internal/controller/helpers.go +++ b/operator/internal/controller/helpers.go @@ -13,8 +13,10 @@ import ( ) // upsertCondition replaces a condition by Type if present, otherwise -// appends. Sets LastTransitionTime to now when the status flips. +// appends. Sets LastTransitionTime to now when the status flips. Preserve the +// input slice so callers can compare old and new conditions before persisting. func upsertCondition(conds []metav1.Condition, c metav1.Condition) []metav1.Condition { + conds = append([]metav1.Condition(nil), conds...) now := metav1.Now() for i := range conds { if conds[i].Type != c.Type { diff --git a/operator/internal/controller/helpers_extra_test.go b/operator/internal/controller/helpers_extra_test.go index 85a8d2ccf..ea6ab4ce0 100644 --- a/operator/internal/controller/helpers_extra_test.go +++ b/operator/internal/controller/helpers_extra_test.go @@ -53,6 +53,19 @@ func TestUpsertCondition_StatusFlip_BumpsTimestamp(t *testing.T) { } } +func TestUpsertConditionPreservesComparisonBase(t *testing.T) { + for _, status := range []metav1.ConditionStatus{metav1.ConditionFalse, metav1.ConditionTrue} { + t.Run(string(status), func(t *testing.T) { + original := metav1.Condition{Type: "SnapshotCleaned", Status: metav1.ConditionFalse, Reason: "Pending", LastTransitionTime: metav1.Now()} + conditions := []metav1.Condition{original} + updated := upsertCondition(conditions, metav1.Condition{Type: original.Type, Status: status, Reason: "Updated"}) + if conditions[0] != original || sameConditions(conditions, updated) { + t.Fatalf("upsert hid the change from status persistence: before=%+v after=%+v", conditions, updated) + } + }) + } +} + func TestSameConditions(t *testing.T) { a := []metav1.Condition{ {Type: "Ready", Status: metav1.ConditionTrue, Reason: "Up"}, From b4268dc02568bc6be25493052461364245f08b31 Mon Sep 17 00:00:00 2001 From: Baylem Date: Thu, 22 Oct 2026 20:31:36 -0400 Subject: [PATCH 04/20] fix(operator): drain failed restore startup before releasing ownership Signed-off-by: Baylem --- docs/operator-operations.md | 7 + .../controller/gameserver_controller.go | 11 +- .../controller/lifecycle_operation.go | 16 ++ .../internal/controller/restore_lifecycle.go | 32 ++++ .../restore_startup_cleanup_test.go | 181 ++++++++++++++++++ operator/specs.md | 2 +- 6 files changed, 246 insertions(+), 3 deletions(-) create mode 100644 operator/internal/controller/restore_startup_cleanup_test.go diff --git a/docs/operator-operations.md b/docs/operator-operations.md index 35a0753d5..1a90d084e 100644 --- a/docs/operator-operations.md +++ b/docs/operator-operations.md @@ -37,6 +37,13 @@ is allowed under that reservation so storage using WaitForFirstConsumer can bind Other operations wait until the restore completes. Existing data-PVC restores continue to require an observed stopped workload and drained writers. +If initial startup fails or the Restore is deleted, cleanup first retires its +startup permission and suspends the target. The operation remains held until the +StatefulSet has observed zero replicas and every live or terminating game/PVC +writer has drained. Recovery of a force-deleted Restore applies the same check +before a successor can proceed, preserving the existing pod template while +requesting zero replicas. A successful Restore leaves its initial workload running. + Stopped-source preparation (#716), Restic preparation and quiesce deadlines (#717), CSI capture deadlines (#719), and staged template adoption (#731) are separate changes building on this ownership boundary. diff --git a/operator/internal/controller/gameserver_controller.go b/operator/internal/controller/gameserver_controller.go index 4c76c16d5..0821a6e86 100644 --- a/operator/internal/controller/gameserver_controller.go +++ b/operator/internal/controller/gameserver_controller.go @@ -1213,6 +1213,8 @@ func (r *GameServerReconciler) desiredReplicas( return 0, 5 * time.Second, nil } return 1, 5 * time.Second, nil + } else if held && lifecycleRecoveryNeedsStop(op) { + return 0, 5 * time.Second, nil } else if held && (op.Kind == "Backup" || len(op.Recovery) != 0) { var ss appsv1.StatefulSet err := r.wipeReader().Get(ctx, client.ObjectKeyFromObject(gs), &ss) @@ -1512,8 +1514,13 @@ func (r *GameServerReconciler) reconcileStatefulSet( } if op.Kind == "Backup" || len(op.Recovery) != 0 { if !ss.CreationTimestamp.IsZero() || ss.ResourceVersion != "" { - // Preserve replicas AND the complete pod template, including - // config hashes, images, init containers and mount topology. + // Orphaned restores may leave a running initial workload. + // Stop it while preserving its complete pod template. Other + // protected operations also preserve the replica count. + if lifecycleRecoveryNeedsStop(op) { + zero := int32(0) + ss.Spec.Replicas = &zero + } return nil } // No workload exists yet. Create it stopped; this gives backup diff --git a/operator/internal/controller/lifecycle_operation.go b/operator/internal/controller/lifecycle_operation.go index 555a85054..fc1d626fb 100644 --- a/operator/internal/controller/lifecycle_operation.go +++ b/operator/internal/controller/lifecycle_operation.go @@ -262,6 +262,15 @@ func legacyLifecycleCopiesLive(ctx context.Context, c client.Client, reader clie return false, nil } +func lifecycleRecoveryNeedsStop(op lifecycleOperation) bool { + for _, predecessor := range op.Recovery { + if predecessor.Kind == "Restore" { + return true + } + } + return false +} + func recoverLifecycleOperation(ctx context.Context, c client.Client, reader client.Reader, gs *gameplanev1alpha1.GameServer, op lifecycleOperation, quiescer AgentQuiescer) (bool, error) { for _, predecessor := range op.Recovery { var busy bool @@ -280,6 +289,13 @@ func recoverLifecycleOperation(ctx context.Context, c client.Client, reader clie return false, err } } + if lifecycleRecoveryNeedsStop(op) { + r := &RestoreReconciler{Client: c, APIReader: reader, Scheme: c.Scheme()} + stopped, _, err := r.restoreTargetStopped(ctx, gs) + if err != nil || !stopped { + return false, err + } + } for _, predecessor := range op.Recovery { if !predecessor.NeedsUnquiesce { continue diff --git a/operator/internal/controller/restore_lifecycle.go b/operator/internal/controller/restore_lifecycle.go index f54d09756..91916ea54 100644 --- a/operator/internal/controller/restore_lifecycle.go +++ b/operator/internal/controller/restore_lifecycle.go @@ -262,11 +262,43 @@ func (r *RestoreReconciler) cleanupRestore(ctx context.Context, rs *gameplanev1a if err != nil { return ctrl.Result{}, err } + if !sameLifecycleOwner(op.lifecycleIdentity, restoreLifecycleIdentity(&gs, rs)) { + return ctrl.Result{}, fmt.Errorf("restore cleanup target has a different lifecycle owner") + } + if !success && (op.InitialStartup || !gs.Spec.Suspend) { + // A CSI restore may have started the game to bind its PVC. + // Retire that permission before asking the workload to stop; + // a replacement operation must not overlap the old process. + op.InitialStartup = false + gs.Spec.Suspend = true + gs.Annotations[lifecycleOperationAnnotation] = lifecycleOperationJSON(op) + if err := r.Update(ctx, &gs); err != nil { + return ctrl.Result{}, err + } + } ready, err := recoverLifecycleOperation(ctx, r.Client, r.apiReader(), &gs, op, r.OperationQuiescer) if err != nil || !ready { return ctrl.Result{RequeueAfter: 5 * time.Second}, err } } + if !success { + if !gs.Spec.Suspend { + gs.Spec.Suspend = true + if err := r.Update(ctx, &gs); err != nil { + return ctrl.Result{}, err + } + } + stopped, progressed, err := r.restoreTargetStopped(ctx, &gs) + if err != nil { + return ctrl.Result{}, err + } + if progressed { + return ctrl.Result{Requeue: true}, nil + } + if !stopped { + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + } + } if err := clearLifecycleOperation(ctx, r.Client, r.apiReader(), &gs, restoreLifecycleIdentity(&gs, rs)); err != nil { return ctrl.Result{}, err } diff --git a/operator/internal/controller/restore_startup_cleanup_test.go b/operator/internal/controller/restore_startup_cleanup_test.go new file mode 100644 index 000000000..5ac8e40e1 --- /dev/null +++ b/operator/internal/controller/restore_startup_cleanup_test.go @@ -0,0 +1,181 @@ +package controller + +import ( + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +func startupCleanupFixture(t *testing.T) (*RestoreReconciler, *gameplanev1alpha1.GameServer, *gameplanev1alpha1.Restore, *appsv1.StatefulSet, *corev1.Pod) { + t.Helper() + r, gs, rs := restoreLifecycleFixture(t) + rs.Annotations = map[string]string{restoreTargetAnnotation: restoreIdentityJSON(gs.Name, gs.UID)} + rs.Finalizers = []string{restoreCleanupFinalizer} + if err := r.Update(t.Context(), rs); err != nil { + t.Fatal(err) + } + op := lifecycleOperation{lifecycleIdentity: restoreLifecycleIdentity(gs, rs)} + op.InitialStartup = true + gs.Spec.Suspend = false + gs.Annotations = map[string]string{restoreGuardAnnotation: restoreIdentityJSON(rs.Name, rs.UID), lifecycleOperationAnnotation: lifecycleOperationJSON(op)} + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + ss := &appsv1.StatefulSet{} + if err := r.Get(t.Context(), client.ObjectKeyFromObject(gs), ss); err != nil { + t.Fatal(err) + } + one := int32(1) + ss.Spec.Replicas = &one + ss.Spec.Template.Spec.Containers = []corev1.Container{{Name: "game", Image: "restored-image"}} + if err := r.Update(t.Context(), ss); err != nil { + t.Fatal(err) + } + ss.Status.Replicas = 1 + ss.Status.ObservedGeneration = ss.Generation + if err := r.Status().Update(t.Context(), ss); err != nil { + t.Fatal(err) + } + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: gs.Name + "-0", Namespace: gs.Namespace, UID: "startup-pod", Finalizers: []string{"test/hold"}, OwnerReferences: []metav1.OwnerReference{{APIVersion: "apps/v1", Kind: "StatefulSet", Name: ss.Name, UID: ss.UID, Controller: ownerBoolPtr(true)}}}, Status: corev1.PodStatus{Phase: corev1.PodRunning}} + if err := r.Create(t.Context(), pod); err != nil { + t.Fatal(err) + } + return r, gs, rs, ss, pod +} + +func TestRestoreStartupCleanupDrainsBeforeRelease(t *testing.T) { + for _, outcome := range []string{"failed", "deleted", "succeeded"} { + t.Run(outcome, func(t *testing.T) { + r, gs, rs, ss, pod := startupCleanupFixture(t) + rs.Status.Phase = gameplanev1alpha1.RestorePhaseFailed + if outcome == "succeeded" { + rs.Status.Phase = gameplanev1alpha1.RestorePhaseSucceeded + } + if err := r.Status().Update(t.Context(), rs); err != nil { + t.Fatal(err) + } + if outcome == "deleted" { + if err := r.Delete(t.Context(), rs); err != nil { + t.Fatal(err) + } + if err := r.Get(t.Context(), client.ObjectKeyFromObject(rs), rs); err != nil { + t.Fatal(err) + } + } + res, err := r.cleanupRestore(t.Context(), rs) + if err != nil { + t.Fatal(err) + } + gs = lifecycleServer(t, r) + if outcome == "succeeded" { + if gs.Spec.Suspend || gs.Annotations[lifecycleOperationAnnotation] != "" || res.RequeueAfter != 0 { + t.Fatalf("successful startup was stopped: %+v %+v", gs, res) + } + return + } + op, held, err := lifecycleOwner(gs) + if err != nil || !held || op.InitialStartup || !gs.Spec.Suspend || res.RequeueAfter == 0 { + t.Fatalf("failed startup released or retained permission to start: %+v %+v %v", gs, res, err) + } + // An observed scale-down does not prove a terminating Pod drained. + zero := int32(0) + ss.Spec.Replicas = &zero + if err := r.Update(t.Context(), ss); err != nil { + t.Fatal(err) + } + ss.Status.Replicas, ss.Status.ObservedGeneration = 0, ss.Generation + if err := r.Status().Update(t.Context(), ss); err != nil { + t.Fatal(err) + } + if err := r.Delete(t.Context(), pod); err != nil { + t.Fatal(err) + } + if res, err := r.cleanupRestore(t.Context(), rs); err != nil || res.RequeueAfter == 0 || lifecycleServer(t, r).Annotations[lifecycleOperationAnnotation] == "" { + t.Fatalf("terminating startup Pod lost exclusion: %+v %v", res, err) + } + if err := r.Get(t.Context(), client.ObjectKeyFromObject(pod), pod); err != nil { + t.Fatal(err) + } + pod.Finalizers = nil + if err := r.Update(t.Context(), pod); err != nil { + t.Fatal(err) + } + for range 2 { + if _, err := r.cleanupRestore(t.Context(), rs); err != nil { + t.Fatal(err) + } + } + gs = lifecycleServer(t, r) + if !gs.Spec.Suspend || gs.Annotations[lifecycleOperationAnnotation] != "" || gs.Annotations[restoreGuardAnnotation] != "" { + t.Fatalf("drained failed startup did not settle suspended: %+v", gs) + } + }) + } +} + +func TestOrphanRestoreStartupDrainsBeforeNextOwnerProceeds(t *testing.T) { + r, gs, rs, ss, pod := startupCleanupFixture(t) + rs.Finalizers = nil + if err := r.Update(t.Context(), rs); err != nil { + t.Fatal(err) + } + if err := r.Delete(t.Context(), rs); err != nil { + t.Fatal(err) + } + b := guardBackup("next") + if err := r.Create(t.Context(), b); err != nil { + t.Fatal(err) + } + want := backupLifecycleIdentity(gs, b, false) + if ready, err := claimLifecycleOperation(t.Context(), r.Client, r.apiReader(), gs, want, nil); err != nil || ready { + t.Fatalf("orphan startup admitted another operation: %t %v", ready, err) + } + gr := &GameServerReconciler{Client: r.Client, APIReader: r.apiReader(), Scheme: r.Scheme} + if replicas, _, err := gr.desiredReplicas(t.Context(), gs, &gameplanev1alpha1.GameTemplate{}, idleAwake); err != nil || replicas != 0 { + t.Fatalf("orphan recovery did not request zero replicas: %d %v", replicas, err) + } + // Even a stale callback asking for one replica must stop the old process + // without changing the template before the successor owns clean data. + if err := gr.reconcileStatefulSet(t.Context(), gs, &gameplanev1alpha1.GameTemplate{}, nil, &materializedConfig{}, 1); err != nil { + t.Fatal(err) + } + if err := r.Get(t.Context(), client.ObjectKeyFromObject(ss), ss); err != nil { + t.Fatal(err) + } + if *ss.Spec.Replicas != 0 || ss.Spec.Template.Spec.Containers[0].Image != "restored-image" { + t.Fatalf("orphan recovery did not preserve the stopped template: %+v", ss.Spec) + } + ss.Status.Replicas, ss.Status.ObservedGeneration = 0, ss.Generation + if err := r.Status().Update(t.Context(), ss); err != nil { + t.Fatal(err) + } + if ready, err := claimLifecycleOperation(t.Context(), r.Client, r.apiReader(), gs, want, nil); err != nil || ready { + t.Fatalf("observed scale-down bypassed live startup Pod: %t %v", ready, err) + } + pod.Finalizers = nil + if err := r.Update(t.Context(), pod); err != nil { + t.Fatal(err) + } + if err := r.Delete(t.Context(), pod); err != nil { + t.Fatal(err) + } + ready := false + for range 2 { + var err error + ready, err = claimLifecycleOperation(t.Context(), r.Client, r.apiReader(), gs, want, nil) + if err != nil { + t.Fatal(err) + } + if ready { + break + } + } + if !ready { + t.Fatal("drained predecessor did not permit successor") + } +} diff --git a/operator/specs.md b/operator/specs.md index 5000b0276..37d215b4a 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -249,7 +249,7 @@ Primary reconcilers register with the manager in `cmd/main.go` and handle CRD li - Suspend: wait for Suspended/Stopped status plus the actual workload: the owned StatefulSet must exist, request zero replicas, report zero replicas, and have observed its current generation. An absent StatefulSet waits for workload reconciliation; a missing template or broken workload must be repaired before restore can proceed. No live or terminating game pod or pod mounting the data PVC may remain. Nonterminal Jobs mounting that PVC must also finish before a new restore Job starts. These checks use the uncached API reader and run again immediately before creating the Job. - The stopped StatefulSet also receives the owning Restore guard. A successful fence write immediately requeues the Restore to repeat the workload and writer checks; only external drainage waits use the polling delay. Its mutation callback honors that guard even when the GameServer cache snapshot predates acquisition, fencing delayed scale-up writes with the StatefulSet resourceVersion. Cleanup removes both guards after workers drain. - Restore Job: mount snapshot (from VolumeSnapshot or restic), run restic restore. Job carries BackoffLimit and ActiveDeadlineSeconds from the reconciler's JobBackoffLimit / JobActiveDeadlineSeconds fields (operator flags --backup-job-backoff-limit / --backup-job-active-deadline-seconds, Helm operator.backupJobBackoffLimit / operator.backupJobActiveDeadlineSeconds; defaults 2 and 86400, applied when unset; spec 018 OD-027); Job failure is recognized only from a Failed=True condition. Legacy in-flight Jobs without the pinned target annotation fail the Restore and are canceled/drained; operators can start a fresh Restore after cleanup. Pod SecurityContext takes FSGroup from the target GameTemplate's spec.security.fsGroup (fallback 65532 if template missing). - - Resume: once the restore Job succeeds and its live pods have drained, persist Succeeded before releasing the matching guard and clearing suspend in the same pass. This ordering survives interrupted status writes or operator restarts; it does not wait for the GameServer to reach Running again. Failure/deletion cancels active owned Jobs with foreground deletion and retains the guard/finalizer until Jobs and live worker pods drain, then leaves the target suspended. A missing or recreated guard owner can be recovered only after its UID's orphan Jobs and workers drain. Target recreation fails the old restore; cleanup never changes the replacement server. + - Resume: once the restore Job succeeds and its live pods have drained, persist Succeeded before releasing the matching guard and clearing suspend in the same pass. This ordering survives interrupted status writes or operator restarts; it does not wait for the GameServer to reach Running again. Failure/deletion cancels active owned Jobs with foreground deletion, retires CSI initial-startup permission and suspends the target while retaining the guard/finalizer. Release additionally requires an owned StatefulSet with desired/observed zero replicas at its current generation, no live or terminating game/PVC writer pods and no pending PVC writer Jobs. Orphan recovery applies the same stopped-target proof before admitting a successor, preserving the template while forcing zero replicas. Successful CSI restores keep their initial workload running. Target recreation fails the old restore; cleanup never changes the replacement server. - Status: report phase (Pending/Suspending/Running/Succeeded/Failed). `RestorePhaseResuming` is declared on the type but never assigned by this reconciler (F-260). - Volume-snapshot restore reference handling: When a volume-snapshot restore stands up a new GameServer from the original server's spec, any references in spec.env (SecretKeyRef, ConfigMapKeyRef) and spec.networking.tunnel.credentialsSecretRef that are owned by the original server must be copied to new objects owned by the restored server, with rewritten names, because the restored server does not own the originals. Ownership of a source object is an OwnerReference to the original server (Kind, Name, UID) and is re-verified against the source object immediately before each copy, not just once during planning. An object that already exists under a copy name is accepted only if the restored server controls it (a controller OwnerReference, `controller: true`); otherwise the restore fails. The restore fails before the new GameServer is created if the original server does not own every referenced object. The references are rewritten in the new GameServer's spec when it is created, and the copies are made right after creation and re-ensured on every reconcile pass. Only an ownership refusal fails the restore; transient errors (stale cache reads, write conflicts, a referenced object not yet visible in the cache) requeue it. A restore's 10-minute deadline starts when the restore first plans the new server (before it is created) and covers both creating the server with its reference copies and waiting for it to reach Running; it is enforced even while a transient error (such as a referenced object that stays missing) keeps recurring, so the restore cannot run past the limit, and once it expires the restore fails with a deadline message. From 84119f28f6a6de8fc9c9149ec4f815f1db9753fe Mon Sep 17 00:00:00 2001 From: Baylem Date: Thu, 22 Oct 2026 20:33:28 -0400 Subject: [PATCH 05/20] fix(ci): drive real changes during event stream checks Signed-off-by: Baylem --- .github/workflows/ci.yaml | 2 +- test/e2e/api_events_roles_scope_e2e_test.go | 57 +++++++++++++++++++-- 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a1c9aecdd..778cdb8cc 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -372,7 +372,7 @@ jobs: needs: [changes] if: needs.changes.outputs.images == 'true' runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 30 permissions: contents: read steps: diff --git a/test/e2e/api_events_roles_scope_e2e_test.go b/test/e2e/api_events_roles_scope_e2e_test.go index 2ab19b2b4..4c3b388c0 100644 --- a/test/e2e/api_events_roles_scope_e2e_test.go +++ b/test/e2e/api_events_roles_scope_e2e_test.go @@ -12,6 +12,9 @@ import ( "strings" "testing" "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ) // TestAPI_EventStreamAndRoleEdits_FollowCallerPermissions checks two @@ -64,19 +67,37 @@ func TestAPI_EventStreamAndRoleEdits_FollowCallerPermissions(t *testing.T) { applyBusyboxTemplate(t, tmpl) createGameServerViaAPI(t, admin, ns, gsName, tmpl) + // /events starts at each collection's current resourceVersion. Drive real + // updates throughout both streams instead of depending on initial replay + // or other parallel tests to happen to mutate a readable resource kind. + changes := func(ctx context.Context) error { + patch, err := json.Marshal(map[string]any{"metadata": map[string]any{"annotations": map[string]string{ + "gameplane.local/e2e-event-sequence": time.Now().UTC().Format(time.RFC3339Nano), + }}}) + if err != nil { + return fmt.Errorf("encode event trigger: %w", err) + } + if _, err := envInstance.Dyn.Resource(gameTemplateGVR).Patch(ctx, tmpl, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + return fmt.Errorf("trigger template event: %w", err) + } + if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Patch(ctx, gsName, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + return fmt.Errorf("trigger server event: %w", err) + } + return nil + } user := envInstance.APIClient(t, userName, userPW) defer user.Close() t.Run("admin event stream includes templates", func(t *testing.T) { - kinds, found := streamEventKinds(t, admin, ns, func(kind, _ string) bool { return kind == "templates" }, 0) + kinds, found := streamEventKinds(t, admin, ns, func(kind, name string) bool { return kind == "templates" && name == tmpl }, 0, changes) if !found { t.Fatalf("admin stream sent no templates event; kinds=%v", kinds) } }) t.Run("event stream sends only kinds the caller may read", func(t *testing.T) { - kinds, found := streamEventKinds(t, user, ns, func(kind, name string) bool { return kind == "servers" && name == gsName }, 3*time.Second) + kinds, found := streamEventKinds(t, user, ns, func(kind, name string) bool { return kind == "servers" && name == gsName }, 3*time.Second, changes) if !found { t.Fatalf("user stream never sent server %s; kinds=%v", gsName, kinds) } @@ -126,11 +147,41 @@ func TestAPI_EventStreamAndRoleEdits_FollowCallerPermissions(t *testing.T) { // streamEventKinds reads GET /events as cli and returns the set of event // kinds it received. When match reports true for a frame, it keeps // reading for grace (0 = stop at once); otherwise it stops after 30s. +// changes keeps both permitted and forbidden kinds active while connecting and +// reading; its worker is cancelled and joined before the helper returns. // found reports whether match was ever true. -func streamEventKinds(t *testing.T, cli *APIClient, ns string, match func(kind, name string) bool, grace time.Duration) (map[string]bool, bool) { +func streamEventKinds(t *testing.T, cli *APIClient, ns string, match func(kind, name string) bool, grace time.Duration, changes func(context.Context) error) (map[string]bool, bool) { t.Helper() ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() + changesDone := make(chan struct{}) + var changesErr error + go func() { + defer close(changesDone) + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + for { + if err := changes(ctx); err != nil { + if ctx.Err() == nil { + changesErr = err + } + cancel() + return + } + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } + }() + defer func() { + cancel() + <-changesDone + if changesErr != nil { + t.Errorf("event trigger failed: %v", changesErr) + } + }() req, err := http.NewRequestWithContext(ctx, http.MethodGet, cli.BaseURL+"/events?namespace="+ns, nil) if err != nil { t.Fatalf("new events request: %v", err) From 02df370369e40d4c25f39f490f7d33e2b4e56bad Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 23 Oct 2026 00:19:17 -0400 Subject: [PATCH 06/20] test: preserve controller updates in lifecycle fixtures Signed-off-by: Baylem --- .../controller/helpers_envtest_test.go | 18 ++++++++++++------ .../controller/networkcapture_envtest_test.go | 19 +++++++++++++------ .../restore_startup_cleanup_test.go | 4 ++-- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/operator/internal/controller/helpers_envtest_test.go b/operator/internal/controller/helpers_envtest_test.go index fbb6dccb0..eaf4cb6c5 100644 --- a/operator/internal/controller/helpers_envtest_test.go +++ b/operator/internal/controller/helpers_envtest_test.go @@ -571,12 +571,18 @@ func markBackupSucceeded(t *testing.T, ns, name, snapshotID, size string) { // reconciler waits for Suspended/Stopped before advancing to Running. func markGameServerPhase(t *testing.T, ns, name string, phase gameplanev1alpha1.GameServerPhase) { t.Helper() - var gs gameplanev1alpha1.GameServer - if err := k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: name}, &gs); err != nil { - t.Fatalf("get gameserver: %v", err) - } - gs.Status.Phase = phase - if err := k8sClient.Status().Update(context.Background(), &gs); err != nil { + // The Restore reconciler also writes the target's ownership annotations. + // Refetch after conflicts so this simulated status observation preserves + // concurrent controller writes without turning them into fixture failures. + ctx := t.Context() + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + var gs gameplanev1alpha1.GameServer + if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: name}, &gs); err != nil { + return err + } + gs.Status.Phase = phase + return k8sClient.Status().Update(ctx, &gs) + }); err != nil { t.Fatalf("status update gameserver: %v", err) } } diff --git a/operator/internal/controller/networkcapture_envtest_test.go b/operator/internal/controller/networkcapture_envtest_test.go index f7cf85007..5c5d2e12d 100644 --- a/operator/internal/controller/networkcapture_envtest_test.go +++ b/operator/internal/controller/networkcapture_envtest_test.go @@ -510,12 +510,19 @@ func TestNetworkCapture_UserStopCallsSidecar(t *testing.T) { // Emulate the API's StopNetworkCapture: patch status directly to // Completed with its exact message, bypassing the reconciler. - nc := getNetworkCapture(t, ns, "cap-stop-001") - now := metav1.NewTime(time.Now().UTC()) - nc.Status.Phase = gameplanev1alpha1.CapturePhaseCompleted - nc.Status.CompletionTime = &now - nc.Status.Message = "stopped by user request" - if err := k8sClient.Status().Update(context.Background(), nc); err != nil { + // The active reconciler may refresh capture progress between Get and Update. + ctx := t.Context() + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + var nc gameplanev1alpha1.NetworkCapture + if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: "cap-stop-001"}, &nc); err != nil { + return err + } + now := metav1.NewTime(time.Now().UTC()) + nc.Status.Phase = gameplanev1alpha1.CapturePhaseCompleted + nc.Status.CompletionTime = &now + nc.Status.Message = "stopped by user request" + return k8sClient.Status().Update(ctx, &nc) + }); err != nil { t.Fatalf("patch capture to user-stopped: %v", err) } diff --git a/operator/internal/controller/restore_startup_cleanup_test.go b/operator/internal/controller/restore_startup_cleanup_test.go index 5ac8e40e1..78da66458 100644 --- a/operator/internal/controller/restore_startup_cleanup_test.go +++ b/operator/internal/controller/restore_startup_cleanup_test.go @@ -51,7 +51,7 @@ func startupCleanupFixture(t *testing.T) (*RestoreReconciler, *gameplanev1alpha1 func TestRestoreStartupCleanupDrainsBeforeRelease(t *testing.T) { for _, outcome := range []string{"failed", "deleted", "succeeded"} { t.Run(outcome, func(t *testing.T) { - r, gs, rs, ss, pod := startupCleanupFixture(t) + r, _, rs, ss, pod := startupCleanupFixture(t) rs.Status.Phase = gameplanev1alpha1.RestorePhaseFailed if outcome == "succeeded" { rs.Status.Phase = gameplanev1alpha1.RestorePhaseSucceeded @@ -71,7 +71,7 @@ func TestRestoreStartupCleanupDrainsBeforeRelease(t *testing.T) { if err != nil { t.Fatal(err) } - gs = lifecycleServer(t, r) + gs := lifecycleServer(t, r) if outcome == "succeeded" { if gs.Spec.Suspend || gs.Annotations[lifecycleOperationAnnotation] != "" || res.RequeueAfter != 0 { t.Fatalf("successful startup was stopped: %+v %+v", gs, res) From ff4d4fcde0afa53352c764d955e4fb854716c86e Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 23 Oct 2026 00:19:17 -0400 Subject: [PATCH 07/20] test: synchronize closed RCON response fixtures Signed-off-by: Baylem --- agent/internal/rcon/nuclearoption_test.go | 29 +++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/agent/internal/rcon/nuclearoption_test.go b/agent/internal/rcon/nuclearoption_test.go index 248c6a7fb..9061d2c94 100644 --- a/agent/internal/rcon/nuclearoption_test.go +++ b/agent/internal/rcon/nuclearoption_test.go @@ -1275,18 +1275,47 @@ type pipeFrameCase struct { forbidErr string // substring the error must NOT contain (e.g. a detail suffix) } +type responseDeadlineConn struct { + net.Conn + deadlineSet chan<- struct{} +} + +func (c *responseDeadlineConn) SetDeadline(deadline time.Time) error { + if err := c.Conn.SetDeadline(deadline); err != nil { + return err + } + select { + case c.deadlineSet <- struct{}{}: + default: + } + return nil +} + // runPipeFrameCases sends each case's raw frame back to the client over net.Pipe // and checks the body or the error. func runPipeFrameCases(t *testing.T, tests []pipeFrameCase) { t.Helper() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + deadlineSet := make(chan struct{}, 1) client := newPipeClient(t, func(remote net.Conn) { if _, _, err := readNuclearOptionRequest(remote); err != nil { return } + if len(tt.frame) == 0 { + // Exec sets its response deadline after sending the request. + // Closing first can fail SetDeadline instead of the status read. + select { + case <-deadlineSet: + case <-t.Context().Done(): + return + } + } writeFrameSegments(remote, tt.frame, tt.oneByte) }) + if len(tt.frame) == 0 { + client.conn = &responseDeadlineConn{Conn: client.conn, deadlineSet: deadlineSet} + } body, err := client.Exec("test-command") if tt.wantErr != "" { From 11e7d65f1be0e632e9ff4788664fdd78bc5e9010 Mon Sep 17 00:00:00 2001 From: Baylem Date: Sat, 24 Oct 2026 11:00:07 -0400 Subject: [PATCH 08/20] fix: allow guarded wipe recovery to drain workloads Signed-off-by: Baylem --- .../controller/gameserver_controller.go | 10 + .../gameserver_lifecycle_reconcile_test.go | 344 ++++++++++++++++++ .../internal/controller/gameserver_wipe.go | 10 +- .../controller/gameserver_wipe_guard.go | 23 +- operator/specs.md | 10 +- 5 files changed, 390 insertions(+), 7 deletions(-) create mode 100644 operator/internal/controller/gameserver_lifecycle_reconcile_test.go diff --git a/operator/internal/controller/gameserver_controller.go b/operator/internal/controller/gameserver_controller.go index 0821a6e86..9bc7aa929 100644 --- a/operator/internal/controller/gameserver_controller.go +++ b/operator/internal/controller/gameserver_controller.go @@ -38,6 +38,10 @@ import ( gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" ) +// Lifecycle annotation conflicts need a fresh reconcile, not an error backoff. +// Give concurrent writers and the informer cache a bounded interval to settle. +const lifecycleConflictRetry = time.Second + // GameServerReconciler reconciles a GameServer object into a StatefulSet, // Service, and PVC. The agent sidecar is injected at pod-spec build time. type GameServerReconciler struct { @@ -336,6 +340,9 @@ func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) // single status patch further down. idle, idleStatus, idleRequeue, idleScheduleErr, err := r.reconcileIdle(ctx, &gs) if err != nil { + if apierrors.IsConflict(err) { + return ctrl.Result{RequeueAfter: lifecycleConflictRetry}, nil + } logger.Error(err, "reconcile idle") return ctrl.Result{}, err } @@ -385,6 +392,9 @@ func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) replicas, stopRequeue, err := r.desiredReplicas(ctx, &gs, &tmpl, idle, &wipeState) if err != nil { + if apierrors.IsConflict(err) { + return ctrl.Result{RequeueAfter: lifecycleConflictRetry}, nil + } logger.Error(err, "compute desired replicas") return ctrl.Result{}, err } diff --git a/operator/internal/controller/gameserver_lifecycle_reconcile_test.go b/operator/internal/controller/gameserver_lifecycle_reconcile_test.go new file mode 100644 index 000000000..695215f5b --- /dev/null +++ b/operator/internal/controller/gameserver_lifecycle_reconcile_test.go @@ -0,0 +1,344 @@ +package controller + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "reflect" + "testing" + "time" + + "github.com/go-logr/logr" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/util/cert" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/log" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +func lifecycleGameServerFixture(t *testing.T, gs *gameplanev1alpha1.GameServer, extras ...client.Object) *GameServerReconciler { + t.Helper() + s := wipeScheme(t) + if err := clientgoscheme.AddToScheme(s); err != nil { + t.Fatal(err) + } + tmpl := &gameplanev1alpha1.GameTemplate{ObjectMeta: metav1.ObjectMeta{Name: gs.Spec.TemplateRef.Name}, Spec: gameplanev1alpha1.GameTemplateSpec{Image: "example.invalid/game:unit"}} + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatal(err) + } + ca, err := cert.NewSelfSignedCACert(cert.Config{CommonName: "lifecycle-reconcile-test"}, key) + if err != nil { + t.Fatal(err) + } + caSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "agent-ca", Namespace: gs.Namespace}, Data: map[string][]byte{ + "ca.crt": pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: ca.Raw}), + "ca.key": pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}), + }} + objects := append([]client.Object{gs, tmpl, caSecret}, extras...) + c := fake.NewClientBuilder().WithScheme(s).WithObjects(objects...). + WithStatusSubresource(gs, &appsv1.StatefulSet{}, &batchv1.Job{}).Build() + return &GameServerReconciler{Client: c, APIReader: c, Scheme: s, AgentCASecretName: caSecret.Name, AgentCASecretNamespace: caSecret.Namespace} +} + +func requireNoWipeJob(t *testing.T, r *GameServerReconciler, gs *gameplanev1alpha1.GameServer) { + t.Helper() + var job batchv1.Job + if err := r.Get(t.Context(), client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-wipe"}, &job); !apierrors.IsNotFound(err) { + t.Fatalf("wipe launched before recovery completed: %v", err) + } +} + +func TestGameServerWipeReconcileDrainsOrphanRestoreStartup(t *testing.T) { + gs := wipeGameServer(false, "wipe") + predecessor := lifecycleIdentity{Kind: "Restore", Name: "deleted", UID: "deleted-restore", TargetUID: gs.UID, InitialStartup: true} + gs.Annotations[lifecycleOperationAnnotation] = lifecycleOperationJSON(lifecycleOperation{lifecycleIdentity: predecessor}) + gs.Annotations[restoreGuardAnnotation] = restoreIdentityJSON(predecessor.Name, predecessor.UID) + ss := wipeStoppedSet(gs) + one := int32(1) + ss.Spec.Replicas, ss.Status.Replicas = &one, 1 + ss.Spec.Template.Spec.Containers = []corev1.Container{{Name: "game", Image: "restored-image"}} + originalTemplate := ss.Spec.Template.DeepCopy() + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: gs.Name + "-0", Namespace: gs.Namespace, UID: "startup", Finalizers: []string{"test/hold"}, + OwnerReferences: []metav1.OwnerReference{{APIVersion: "apps/v1", Kind: "StatefulSet", Name: ss.Name, UID: ss.UID, Controller: ownerBoolPtr(true)}}}, Status: corev1.PodStatus{Phase: corev1.PodRunning}} + r := lifecycleGameServerFixture(t, gs, ss, pod) + request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(gs)} + // Admission persists the successor. The next public pass must reach the + // StatefulSet controller instead of waiting for its own scale-down. + for range 2 { + if _, err := r.Reconcile(t.Context(), request); err != nil { + t.Fatal(err) + } + } + current := wipeCurrent(t, r.Client, gs) + op, held, err := lifecycleOwner(current) + if err != nil || !held || op.Kind != "Wipe" || len(op.Recovery) != 1 || op.Recovery[0].UID != predecessor.UID || !current.Spec.Suspend { + t.Fatalf("orphan recovery lost its identity or exclusion: %+v %v", current, err) + } + if err := r.Get(t.Context(), client.ObjectKeyFromObject(ss), ss); err != nil { + t.Fatal(err) + } + if ss.Spec.Replicas == nil || *ss.Spec.Replicas != 0 || !reflect.DeepEqual(ss.Spec.Template, *originalTemplate) { + t.Fatalf("recovery did not scale down with the original template: %+v", ss.Spec) + } + requireNoWipeJob(t, r, gs) + ss.Status.Replicas, ss.Status.ObservedGeneration = 0, ss.Generation + if err := r.Status().Update(t.Context(), ss); err != nil { + t.Fatal(err) + } + if err := r.Delete(t.Context(), pod); err != nil { + t.Fatal(err) + } + if _, err := r.Reconcile(t.Context(), request); err != nil { + t.Fatal(err) + } + requireNoWipeJob(t, r, gs) + current = wipeCurrent(t, r.Client, gs) + op, held, err = lifecycleOwner(current) + if err != nil || !held || len(op.Recovery) != 1 { + t.Fatalf("terminating startup pod lost the recovery fence: %+v %v", current, err) + } + if err := r.Get(t.Context(), client.ObjectKeyFromObject(pod), pod); err != nil { + t.Fatal(err) + } + pod.Finalizers = nil + if err := r.Update(t.Context(), pod); err != nil { + t.Fatal(err) + } + for range 3 { + if _, err := r.Reconcile(t.Context(), request); err != nil { + t.Fatal(err) + } + } + var job batchv1.Job + if err := r.Get(t.Context(), client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-wipe"}, &job); err != nil { + t.Fatalf("drained predecessor did not permit wipe: %v", err) + } + current = wipeCurrent(t, r.Client, gs) + op, held, err = lifecycleOwner(current) + if err != nil || !held || op.Kind != "Wipe" || len(op.Recovery) != 0 || !current.Spec.Suspend || job.Labels[wipeTokenLabel] != "wipe" { + t.Fatalf("wipe launch lost exclusive ownership: %+v %v", current, err) + } +} + +func TestGameServerWipeReconcileWaitsForPredecessorCleanup(t *testing.T) { + gs := wipeGameServer(true, "wipe") + gs.Annotations[wipeGuardAnnotation] = "wipe" + predecessor := lifecycleIdentity{Kind: "Backup", Name: "deleted", UID: "deleted-backup", TargetUID: gs.UID, NeedsUnquiesce: true} + gs.Annotations[lifecycleOperationAnnotation] = lifecycleOperationJSON(lifecycleOperation{lifecycleIdentity: wipeLifecycleIdentity(gs, "wipe"), Recovery: []lifecycleIdentity{predecessor}}) + // A labelled worker still counts even if its Job and PVC mount metadata + // are gone. A stopped target alone does not complete predecessor cleanup. + worker := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "orphan-copy", Namespace: gs.Namespace, Labels: map[string]string{backupUIDLabel: string(predecessor.UID)}}, Status: corev1.PodStatus{Phase: corev1.PodRunning}} + gamePod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: gs.Name + "-0", Namespace: gs.Namespace, UID: "original-game"}, Status: corev1.PodStatus{Phase: corev1.PodRunning}} + r := lifecycleGameServerFixture(t, gs, wipeStoppedSet(gs), worker, gamePod) + cleanupErr := errors.New("save-on unavailable") + quiescer := &scrapeQuiescer{unquiesceErr: cleanupErr} + r.OperationQuiescer = quiescer + request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(gs)} + if _, err := r.Reconcile(t.Context(), request); err != nil { + t.Fatal(err) + } + requireNoWipeJob(t, r, gs) + if err := r.Delete(t.Context(), worker); err != nil { + t.Fatal(err) + } + if _, err := r.Reconcile(t.Context(), request); !errors.Is(err, cleanupErr) { + t.Fatalf("unfinished save-on did not retain its error: %v", err) + } + requireNoWipeJob(t, r, gs) + current := wipeCurrent(t, r.Client, gs) + op, held, err := lifecycleOwner(current) + if err != nil || !held || len(op.Recovery) != 1 || !current.Spec.Suspend { + t.Fatalf("unfinished save-on lost its recovery fence: %+v %v", current, err) + } + quiescer.unquiesceErr = nil + if _, err := r.Reconcile(t.Context(), request); err != nil { + t.Fatal(err) + } + requireNoWipeJob(t, r, gs) + if err := r.Delete(t.Context(), gamePod); err != nil { + t.Fatal(err) + } + for range 2 { + if _, err := r.Reconcile(t.Context(), request); err != nil { + t.Fatal(err) + } + } + var job batchv1.Job + if err := r.Get(t.Context(), client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-wipe"}, &job); err != nil { + t.Fatalf("completed save-on did not permit wipe: %v", err) + } + if quiescer.unquiesced != 1 { + t.Fatalf("save-on completed %d times, want once", quiescer.unquiesced) + } +} + +func TestWipeRecoveryRetainsCleanupFences(t *testing.T) { + for _, mode := range []string{"cancel", "replace", "success"} { + t.Run(mode, func(t *testing.T) { + gs := wipeGameServer(true, "wipe") + gs.Annotations[wipeGuardAnnotation] = "wipe" + predecessor := lifecycleIdentity{Kind: "Backup", Name: "deleted", UID: "deleted-backup", TargetUID: gs.UID} + gs.Annotations[lifecycleOperationAnnotation] = lifecycleOperationJSON(lifecycleOperation{lifecycleIdentity: wipeLifecycleIdentity(gs, "wipe"), Recovery: []lifecycleIdentity{predecessor}}) + ss := wipeStoppedSet(gs) + worker := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "orphan-copy", Namespace: gs.Namespace, Labels: map[string]string{backupUIDLabel: string(predecessor.UID)}}, Status: corev1.PodStatus{Phase: corev1.PodRunning}} + objects := []client.Object{gs, ss, worker} + switch mode { + case "cancel": + delete(gs.Annotations, WipeRequestedAnnotation) + case "replace": + gs.Annotations[WipeRequestedAnnotation] = "replacement" + case "success": + gs.Annotations[wipeSuccessAnnotation] = "wipe" + objects = append(objects, &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: gs.Name + "-wipe", Namespace: gs.Namespace, Labels: map[string]string{wipeTokenLabel: "wipe"}, + OwnerReferences: []metav1.OwnerReference{{APIVersion: gameplanev1alpha1.GroupVersion.String(), Kind: "GameServer", Name: gs.Name, UID: gs.UID, Controller: ownerBoolPtr(true)}}}, Status: batchv1.JobStatus{Succeeded: 1}}) + } + s := wipeScheme(t) + c := fake.NewClientBuilder().WithScheme(s).WithObjects(objects...).Build() + r := &GameServerReconciler{Client: c, APIReader: c, Scheme: s} + if err := r.reconcileWipe(t.Context(), gs, &gameplanev1alpha1.GameTemplate{}); err != nil { + t.Fatal(err) + } + current := wipeCurrent(t, c, gs) + op, held, err := lifecycleOwner(current) + if err != nil || !held || op.Token != "wipe" || len(op.Recovery) != 1 || current.Annotations[wipeGuardAnnotation] != "wipe" || current.Annotations[WipeCompletedAnnotation] != "" || !current.Spec.Suspend { + t.Fatalf("cleanup changed unfinished recovery: %+v %v", current, err) + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(ss), ss); err != nil || ss.Annotations[wipeGuardAnnotation] != "wipe" { + t.Fatalf("cleanup removed the workload fence: %+v %v", ss, err) + } + if mode == "success" { + var job batchv1.Job + if err := c.Get(t.Context(), client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-wipe"}, &job); err != nil || !job.DeletionTimestamp.IsZero() { + t.Fatalf("success cleanup ran before predecessor drained: %+v %v", job, err) + } + } else { + requireNoWipeJob(t, r, gs) + } + }) + } +} + +func TestWipeLaunchRevalidatesDurableOwnerAfterStopProof(t *testing.T) { + for _, mode := range []string{"recovery", "other owner", "missing owner", "malformed owner"} { + t.Run(mode, func(t *testing.T) { + gs := wipeGameServer(true, "wipe") + gs.Annotations[wipeGuardAnnotation] = "wipe" + gs.Annotations[lifecycleOperationAnnotation] = lifecycleOperationJSON(lifecycleOperation{lifecycleIdentity: wipeLifecycleIdentity(gs, "wipe")}) + s := wipeScheme(t) + c := fake.NewClientBuilder().WithScheme(s).WithObjects(gs, wipeStoppedSet(gs)).Build() + observedWorkload, changedOwner := false, false + reader := interceptor.NewClient(c, interceptor.Funcs{Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if err := cl.Get(ctx, key, obj, opts...); err != nil { + return err + } + if _, ok := obj.(*appsv1.StatefulSet); ok { + observedWorkload = true + } + live, ok := obj.(*gameplanev1alpha1.GameServer) + if !ok || !observedWorkload || changedOwner { + return nil + } + changedOwner = true + op := lifecycleOperation{lifecycleIdentity: wipeLifecycleIdentity(live, "wipe")} + switch mode { + case "recovery": + op.Recovery = []lifecycleIdentity{{Kind: "Backup", Name: "predecessor", UID: "predecessor", TargetUID: gs.UID, NeedsUnquiesce: true}} + case "other owner": + op.lifecycleIdentity = lifecycleIdentity{Kind: "Restore", Name: "replacement", UID: "replacement", TargetUID: gs.UID} + } + live.Annotations[lifecycleOperationAnnotation] = lifecycleOperationJSON(op) + if mode == "missing owner" { + delete(live.Annotations, lifecycleOperationAnnotation) + } else if mode == "malformed owner" { + live.Annotations[lifecycleOperationAnnotation] = "invalid" + } + return cl.Update(ctx, live) + }}) + r := &GameServerReconciler{Client: c, APIReader: reader, Scheme: s} + err := r.reconcileWipe(t.Context(), gs, &gameplanev1alpha1.GameTemplate{}) + if mode == "malformed owner" { + if err == nil { + t.Fatal("malformed owner did not fail closed") + } + } else if err != nil { + t.Fatal(err) + } + if !changedOwner { + t.Fatal("did not reach the live owner read after stop proof") + } + requireNoWipeJob(t, r, gs) + }) + } +} + +type lifecycleErrorLog struct{ errors *[]error } + +func (*lifecycleErrorLog) Init(logr.RuntimeInfo) {} +func (*lifecycleErrorLog) Enabled(int) bool { return true } +func (*lifecycleErrorLog) Info(int, string, ...any) {} +func (l *lifecycleErrorLog) WithValues(...any) logr.LogSink { return l } +func (l *lifecycleErrorLog) WithName(string) logr.LogSink { return l } +func (l *lifecycleErrorLog) Error(err error, _ string, _ ...any) { *l.errors = append(*l.errors, err) } + +func TestGameServerLifecyclePatchConflictsRetryQuietly(t *testing.T) { + for _, transition := range []string{"idle", "restart"} { + for _, conflict := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/conflict=%t", transition, conflict), func(t *testing.T) { + gs := wipeGameServer(false, "") + if transition == "idle" { + // Disabling idle while asleep requests an annotation clear. + gs.Annotations[IdleAsleepSinceAnnotation] = time.Now().Add(-time.Hour).UTC().Format(time.RFC3339) + } else { + gs.Annotations[RestartRequestedAnnotation] = "restart" + } + r := lifecycleGameServerFixture(t, gs) + cause := errors.New("API unavailable") + var patchErr error = cause + if conflict { + patchErr = apierrors.NewConflict(schema.GroupResource{Group: "gameplane.local", Resource: "gameservers"}, gs.Name, cause) + } + patchErr = fmt.Errorf("write response: %w", patchErr) + patches := 0 + r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if _, ok := obj.(*gameplanev1alpha1.GameServer); ok { + patches++ + return patchErr + } + return c.Patch(ctx, obj, patch, opts...) + }}) + var logged []error + ctx := log.IntoContext(t.Context(), logr.New(&lifecycleErrorLog{errors: &logged})) + result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(gs)}) + if patches != 1 { + t.Fatalf("lifecycle patch was not reached exactly once: %d", patches) + } + if conflict { + if err != nil || result != (ctrl.Result{RequeueAfter: time.Second}) || len(logged) != 0 { + t.Fatalf("expected quiet one-second conflict retry: result=%+v error=%v logs=%v", result, err, logged) + } + } else if !errors.Is(err, cause) || result != (ctrl.Result{}) || len(logged) != 1 || !errors.Is(logged[0], cause) { + t.Fatalf("real error lost its identity/log/backoff: result=%+v error=%v logs=%v", result, err, logged) + } + var ss appsv1.StatefulSet + if err := r.Get(t.Context(), client.ObjectKeyFromObject(gs), &ss); !apierrors.IsNotFound(err) { + t.Fatalf("failed lifecycle patch still mutated the workload: %v", err) + } + }) + } + } +} diff --git a/operator/internal/controller/gameserver_wipe.go b/operator/internal/controller/gameserver_wipe.go index c59ecf998..e3414336a 100644 --- a/operator/internal/controller/gameserver_wipe.go +++ b/operator/internal/controller/gameserver_wipe.go @@ -74,6 +74,11 @@ func (r *GameServerReconciler) reconcileWipe( if changed { return nil } + if guard != "" { + if ready, err := wipeLifecycleReady(gs, guard); err != nil || !ready { + return err + } + } if guard == "" { if len(state.jobs) == 0 { return nil @@ -137,9 +142,12 @@ func (r *GameServerReconciler) reconcileWipe( if err := r.wipeReader().Get(ctx, client.ObjectKeyFromObject(gs), &live); err != nil { return client.IgnoreNotFound(err) } - if live.UID != gs.UID || live.Annotations[wipeGuardAnnotation] != guard || live.Annotations[WipeRequestedAnnotation] != guard || !pendingWipe(&live) || live.Annotations[restoreGuardAnnotation] != "" { + if live.UID != gs.UID || live.Annotations[wipeGuardAnnotation] != guard || live.Annotations[WipeRequestedAnnotation] != guard || !pendingWipe(&live) || live.Annotations[restoreGuardAnnotation] != "" || live.Annotations[backupGuardAnnotation] != "" { return nil } + if ready, err := wipeLifecycleReady(&live, guard); err != nil || !ready { + return err + } return r.createWipeJob(ctx, &live, tmpl, jobName, guard) } diff --git a/operator/internal/controller/gameserver_wipe_guard.go b/operator/internal/controller/gameserver_wipe_guard.go index bdf00bfc8..45a6becc4 100644 --- a/operator/internal/controller/gameserver_wipe_guard.go +++ b/operator/internal/controller/gameserver_wipe_guard.go @@ -166,10 +166,11 @@ func (r *GameServerReconciler) ensureWipeGuard(ctx context.Context, gs *gameplan } *gs = live if token := gs.Annotations[wipeGuardAnnotation]; token != "" { - ready, err := claimLifecycleOperation(ctx, r.Client, r.wipeReader(), gs, wipeLifecycleIdentity(gs, token), r.OperationQuiescer) - // Upgrading an existing guard does not require another pass. Only - // unfinished predecessor cleanup must prevent this wipe proceeding. - return err == nil && !ready, err + _, err := claimLifecycleOperation(ctx, r.Client, r.wipeReader(), gs, wipeLifecycleIdentity(gs, token), r.OperationQuiescer) + // Existing ownership must not skip workload reconciliation: orphaned + // Restore recovery may need it to scale down the initial startup pod. + // reconcileWipe separately blocks cleanup and launch until recovery ends. + return false, err } state := wipeInventory(inventory) if err := r.loadWipeWorkers(ctx, gs, state); err != nil { @@ -196,6 +197,20 @@ func (r *GameServerReconciler) ensureWipeGuard(ctx context.Context, gs *gameplan return err == nil && gs.Annotations[wipeGuardAnnotation] == token, err } +// wipeLifecycleReady requires the durable owner, not just its compatibility +// guard. Pending predecessors still own cleanup obligations and cannot be +// discarded by wipe completion, cancellation, replacement, or a new worker. +func wipeLifecycleReady(gs *gameplanev1alpha1.GameServer, token string) (bool, error) { + if gs.Annotations[lifecycleOperationAnnotation] == "" { + return false, nil + } + op, held, err := lifecycleOwner(gs) + if err != nil { + return false, err + } + return held && sameLifecycleOwner(op.lifecycleIdentity, wipeLifecycleIdentity(gs, token)) && len(op.Recovery) == 0, nil +} + // Stamp the owned workload with a checked write before workers launch. A stale // workload update then conflicts, while callbacks reading the fence stay at 0. func (r *GameServerReconciler) wipeTargetStopped(ctx context.Context, gs *gameplanev1alpha1.GameServer) (bool, error) { diff --git a/operator/specs.md b/operator/specs.md index 37d215b4a..5af0c5346 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -150,7 +150,7 @@ Primary reconcilers register with the manager in `cmd/main.go` and handle CRD li - Node affinity: honor spec.nodeSelector and spec.affinity preferences. - Ingress NetworkPolicy: enforce per-template advertised ports, admit CIDRs from `--game-ingress-from-cidr`. - Fixed-name child cleanup: Delete fixed-name objects (`-config` Secret, `-files` Secret, `-rcon` Secret, `-auto` BackupSchedule, `-wipe` Job) only when the GameServer controls them via OwnerReference; unowned objects with these names stay in place (same rule as game-ingress NetworkPolicy). Wipe cleanup carries the verified Job UID as a delete precondition; a foreign Job cannot acknowledge a pending wipe or resume its server. - - **Conflict requeue:** When reconcileStatefulSet or reconcileWipe encounters an apierrors.IsConflict error (concurrent update races), requeue without error logging — conflicts are expected and transient. The Module, NetworkCapture and Backup reconcilers apply the same rule to every error they return, via `requeueOnConflict` (`conflict.go`). + - **Conflict requeue:** When reconcileStatefulSet or reconcileWipe encounters an apierrors.IsConflict error (concurrent update races), requeue without error logging — conflicts are expected and transient. Idle-transition and restart-acknowledgment conflicts, including wrapped errors, also return before logging and retry after one second with fresh state. Other errors retain their identity, error logging and controller-runtime backoff. The Module, NetworkCapture and Backup reconcilers apply the conflict rule to every error they return, via `requeueOnConflict` (`conflict.go`). - Load-balancer address management: translate spec.networking.addressPool / .address preferences onto the Service based on the cluster's address-manager flavor (MetalLB, Cilium, or none), report assignment status via the AddressAssignment condition. - **Network capture foundation** (**Phase 2 Foundational**): Pre-provision a `captures` emptyDir (1 GiB) unconditionally on every game pod's StatefulSet template (required because pod.spec.volumes is immutable on running pods; a rolling restart of all existing game pods is incurred once on upgrade). Extend the `-agent` Service with a second numeric ServicePort 9091 for the capture sidecar's control endpoint. RBAC markers grant `pods/ephemeralcontainers` access (get, list, watch, patch, update) and full CRUD on `networkcaptures` resources. The capture sidecar injection as an ephemeral container is implemented: `buildCaptureEphemeralContainer` (`gameserver_controller.go`) is the single source of truth for the container spec, injected eagerly by `GameServerReconciler.reconcileCapture` when `spec.capture.enabled` is set, with `NetworkCaptureReconciler.injectCaptureContainer` as an idempotent fallback for the case where its own cache is ahead of the GameServer reconciler's write. - **Split concerns:** @@ -278,7 +278,13 @@ lineage is retained on the GameServer across repeated force deletion. Owned Jobs are canceled with UID preconditions and foreground propagation; independently inventoried live or terminating worker pods and pending CSI captures retain the guard. Saving is restored only after recovery workers drain. The workload remains -frozen during recovery even when the successor is a Restore or wipe. Terminal +frozen during recovery even when the successor is a Restore or wipe, except +that an orphaned Restore's initial workload must scale to zero with its template +preserved. An existing wipe guard does not skip this workload reconciliation. +Wipe cancellation, replacement, success cleanup and worker creation wait until +all predecessor workers and quiesce cleanup finish. Immediately before worker +creation, an uncached read must still show the same durable wipe owner with no +pending recovery; the compatibility guard alone cannot authorize a launch. Terminal state and existing cleanup intent are preserved before releasing the owner; failed/canceled restores remain suspended. Generation changes to active Backup or Restore specs fail the operation and enter cleanup rather than authorizing a From d9f3f8e2e4443bfc10bb57e8600cd6bf718188fc Mon Sep 17 00:00:00 2001 From: Baylem Date: Sun, 25 Oct 2026 19:05:58 -0400 Subject: [PATCH 09/20] test: isolate stopped backup recovery fixtures Signed-off-by: Baylem --- test/e2e/backup_stopped_e2e_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/e2e/backup_stopped_e2e_test.go b/test/e2e/backup_stopped_e2e_test.go index d469a635d..e02258db2 100644 --- a/test/e2e/backup_stopped_e2e_test.go +++ b/test/e2e/backup_stopped_e2e_test.go @@ -21,7 +21,10 @@ import ( func TestBackup_StoppedCopiesRemainRestorable(t *testing.T) { t.Parallel() ctx := context.Background() - ns, gs, tmpl := "gameplane-games", "e2e-stopped-backup", "e2e-stopped-backup-template" + suffix := time.Now().UnixNano() + ns := "gameplane-games" + gs := fmt.Sprintf("e2e-stopped-backup-%d", suffix) + tmpl := fmt.Sprintf("e2e-stopped-backup-template-%d", suffix) ensureResticRepo(t) applyBusyboxTemplate(t, tmpl) applyBusyboxGameServer(t, ns, gs, tmpl) @@ -56,7 +59,7 @@ func TestBackup_StoppedCopiesRemainRestorable(t *testing.T) { return apierrors.IsNotFound(err), fmt.Sprintf("pod absence: %v", err) }) for i := 1; i <= 2; i++ { - name := fmt.Sprintf("e2e-stopped-copy-%d", i) + name := fmt.Sprintf("e2e-stopped-copy-%d-%d", suffix, i) backup := &unstructured.Unstructured{Object: map[string]any{ "apiVersion": "gameplane.local/v1alpha1", "kind": "Backup", "metadata": map[string]any{"name": name, "namespace": ns}, From 1842bdaaa9b165bcd9f184d6c549f7324b48ac00 Mon Sep 17 00:00:00 2001 From: Baylem Date: Sun, 25 Oct 2026 19:16:53 -0400 Subject: [PATCH 10/20] test: simplify lifecycle regression setup Signed-off-by: Baylem --- .../controller/gameserver_lifecycle_reconcile_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/operator/internal/controller/gameserver_lifecycle_reconcile_test.go b/operator/internal/controller/gameserver_lifecycle_reconcile_test.go index 695215f5b..a74724a8f 100644 --- a/operator/internal/controller/gameserver_lifecycle_reconcile_test.go +++ b/operator/internal/controller/gameserver_lifecycle_reconcile_test.go @@ -262,9 +262,10 @@ func TestWipeLaunchRevalidatesDurableOwnerAfterStopProof(t *testing.T) { op.lifecycleIdentity = lifecycleIdentity{Kind: "Restore", Name: "replacement", UID: "replacement", TargetUID: gs.UID} } live.Annotations[lifecycleOperationAnnotation] = lifecycleOperationJSON(op) - if mode == "missing owner" { + switch mode { + case "missing owner": delete(live.Annotations, lifecycleOperationAnnotation) - } else if mode == "malformed owner" { + case "malformed owner": live.Annotations[lifecycleOperationAnnotation] = "invalid" } return cl.Update(ctx, live) @@ -308,7 +309,7 @@ func TestGameServerLifecyclePatchConflictsRetryQuietly(t *testing.T) { } r := lifecycleGameServerFixture(t, gs) cause := errors.New("API unavailable") - var patchErr error = cause + patchErr := cause if conflict { patchErr = apierrors.NewConflict(schema.GroupResource{Group: "gameplane.local", Resource: "gameservers"}, gs.Name, cause) } From f209797bb909b859a8d243c5815897a992acf92c Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 4 Nov 2026 22:30:38 -0500 Subject: [PATCH 11/20] fix(ci): install pinned Trivy from GitHub releases Signed-off-by: Baylem --- .github/actions/install-trivy/action.yaml | 27 +++++++++++++++++++++++ .github/workflows/ci.yaml | 4 +--- .github/workflows/publish-edge.yaml | 5 ++--- .github/workflows/release.yaml | 4 +--- 4 files changed, 31 insertions(+), 9 deletions(-) create mode 100644 .github/actions/install-trivy/action.yaml diff --git a/.github/actions/install-trivy/action.yaml b/.github/actions/install-trivy/action.yaml new file mode 100644 index 000000000..7fe53b816 --- /dev/null +++ b/.github/actions/install-trivy/action.yaml @@ -0,0 +1,27 @@ +name: Install pinned Trivy +description: Install the checksum-verified Linux amd64 scanner from its GitHub release. +runs: + using: composite + steps: + - name: Download and verify Trivy + shell: bash + run: | + set -euo pipefail + if [[ "$(uname -s)" != Linux || "$(uname -m)" != x86_64 ]]; then + echo "This installer requires a Linux amd64 runner" >&2 + exit 1 + fi + version=0.74.0 + sha256=2ae6fe3ee734b7fdf11335663e18c75ea12dccc76062f09f164a3b0f8be4371a + archive="trivy_${version}_Linux-64bit.tar.gz" + download_dir=$(mktemp -d "${RUNNER_TEMP:?}/trivy-download.XXXXXX") + trap 'rm -rf -- "$download_dir"' EXIT + # Bound DNS, connection and HTTP retries. Verify before extraction or execution. + curl --fail --show-error --location --retry 3 --retry-all-errors \ + --retry-delay 2 --retry-max-time 180 --connect-timeout 15 --max-time 90 \ + --output "$download_dir/$archive" \ + "https://github.com/aquasecurity/trivy/releases/download/v${version}/${archive}" + printf '%s %s\n' "$sha256" "$download_dir/$archive" | sha256sum --check --status + tar -xzf "$download_dir/$archive" -C "$download_dir" trivy + sudo install -m 0755 "$download_dir/trivy" /usr/local/bin/trivy + trivy --version diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index be4921ce3..381cc5fdc 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -624,9 +624,7 @@ jobs: path: ~/.cache/trivy key: trivy-db-${{ steps.week.outputs.week }} - name: install trivy - uses: aquasecurity/setup-trivy@81e514348e19b6112ce2a7e3ecbafe19c1e1f567 # v0.3.1 - with: - version: v0.74.0 + uses: ./.github/actions/install-trivy - name: prove the gate rejects a vulnerable image run: ./hack/check-image-scan_test.sh - name: scan every e2e image diff --git a/.github/workflows/publish-edge.yaml b/.github/workflows/publish-edge.yaml index f91e9d12d..eadbead8a 100644 --- a/.github/workflows/publish-edge.yaml +++ b/.github/workflows/publish-edge.yaml @@ -43,6 +43,7 @@ on: - "**/Dockerfile" - "**/Dockerfile.*" - ".github/workflows/publish-edge.yaml" + - ".github/actions/install-trivy/**" - "hack/ci_scope.py" - ".dockerignore" - "signing/cosign.pub" @@ -149,9 +150,7 @@ jobs: # Spec 013 G2: never sign an image that has a fixed HIGH/CRITICAL # vulnerability. Same flags as the `vuln (images)` CI gate. - name: install trivy - uses: aquasecurity/setup-trivy@81e514348e19b6112ce2a7e3ecbafe19c1e1f567 # v0.3.1 - with: - version: v0.74.0 + uses: ./.github/actions/install-trivy - name: scan pushed image env: IMAGE: ${{ steps.image.outputs.name }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ba7f864bb..85d9626bb 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -92,9 +92,7 @@ jobs: # Spec 013 G2: never sign an image that has a fixed HIGH/CRITICAL # vulnerability. Same flags as the `vuln (images)` CI gate. - name: install trivy - uses: aquasecurity/setup-trivy@81e514348e19b6112ce2a7e3ecbafe19c1e1f567 # v0.3.1 - with: - version: v0.74.0 + uses: ./.github/actions/install-trivy - name: scan pushed image env: IMAGE: ${{ steps.image.outputs.name }} From 28add9582995a5a45c87735d45302caed0fa5b53 Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 6 Nov 2026 02:07:40 -0500 Subject: [PATCH 12/20] test: cover disabled-agent cleanup with a pod identity Signed-off-by: Baylem --- operator/internal/controller/backup_stopped_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/operator/internal/controller/backup_stopped_test.go b/operator/internal/controller/backup_stopped_test.go index 3d0f30f39..894725004 100644 --- a/operator/internal/controller/backup_stopped_test.go +++ b/operator/internal/controller/backup_stopped_test.go @@ -259,9 +259,12 @@ func TestBackupDisabledAgentPreservesRawCopyWithoutClaimingQuiesce(t *testing.T) if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "first"}, &job); err != nil { t.Fatalf("disabled client prevented raw copy: %v", err) } - if err := callBackupQuiesce(t.Context(), r.AgentClient, "ns", "server", "", true); err == nil { + if err := callBackupQuiesce(t.Context(), r.AgentClient, "ns", "server", "game-pod-uid", true); err == nil { t.Fatal("disabled client cleared an existing cleanup obligation") } + if err := callBackupQuiesce(t.Context(), r.AgentClient, "ns", "server", "", true); err == nil { + t.Fatal("legacy cleanup sent an unbound save-on") + } } func TestBackupDisabledAgentCannotBypassPendingLiveQuiesce(t *testing.T) { From 6045280593ec081a5950eb145a5f763dfaac1c58 Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 6 Nov 2026 02:13:28 -0500 Subject: [PATCH 13/20] fix(deps): update compress for GO-2026-6865 Select github.com/klauspost/compress v1.20.2 in all six affected standalone module graphs, including transitive consumers, and refresh their checksums. Signed-off-by: Baylem --- agent/go.mod | 1 + agent/go.sum | 4 ++-- api/go.mod | 2 +- api/go.sum | 4 ++-- mcp-server/go.mod | 2 +- mcp-server/go.sum | 4 ++-- operator/go.mod | 2 +- operator/go.sum | 4 ++-- telemetry-receiver/go.mod | 1 + telemetry-receiver/go.sum | 4 ++-- tunnel/go.mod | 2 +- tunnel/go.sum | 4 ++-- 12 files changed, 18 insertions(+), 16 deletions(-) diff --git a/agent/go.mod b/agent/go.mod index 35bc6ba66..426be7f19 100644 --- a/agent/go.mod +++ b/agent/go.mod @@ -43,6 +43,7 @@ require ( github.com/go-openapi/swag/yamlutils v0.28.0 // indirect github.com/google/gnostic-models v0.7.1 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.20.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect diff --git a/agent/go.sum b/agent/go.sum index d2a15f084..d98b2a11e 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -59,8 +59,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ= -github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/klauspost/compress v1.20.2 h1:v5tiZvm1KGLyg5d5oXdXe3lsSPrbEUiKoXGia//I7Qo= +github.com/klauspost/compress v1.20.2/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= diff --git a/api/go.mod b/api/go.mod index 4d5083c09..21e3fc1de 100644 --- a/api/go.mod +++ b/api/go.mod @@ -86,7 +86,7 @@ require ( github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.20.1 // indirect + github.com/klauspost/compress v1.20.2 // indirect github.com/klauspost/cpuid/v2 v2.4.0 // indirect github.com/klauspost/crc32 v1.3.0 // indirect github.com/mattn/go-isatty v0.0.24 // indirect diff --git a/api/go.sum b/api/go.sum index 375f7a5a0..9060b639a 100644 --- a/api/go.sum +++ b/api/go.sum @@ -95,8 +95,8 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ= -github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/klauspost/compress v1.20.2 h1:v5tiZvm1KGLyg5d5oXdXe3lsSPrbEUiKoXGia//I7Qo= +github.com/klauspost/compress v1.20.2/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= diff --git a/mcp-server/go.mod b/mcp-server/go.mod index 7418728b5..7147c09c2 100644 --- a/mcp-server/go.mod +++ b/mcp-server/go.mod @@ -40,7 +40,7 @@ require ( github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 // indirect github.com/google/uuid v1.6.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.19.2 // indirect + github.com/klauspost/compress v1.20.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect diff --git a/mcp-server/go.sum b/mcp-server/go.sum index 9f0a1eb3f..8172723c7 100644 --- a/mcp-server/go.sum +++ b/mcp-server/go.sum @@ -75,8 +75,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= -github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.20.2 h1:v5tiZvm1KGLyg5d5oXdXe3lsSPrbEUiKoXGia//I7Qo= +github.com/klauspost/compress v1.20.2/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/modelcontextprotocol/go-sdk v1.8.0 h1:KIvahhYqwtbeniWVPs3TcXEA7b8jEtwfBpOTAI+Urx4= diff --git a/operator/go.mod b/operator/go.mod index 01e1c8b40..ed1f8736c 100644 --- a/operator/go.mod +++ b/operator/go.mod @@ -103,7 +103,7 @@ require ( github.com/jedisct1/go-minisign v0.0.0-20260527172527-a09352b57a22 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kevinburke/ssh_config v1.6.0 // indirect - github.com/klauspost/compress v1.20.1 // indirect + github.com/klauspost/compress v1.20.2 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.24 // indirect github.com/miekg/pkcs11 v1.1.2 // indirect diff --git a/operator/go.sum b/operator/go.sum index e9ef5dfd8..3a83a8a3f 100644 --- a/operator/go.sum +++ b/operator/go.sum @@ -340,8 +340,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kevinburke/ssh_config v1.6.0 h1:J1FBfmuVosPHf5GRdltRLhPJtJpTlMdKTBjRgTaQBFY= github.com/kevinburke/ssh_config v1.6.0/go.mod h1:q2RIzfka+BXARoNexmF9gkxEX7DmvbW9P4hIVx2Kg4M= -github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ= -github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/klauspost/compress v1.20.2 h1:v5tiZvm1KGLyg5d5oXdXe3lsSPrbEUiKoXGia//I7Qo= +github.com/klauspost/compress v1.20.2/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= diff --git a/telemetry-receiver/go.mod b/telemetry-receiver/go.mod index cf76607c3..05493894c 100644 --- a/telemetry-receiver/go.mod +++ b/telemetry-receiver/go.mod @@ -19,6 +19,7 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.20.2 // indirect github.com/mattn/go-isatty v0.0.24 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect diff --git a/telemetry-receiver/go.sum b/telemetry-receiver/go.sum index 890254dc7..7f34eaf77 100644 --- a/telemetry-receiver/go.sum +++ b/telemetry-receiver/go.sum @@ -12,8 +12,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= -github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ= -github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/klauspost/compress v1.20.2 h1:v5tiZvm1KGLyg5d5oXdXe3lsSPrbEUiKoXGia//I7Qo= +github.com/klauspost/compress v1.20.2/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= diff --git a/tunnel/go.mod b/tunnel/go.mod index 56ae07048..fc3027b00 100644 --- a/tunnel/go.mod +++ b/tunnel/go.mod @@ -21,7 +21,7 @@ require ( github.com/hdevalence/ed25519consensus v0.2.0 // indirect github.com/huin/goupnp v1.3.0 // indirect github.com/jsimonetti/rtnetlink v1.4.2 // indirect - github.com/klauspost/compress v1.20.1 // indirect + github.com/klauspost/compress v1.20.2 // indirect github.com/mdlayher/netlink v1.11.2 // indirect github.com/mdlayher/socket v0.7.0 // indirect github.com/mitchellh/go-ps v1.0.0 // indirect diff --git a/tunnel/go.sum b/tunnel/go.sum index c9c0ce0d6..ab6f5885c 100644 --- a/tunnel/go.sum +++ b/tunnel/go.sum @@ -114,8 +114,8 @@ github.com/jsimonetti/rtnetlink v1.4.2 h1:Df9w9TZ3npHTyDn0Ev9e1uzmN2odmXd0QX+J5G github.com/jsimonetti/rtnetlink v1.4.2/go.mod h1:92s6LJdE+1iOrw+F2/RO7LYI2Qd8pPpFNNUYW06gcoM= github.com/kamstrup/intmap v0.5.2 h1:qnwBm1mh4XAnW9W9Ue9tZtTff8pS6+s6iKF6JRIV2Dk= github.com/kamstrup/intmap v0.5.2/go.mod h1:gWUVWHKzWj8xpJVFf5GC0O26bWmv3GqdnIX/LMT6Aq4= -github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ= -github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/klauspost/compress v1.20.2 h1:v5tiZvm1KGLyg5d5oXdXe3lsSPrbEUiKoXGia//I7Qo= +github.com/klauspost/compress v1.20.2/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/kortschak/wol v0.0.0-20200729010619-da482cc4850a h1:+RR6SqnTkDLWyICxS1xpjCi/3dhyV+TgZwA6Ww3KncQ= github.com/kortschak/wol v0.0.0-20200729010619-da482cc4850a/go.mod h1:YTtCCM3ryyfiu4F7t8HQ1mxvp1UBdWM2r6Xa+nGWvDk= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= From 939160186e6477246bd06358fdb826da48e3a949 Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 6 Nov 2026 02:17:26 -0500 Subject: [PATCH 14/20] fix(deps): retain workspace graph checksum Keep the historical compress go.mod checksum required when loading transitive workspace module metadata. The selected compress source remains v1.20.2. Signed-off-by: Baylem --- go.work.sum | 1 + 1 file changed, 1 insertion(+) diff --git a/go.work.sum b/go.work.sum index da1f24437..c733fe949 100644 --- a/go.work.sum +++ b/go.work.sum @@ -514,6 +514,7 @@ github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= github.com/kisielk/errcheck v1.10.0/go.mod h1:kQxWMMVZgIkDq7U8xtG/n2juOjbLgZtedi0D+/VL/i8= github.com/kkHAIKE/contextcheck v1.1.6/go.mod h1:3dDbMRNBFaq8HFXWC1JyvDSPm43CmE6IuHam8Wr0rkg= +github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= From 96450e9b04edaae4b261e98517d02711396982cb Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 6 Nov 2026 13:33:49 -0500 Subject: [PATCH 15/20] fix(operator): bound initial backup source admission Signed-off-by: Baylem (cherry picked from commit 840ff8a83bf2fe7a6076609899950f37d03f40ab) Signed-off-by: Baylem --- docs/operator-operations.md | 7 + operator/internal/controller/backup_guard.go | 2 +- .../internal/controller/backup_preparation.go | 20 ++- .../controller/backup_preparation_test.go | 156 ++++++++++++++++++ 4 files changed, 182 insertions(+), 3 deletions(-) create mode 100644 operator/internal/controller/backup_preparation_test.go diff --git a/docs/operator-operations.md b/docs/operator-operations.md index 1ea61c7fe..73785fcf2 100644 --- a/docs/operator-operations.md +++ b/docs/operator-operations.md @@ -5,6 +5,13 @@ including backups with `quiesce: false` and CSI VolumeSnapshots. A held operatio also defers idle sleep/wake, restart, and pod-template rollout. Requests remain pending and reconciliation resumes after the operation's cleanup completes. +For a quiesced backup of a running workload, an initially absent, non-Running or +terminating source Pod gets a two-minute startup grace period from the Backup's +creation time. If it is still unavailable afterward, the Backup fails and runs +normal cleanup to release its lifecycle owner and workload fence. A source that +is already Running can be admitted after that window; prepared source identity +and outstanding worker or save-on obligations retain their existing safety checks. + The GameServer annotation `operations.gameplane.local/owner` identifies the operation kind, name, UID, admitted generation and target UID. A wipe is identified by the GameServer UID and its request token. The existing backup, restore and wipe diff --git a/operator/internal/controller/backup_guard.go b/operator/internal/controller/backup_guard.go index 56083c303..41c825ecc 100644 --- a/operator/internal/controller/backup_guard.go +++ b/operator/internal/controller/backup_guard.go @@ -271,7 +271,7 @@ func (r *BackupReconciler) prepareBackupQuiesce(ctx context.Context, b *gameplan return ctrl.Result{RequeueAfter: 5 * time.Second}, err } ready, err := r.prepareBackupSource(ctx, b, gs) - if errors.Is(err, errBackupSourceChanged) { + if errors.Is(err, errBackupSourceChanged) || errors.Is(err, errBackupSourceStartupTimeout) { _, err = r.fail(ctx, b, err.Error()) return ctrl.Result{RequeueAfter: time.Nanosecond}, err } diff --git a/operator/internal/controller/backup_preparation.go b/operator/internal/controller/backup_preparation.go index 6443d9ba8..cd2305c6f 100644 --- a/operator/internal/controller/backup_preparation.go +++ b/operator/internal/controller/backup_preparation.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "time" appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" @@ -24,6 +25,21 @@ const ( var errBackupSourceChanged = errors.New("backup source changed after preparation") +var errBackupSourceStartupTimeout = errors.New("backup source startup grace period expired") + +// Allow ordinary startup/restart races without holding the workload fence +// indefinitely when an initial live source never becomes Running. +const backupSourceStartupGracePeriod = 2 * time.Minute + +// waitForBackupSource bounds initial Pod admission using durable API creation +// time, so operator restarts and reconcile retries cannot restart the grace. +func waitForBackupSource(b *gameplanev1alpha1.Backup, reason string) (bool, error) { + if time.Since(b.CreationTimestamp.Time) < backupSourceStartupGracePeriod { + return false, nil + } + return false, fmt.Errorf("%w after %s: %s", errBackupSourceStartupTimeout, backupSourceStartupGracePeriod, reason) +} + type podAgentQuiescer interface { CheckPod(context.Context, string, string, string) error QuiescePod(context.Context, string, string, string) error @@ -146,7 +162,7 @@ func (r *BackupReconciler) prepareBackupSource(ctx context.Context, b *gameplane if p != nil { return false, fmt.Errorf("%w: original pod absent", errBackupSourceChanged) } - return false, nil + return waitForBackupSource(b, "source pod absent") } if err != nil { return false, err @@ -158,7 +174,7 @@ func (r *BackupReconciler) prepareBackupSource(ctx context.Context, b *gameplane return false, fmt.Errorf("backup source pod is not owned by the bound workload") } if !pod.DeletionTimestamp.IsZero() || pod.Status.Phase != corev1.PodRunning { - return false, nil + return waitForBackupSource(b, "source pod is not Running or is terminating") } candidate.Outcome, candidate.PodUID = "live", pod.UID } diff --git a/operator/internal/controller/backup_preparation_test.go b/operator/internal/controller/backup_preparation_test.go new file mode 100644 index 000000000..2bc6f8dc3 --- /dev/null +++ b/operator/internal/controller/backup_preparation_test.go @@ -0,0 +1,156 @@ +package controller + +import ( + "strings" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +// Removing the admission deadline must leave the old Backup pending and keep +// its owner/fence held, preventing the successor from creating its copy Job. +func TestBackupInitialSourceStartupGrace(t *testing.T) { + for _, state := range []string{"absent", "pending", "terminating"} { + for _, expired := range []bool{false, true} { + name := state + "/within-grace" + if expired { + name = state + "/expired" + } + t.Run(name, func(t *testing.T) { + first := guardBackup("first") + first.CreationTimestamp = metav1.Now() + if expired { + first.CreationTimestamp = metav1.NewTime(time.Now().Add(-3 * time.Minute)) + } + r, q := guardFixture(t, first, guardBackup("second")) + ctx := t.Context() + podKey := client.ObjectKey{Namespace: "ns", Name: "server-0"} + var pod corev1.Pod + if err := r.Get(ctx, podKey, &pod); err != nil { + t.Fatal(err) + } + switch state { + case "absent": + if err := r.Delete(ctx, &pod); err != nil { + t.Fatal(err) + } + case "pending": + pod.Status.Phase = corev1.PodPending + if err := r.Update(ctx, &pod); err != nil { + t.Fatal(err) + } + case "terminating": + pod.Finalizers = []string{"test/hold"} + if err := r.Update(ctx, &pod); err != nil { + t.Fatal(err) + } + if err := r.Delete(ctx, &pod); err != nil { + t.Fatal(err) + } + } + result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(first)}) + if err != nil || result.RequeueAfter <= 0 { + t.Fatalf("source admission did not requeue: %+v, %v", result, err) + } + guardPass(t, r, "first") + first = guardGetBackup(t, r, "first") + if expired { + if first.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || first.Status.CompletionTime == nil || !strings.Contains(first.Status.Message, "startup grace period expired") { + t.Fatalf("expired source did not fail visibly: %+v", first.Status) + } + } else if first.Status.Phase == gameplanev1alpha1.BackupPhaseFailed { + t.Fatalf("startup race failed before grace expired: %+v", first.Status) + } + if first.Status.CaptureManifest != nil || q.quiesced != 0 || q.unquiesced != 0 { + t.Fatalf("unready source attempted capture: manifest=%+v agent=%+v", first.Status.CaptureManifest, q) + } + var jobs batchv1.JobList + if err := r.List(ctx, &jobs); err != nil || len(jobs.Items) != 0 { + t.Fatalf("unready source created copy workers: %+v, %v", jobs.Items, err) + } + serverKey := client.ObjectKey{Namespace: "ns", Name: "server"} + var gs gameplanev1alpha1.GameServer + var ss appsv1.StatefulSet + if err := r.Get(ctx, serverKey, &gs); err != nil { + t.Fatal(err) + } + if err := r.Get(ctx, serverKey, &ss); err != nil { + t.Fatal(err) + } + if held := gs.Annotations[lifecycleOperationAnnotation] != ""; held == expired { + t.Fatalf("wrong lifecycle owner after admission: %q", gs.Annotations[lifecycleOperationAnnotation]) + } + if held := ss.Annotations[lifecycleOperationAnnotation] != ""; held == expired { + t.Fatalf("wrong workload fence after admission: %q", ss.Annotations[lifecycleOperationAnnotation]) + } + // A healthy source can recover during the grace period. After an + // expired attempt, a successor must also be able to acquire it. + switch state { + case "absent": + pod.ResourceVersion = "" + if err := r.Create(ctx, &pod); err != nil { + t.Fatal(err) + } + case "pending": + pod.Status.Phase = corev1.PodRunning + if err := r.Update(ctx, &pod); err != nil { + t.Fatal(err) + } + case "terminating": + if err := r.Get(ctx, podKey, &pod); err != nil { + t.Fatal(err) + } + pod.Finalizers = nil + if err := r.Update(ctx, &pod); err != nil { + t.Fatal(err) + } + pod.ResourceVersion, pod.DeletionTimestamp = "", nil + pod.UID = "replacement-pod-uid" + if err := r.Create(ctx, &pod); err != nil { + t.Fatal(err) + } + } + admitted := "first" + if expired { + admitted = "second" + } + guardPass(t, r, admitted) + var job batchv1.Job + if err := r.Get(ctx, client.ObjectKey{Namespace: "ns", Name: admitted}, &job); err != nil { + t.Fatalf("healthy source could not proceed: %v", err) + } + if q.quiesced != 1 { + t.Fatalf("healthy source was not quiesced once: %+v", q) + } + if expired { + if err := r.Get(ctx, client.ObjectKey{Namespace: "ns", Name: "first"}, &job); !apierrors.IsNotFound(err) { + t.Fatalf("expired attempt started a worker: %v", err) + } + } + }) + } + } +} + +func TestBackupHealthySourceAdmittedAfterStartupGrace(t *testing.T) { + b := guardBackup("first") + b.CreationTimestamp = metav1.NewTime(time.Now().Add(-3 * time.Minute)) + r, q := guardFixture(t, b) + guardPass(t, r, "first") + var job batchv1.Job + if err := r.Get(t.Context(), client.ObjectKeyFromObject(b), &job); err != nil { + t.Fatalf("backup age rejected a ready source: %v", err) + } + if q.quiesced != 1 { + t.Fatalf("ready source was not quiesced: %+v", q) + } +} From c3d31cde7c76445ec07f6cd2b4ea2722aa21952b Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 6 Nov 2026 13:37:10 -0500 Subject: [PATCH 16/20] test(backup): assert source admission before metadata support Signed-off-by: Baylem --- operator/internal/controller/backup_preparation_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/operator/internal/controller/backup_preparation_test.go b/operator/internal/controller/backup_preparation_test.go index 2bc6f8dc3..bd276fe04 100644 --- a/operator/internal/controller/backup_preparation_test.go +++ b/operator/internal/controller/backup_preparation_test.go @@ -70,8 +70,8 @@ func TestBackupInitialSourceStartupGrace(t *testing.T) { } else if first.Status.Phase == gameplanev1alpha1.BackupPhaseFailed { t.Fatalf("startup race failed before grace expired: %+v", first.Status) } - if first.Status.CaptureManifest != nil || q.quiesced != 0 || q.unquiesced != 0 { - t.Fatalf("unready source attempted capture: manifest=%+v agent=%+v", first.Status.CaptureManifest, q) + if first.Annotations[backupPreparationAnnotation] != "" || q.quiesced != 0 || q.unquiesced != 0 { + t.Fatalf("unready source attempted capture: preparation=%q agent=%+v", first.Annotations[backupPreparationAnnotation], q) } var jobs batchv1.JobList if err := r.List(ctx, &jobs); err != nil || len(jobs.Items) != 0 { From d74269da41d3cfff1127bd34f5990b09f543ac81 Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 6 Nov 2026 13:37:39 -0500 Subject: [PATCH 17/20] test(operator): update backup source pod status through subresource Signed-off-by: Baylem (cherry picked from commit b3b893b88a88971933ef57a37c4118f3af1a4b1c) Signed-off-by: Baylem --- operator/internal/controller/backup_preparation_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/operator/internal/controller/backup_preparation_test.go b/operator/internal/controller/backup_preparation_test.go index bd276fe04..c69ce0d02 100644 --- a/operator/internal/controller/backup_preparation_test.go +++ b/operator/internal/controller/backup_preparation_test.go @@ -45,7 +45,7 @@ func TestBackupInitialSourceStartupGrace(t *testing.T) { } case "pending": pod.Status.Phase = corev1.PodPending - if err := r.Update(ctx, &pod); err != nil { + if err := r.Status().Update(ctx, &pod); err != nil { t.Fatal(err) } case "terminating": @@ -102,7 +102,7 @@ func TestBackupInitialSourceStartupGrace(t *testing.T) { } case "pending": pod.Status.Phase = corev1.PodRunning - if err := r.Update(ctx, &pod); err != nil { + if err := r.Status().Update(ctx, &pod); err != nil { t.Fatal(err) } case "terminating": From 09831c3978fa57bd4d2954db1fd2392653d755e1 Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 6 Nov 2026 14:50:45 -0500 Subject: [PATCH 18/20] fix(deps): update x/net in shipped Go binaries Select golang.org/x/net v0.62.0 for GO-2026-6860 (CVE-2026-78661) in all seven affected production module graphs, including the isolated frpc build. Retain the scanner gate and update only the required module graph minimums and checksums. Signed-off-by: Baylem --- agent/go.mod | 6 +++--- agent/go.sum | 6 ++++++ api/go.mod | 2 +- api/go.sum | 2 ++ mcp-server/go.mod | 10 +++++----- mcp-server/go.sum | 11 +++++++++++ operator/go.mod | 2 +- operator/go.sum | 2 ++ sentinel/go.mod | 8 ++++---- sentinel/go.sum | 8 ++++++++ tunnel/frpc/go.mod | 2 +- tunnel/frpc/go.sum | 2 ++ tunnel/go.mod | 2 +- tunnel/go.sum | 2 ++ 14 files changed, 49 insertions(+), 16 deletions(-) diff --git a/agent/go.mod b/agent/go.mod index 426be7f19..122b11f87 100644 --- a/agent/go.mod +++ b/agent/go.mod @@ -53,10 +53,10 @@ require ( github.com/x448/float16 v0.8.4 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect - golang.org/x/net v0.60.0 // indirect + golang.org/x/net v0.62.0 // indirect golang.org/x/oauth2 v0.37.0 // indirect - golang.org/x/term v0.46.0 // indirect - golang.org/x/text v0.42.0 // indirect + golang.org/x/term v0.47.0 // indirect + golang.org/x/text v0.43.0 // indirect golang.org/x/time v0.15.0 // indirect google.golang.org/protobuf v1.36.12 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect diff --git a/agent/go.sum b/agent/go.sum index d98b2a11e..5972cf8a8 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -100,14 +100,20 @@ go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= +golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58= golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= +golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM= +golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY= golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= +golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= diff --git a/api/go.mod b/api/go.mod index 21e3fc1de..56ba6a627 100644 --- a/api/go.mod +++ b/api/go.mod @@ -110,7 +110,7 @@ require ( go.uber.org/zap v1.28.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect - golang.org/x/net v0.60.0 // indirect + golang.org/x/net v0.62.0 // indirect golang.org/x/term v0.47.0 // indirect golang.org/x/time v0.15.0 // indirect gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect diff --git a/api/go.sum b/api/go.sum index 9060b639a..5c9bca2c6 100644 --- a/api/go.sum +++ b/api/go.sum @@ -193,6 +193,8 @@ golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg= golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs= golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= +golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58= golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= diff --git a/mcp-server/go.mod b/mcp-server/go.mod index 7147c09c2..9ce5f3f88 100644 --- a/mcp-server/go.mod +++ b/mcp-server/go.mod @@ -57,12 +57,12 @@ require ( go.uber.org/zap v1.28.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect - golang.org/x/net v0.60.0 // indirect + golang.org/x/net v0.62.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sync v0.23.0 // indirect - golang.org/x/sys v0.48.0 // indirect - golang.org/x/term v0.46.0 // indirect - golang.org/x/text v0.42.0 // indirect + golang.org/x/sync v0.24.0 // indirect + golang.org/x/sys v0.49.0 // indirect + golang.org/x/term v0.47.0 // indirect + golang.org/x/text v0.43.0 // indirect golang.org/x/time v0.15.0 // indirect gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect diff --git a/mcp-server/go.sum b/mcp-server/go.sum index 8172723c7..6539daf59 100644 --- a/mcp-server/go.sum +++ b/mcp-server/go.sum @@ -136,20 +136,31 @@ golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= +golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= +golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= +golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= +golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM= +golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY= golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= +golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= +golang.org/x/tools v0.51.0 h1:k4Xc/1Om9jwkBJBo4NVLMSARBoWtK10mx+W5BnXCeAI= gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0= gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= diff --git a/operator/go.mod b/operator/go.mod index ed1f8736c..413a23fa1 100644 --- a/operator/go.mod +++ b/operator/go.mod @@ -152,7 +152,7 @@ require ( go.uber.org/zap v1.28.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect - golang.org/x/net v0.60.0 // indirect + golang.org/x/net v0.62.0 // indirect golang.org/x/oauth2 v0.37.0 // indirect golang.org/x/sync v0.24.0 // indirect golang.org/x/term v0.47.0 // indirect diff --git a/operator/go.sum b/operator/go.sum index 3a83a8a3f..dc7b5700b 100644 --- a/operator/go.sum +++ b/operator/go.sum @@ -583,6 +583,8 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= +golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/sentinel/go.mod b/sentinel/go.mod index f0a24254b..023341121 100644 --- a/sentinel/go.mod +++ b/sentinel/go.mod @@ -38,11 +38,11 @@ require ( github.com/x448/float16 v0.8.4 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect - golang.org/x/net v0.60.0 // indirect + golang.org/x/net v0.62.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sys v0.48.0 // indirect - golang.org/x/term v0.46.0 // indirect - golang.org/x/text v0.42.0 // indirect + golang.org/x/sys v0.49.0 // indirect + golang.org/x/term v0.47.0 // indirect + golang.org/x/text v0.43.0 // indirect golang.org/x/time v0.15.0 // indirect google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect diff --git a/sentinel/go.sum b/sentinel/go.sum index 926892929..6b2bfd7a7 100644 --- a/sentinel/go.sum +++ b/sentinel/go.sum @@ -78,14 +78,22 @@ go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= +golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= +golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= +golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM= +golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY= golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= +golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= diff --git a/tunnel/frpc/go.mod b/tunnel/frpc/go.mod index cdc68e414..b7319cbd0 100644 --- a/tunnel/frpc/go.mod +++ b/tunnel/frpc/go.mod @@ -38,7 +38,7 @@ require ( github.com/vishvananda/netns v0.0.4 // indirect github.com/xtaci/kcp-go/v5 v5.6.13 // indirect golang.org/x/crypto v0.58.0 // indirect - golang.org/x/net v0.60.0 // indirect + golang.org/x/net v0.62.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sync v0.24.0 // indirect golang.org/x/sys v0.49.0 // indirect diff --git a/tunnel/frpc/go.sum b/tunnel/frpc/go.sum index 73d2adc13..dab154d3d 100644 --- a/tunnel/frpc/go.sum +++ b/tunnel/frpc/go.sum @@ -111,6 +111,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= +golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= diff --git a/tunnel/go.mod b/tunnel/go.mod index fc3027b00..3e492ebc3 100644 --- a/tunnel/go.mod +++ b/tunnel/go.mod @@ -38,7 +38,7 @@ require ( go4.org/netipx v0.0.0-20260823151212-3075585bcbeb // indirect golang.org/x/crypto v0.58.0 // indirect golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba // indirect - golang.org/x/net v0.61.0 // indirect + golang.org/x/net v0.62.0 // indirect golang.org/x/oauth2 v0.37.0 // indirect golang.org/x/sync v0.24.0 // indirect golang.org/x/sys v0.49.0 // indirect diff --git a/tunnel/go.sum b/tunnel/go.sum index ab6f5885c..eed4f09ba 100644 --- a/tunnel/go.sum +++ b/tunnel/go.sum @@ -204,6 +204,8 @@ golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= golang.org/x/net v0.61.0 h1:RR+6j/BTBCrggOLugxa+66R8FvoN6MHlzDlZIbN9YFw= golang.org/x/net v0.61.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= +golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= +golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From 0cc80c120a23e405e58c8a10790301275e7732a7 Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 6 Nov 2026 15:30:20 -0500 Subject: [PATCH 19/20] fix(deps): normalize source module checksums after x/net update Match the repository tidy output for the six source modules. Remove superseded x module checksums and retain the selected x/tools v0.51.0 go.mod checksum. Keep the isolated frpc build module unchanged. Signed-off-by: Baylem --- agent/go.sum | 6 ------ api/go.sum | 2 -- mcp-server/go.sum | 13 +------------ operator/go.sum | 2 -- sentinel/go.sum | 8 -------- tunnel/go.sum | 2 -- 6 files changed, 1 insertion(+), 32 deletions(-) diff --git a/agent/go.sum b/agent/go.sum index 5972cf8a8..4e4caf01e 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -98,20 +98,14 @@ go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= -golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= -golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58= golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= -golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= -golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM= golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY= -golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= -golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= diff --git a/api/go.sum b/api/go.sum index 5c9bca2c6..1eb60b7e3 100644 --- a/api/go.sum +++ b/api/go.sum @@ -191,8 +191,6 @@ golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w= golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg= golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs= -golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= -golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= diff --git a/mcp-server/go.sum b/mcp-server/go.sum index 6539daf59..df19a5b8c 100644 --- a/mcp-server/go.sum +++ b/mcp-server/go.sum @@ -134,33 +134,22 @@ go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= -golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= -golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= -golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= -golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= -golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= -golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= -golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= -golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM= golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY= -golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= -golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= -golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= -golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= golang.org/x/tools v0.51.0 h1:k4Xc/1Om9jwkBJBo4NVLMSARBoWtK10mx+W5BnXCeAI= +golang.org/x/tools v0.51.0/go.mod h1:9eEncMayCV6zRMGhR5eZEC2iBx98qWcF1HZ9Z7wJOoA= gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0= gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= diff --git a/operator/go.sum b/operator/go.sum index dc7b5700b..9025bd04e 100644 --- a/operator/go.sum +++ b/operator/go.sum @@ -581,8 +581,6 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= -golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= diff --git a/sentinel/go.sum b/sentinel/go.sum index 6b2bfd7a7..85e3e603a 100644 --- a/sentinel/go.sum +++ b/sentinel/go.sum @@ -76,22 +76,14 @@ go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= -golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= -golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= -golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= -golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= -golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= -golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM= golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY= -golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= -golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= diff --git a/tunnel/go.sum b/tunnel/go.sum index eed4f09ba..df5b45492 100644 --- a/tunnel/go.sum +++ b/tunnel/go.sum @@ -202,8 +202,6 @@ golang.org/x/image v0.46.0 h1:b1+oYj0Jbp6K5MDT4i4/eZpYlk3V8SJhhDKh6LBHAyQ= golang.org/x/image v0.46.0/go.mod h1:3B3W05VGVQyuXucLINLjXKrqISASfi4Xj+iCVkLMwew= golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= -golang.org/x/net v0.61.0 h1:RR+6j/BTBCrggOLugxa+66R8FvoN6MHlzDlZIbN9YFw= -golang.org/x/net v0.61.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= From cfb7258c179b0235b6216e288229e31bf6c160ab Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 6 Nov 2026 16:37:55 -0500 Subject: [PATCH 20/20] fix(deps): retain workspace graph metadata after x/net update Keep the historical x/net v0.60.0 and x/tools v0.49.0 go.mod checksums loaded by workspace-mode generation. The selected source remains x/net v0.62.0. Regeneration, chart synchronization and all source-module tidy commands are idempotent with these metadata entries retained. Signed-off-by: Baylem --- go.work.sum | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go.work.sum b/go.work.sum index c733fe949..8e5e543d3 100644 --- a/go.work.sum +++ b/go.work.sum @@ -893,6 +893,7 @@ golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= @@ -983,6 +984,7 @@ golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0= golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0= golang.org/x/tools/go/expect v0.1.0-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA=