From 3f78ea9d82072e546b420db08980c3e5b4a82e4d Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 21 Oct 2026 11:14:45 -0400 Subject: [PATCH 01/20] fix(wake): bound requests and preserve HostPort endpoint handoffs Signed-off-by: Baylem --- .github/workflows/ci.yaml | 10 +- charts/gameplane/templates/operator.yaml | 4 + deploy/kind/e2e.sh | 13 +- .../2026-07-24-wake-on-connect-design.md | 42 ++- operator/config/rbac/role.yaml | 6 + .../controller/gameserver_controller.go | 24 +- .../controller/gameserver_hostport.go | 186 ++++++++++++ .../controller/gameserver_hostport_test.go | 132 +++++++++ .../controller/gameserver_sentinel.go | 53 +++- .../gameserver_sentinel_envtest_test.go | 54 +++- .../internal/controller/gameserver_status.go | 11 + operator/specs.md | 18 ++ sentinel/main.go | 76 +++-- sentinel/specs.md | 5 +- sentinel/udp_wake.go | 41 +++ sentinel/wake_budget_test.go | 264 ++++++++++++++++++ test/e2e/buckets.sh | 8 +- test/e2e/hostport_wake_e2e_test.go | 238 ++++++++++++++++ test/e2e/specs.md | 11 + 19 files changed, 1149 insertions(+), 47 deletions(-) create mode 100644 operator/internal/controller/gameserver_hostport.go create mode 100644 operator/internal/controller/gameserver_hostport_test.go create mode 100644 sentinel/udp_wake.go create mode 100644 sentinel/wake_budget_test.go create mode 100644 test/e2e/hostport_wake_e2e_test.go diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a1c9aecdd..967f99c60 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1664,13 +1664,17 @@ jobs: # (operator does zero logins and is wait-dominated, so it runs # wide; the api buckets stay within the login limiter's budget). # The arm64 job below uses the same steps and per-bucket settings. - bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components] + bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, wake-hostport] arch: [amd64] include: - bucket: operator parallel: 8 test_timeout: 20m job_timeout: 60 # Boots a kind cluster, installs the chart, runs the bucket's suite. + - bucket: wake-hostport + parallel: 1 + test_timeout: 15m + job_timeout: 60 - bucket: api-auth parallel: 4 test_timeout: 15m @@ -1718,7 +1722,9 @@ jobs: with: arch: ${{ matrix.arch }} - name: bring up cluster + helm install - run: ./deploy/kind/e2e.sh up "$CLUSTER" e2e + env: + GAMEPLANE_E2E_PROFILE: ${{ matrix.bucket == 'wake-hostport' && 'two-node' || 'default' }} + run: ./deploy/kind/e2e.sh up "$CLUSTER" e2e "$GAMEPLANE_E2E_PROFILE" - name: run go e2e (${{ matrix.bucket }}) run: | RUN_REGEX="$(./test/e2e/buckets.sh regex '${{ matrix.bucket }}')" diff --git a/charts/gameplane/templates/operator.yaml b/charts/gameplane/templates/operator.yaml index 772e0b10e..d395f52eb 100644 --- a/charts/gameplane/templates/operator.yaml +++ b/charts/gameplane/templates/operator.yaml @@ -16,6 +16,10 @@ metadata: name: {{ .Release.Name }}-operator-watch labels: {{- include "gameplane.labels" . | nindent 4 }} rules: + # Direct node identity/readiness checks for durable Hostport wake endpoints. + - apiGroups: [""] + resources: [nodes] + verbs: [get] - apiGroups: ["gameplane.local"] resources: [gameservers, gametemplates, backups, backupschedules, restores, modules, modulesources, networkcaptures] verbs: [get, list, watch] diff --git a/deploy/kind/e2e.sh b/deploy/kind/e2e.sh index 3c2f7b12f..c6f1d8e72 100755 --- a/deploy/kind/e2e.sh +++ b/deploy/kind/e2e.sh @@ -6,7 +6,7 @@ # deploy/kind/e2e.sh down [cluster-name] # # Diffs from dev-up (deploy/kind/up.sh): -# - Single-node cluster (faster boot, sufficient for E2E coverage). +# - Single-node cluster by default; two-node profile tests Hostport handoff. # - Skips ingress-nginx (the dashboard isn't exercised here). # - Loads pre-built gameplane-test/{operator,api,agent,sentinel,capture-sidecar,telemetry-receiver,fakeoidc}: # images (and gameplane-test/gameprobe: when present). @@ -28,9 +28,9 @@ CLUSTER="${2:-gameplane-e2e}" TAG="${3:-${GAMEPLANE_E2E_TAG:-e2e}}" PROFILE="${4:-default}" case "${PROFILE}" in -default | postgres) ;; +default | postgres | two-node) ;; *) - echo "unknown profile '${PROFILE}' (want default or postgres)" >&2 + echo "unknown profile '${PROFILE}' (want default, postgres or two-node)" >&2 exit 2 ;; esac @@ -188,12 +188,17 @@ up) echo "removing pre-existing cluster ${CLUSTER} for a clean slate" kind delete cluster --name "${CLUSTER}" || true fi - echo "creating kind cluster ${CLUSTER} (single-node)" + echo "creating kind cluster ${CLUSTER} (${PROFILE})" + extra_nodes="" + if [[ "${PROFILE}" == "two-node" ]]; then + extra_nodes=" - role: worker" + fi kind create cluster --name "${CLUSTER}" --wait 90s --config - </dev/null diff --git a/docs/superpowers/specs/2026-07-24-wake-on-connect-design.md b/docs/superpowers/specs/2026-07-24-wake-on-connect-design.md index 892617188..c63f44681 100644 --- a/docs/superpowers/specs/2026-07-24-wake-on-connect-design.md +++ b/docs/superpowers/specs/2026-07-24-wake-on-connect-design.md @@ -1,7 +1,7 @@ # Wake-on-connect for idle auto-sleep **Date:** 2026-07-24 -**Status:** approved, not yet implemented +**Status:** implemented; Hostport handoff revised October 2026 ## Problem @@ -77,7 +77,45 @@ A new Service `-game-direct` — an always-ClusterIP Service selecting the g 4. Game Service selector flips back to `gameserver: ` 5. Sentinel Deployment deleted last -Connections already held in the proxy survive the flip-back because they connect through `-game-direct`, not the re-pointed game Service. In Hostport mode the sentinel relinquishes the host port on deletion; the game pod briefly Pending self-heals via scheduler retry. +Connections already held in the proxy survive the flip-back because they connect through `-game-direct`, not the re-pointed game Service. + +**Hostport correction (#730, October 2026):** Hostport uses a separate handoff. +Before its first sleep, the operator saves the live game's node name and Node UID +in `gameplane.local/hostport-node` and `gameplane.local/hostport-node-uid`. These +annotations survive Pod deletion and operator restart; the dashboard's current +`gameplane.local/node` annotation remains a separate, transient observation. +The game completes its normal graceful stop and scales to zero. Only after the +StatefulSet acknowledges that generation and a direct API list confirms all game +Pods have disappeared does the operator create the sentinel. The sentinel has +required node affinity for that identity, the game's tolerations and scheduling +constraints, and a Recreate strategy. It never bypasses the scheduler with +`spec.nodeName`. + +On wake, foreground deletion removes the sentinel Deployment and its children. +The game remains at zero until a direct API read confirms the Deployment is gone +and no sentinel Pods remain, including terminating Pods. The game then starts +with the same required endpoint-node affinity. Hostport clients still reconnect; +the old "briefly Pending, self-heals" sequencing is superseded. Service-backed +exposure retains the hold-and-proxy drain described above. + +`HostportWakeReady=False` explains `GameDraining`, `SentinelDraining`, +`EndpointNodeUnknown`, `EndpointNodeUnavailable`, `EndpointUnschedulable`, or +`ListenerPending`. A full node stays pinned and reports the scheduler's reason; +a missing, replaced or unready node makes wake unavailable. The operator never +silently changes the advertised node. Unrelated child reconciliation continues +during these waits. After restoring node capacity/readiness, reconciliation +resumes automatically. For intentional migration, suspend the server, wait for +both game and sentinel Pods to disappear, clear both durable endpoint annotations, +update scheduling/storage placement and the client-facing node address, then +resume the game before sleeping it again. Do not clear the annotations while a +listener remains active. An installation upgraded while already asleep without +a recorded game node must first start its game to establish placement. + +**Wake budget correction (#729):** The join deadline starts before the Kubernetes +PATCH, which has a separate five-second maximum within that budget. Concurrent +callers share the actual result of the in-flight PATCH. UDP packet reads enqueue +coalesced requests to one bounded worker and never wait on the PATCH. Successful +TCP proxy sessions use the longer session/drain context after upstream connects. ### Connection handling state machine diff --git a/operator/config/rbac/role.yaml b/operator/config/rbac/role.yaml index e9d3d9337..666956650 100644 --- a/operator/config/rbac/role.yaml +++ b/operator/config/rbac/role.yaml @@ -26,6 +26,12 @@ rules: - list - patch - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get - apiGroups: - "" resources: diff --git a/operator/internal/controller/gameserver_controller.go b/operator/internal/controller/gameserver_controller.go index 67e8be4e3..5e5906e38 100644 --- a/operator/internal/controller/gameserver_controller.go +++ b/operator/internal/controller/gameserver_controller.go @@ -386,6 +386,9 @@ func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) logger.Error(err, "compute desired replicas") return ctrl.Result{}, err } + if plan.blockGame { + replicas = 0 + } if err := r.reconcileStatefulSet(ctx, &gs, &tmpl, ver, mc, replicas, &wipeState); err != nil { if apierrors.IsConflict(err) { return ctrl.Result{Requeue: true}, nil @@ -442,7 +445,7 @@ func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) } } - requeue, err := r.reconcileStatus(ctx, &gs, idle, idleStatus, idleScheduleErr, tunnelPlan, &tmpl, svcEvents, conflictingServer) + requeue, err := r.reconcileStatus(ctx, &gs, idle, idleStatus, idleScheduleErr, tunnelPlan, &tmpl, svcEvents, conflictingServer, plan.hostportCondition) if err != nil { return ctrl.Result{}, err } @@ -452,7 +455,7 @@ func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) // or nothing fires the transition until an unrelated event happens to // wake the reconciler. hints := []time.Duration{stopRequeue, idleRequeue} - if plan.wantSentinel { + if plan.wantSentinel || plan.hostportCondition != nil { // Bounded backstop for planSentinel's wait on the sentinel or the // game pod. Both are owned (Owns(&appsv1.Deployment{}), // Owns(&appsv1.StatefulSet{})) so a readiness transition normally @@ -1592,7 +1595,22 @@ func (r *GameServerReconciler) reconcileStatefulSet( // a removed scheduling pin active on the StatefulSet. ss.Spec.Template.Spec.NodeSelector = gs.Spec.NodeSelector ss.Spec.Template.Spec.Tolerations = gs.Spec.Tolerations - ss.Spec.Template.Spec.Affinity = gs.Spec.Affinity + // Do not change a live Pod's template merely because its node was + // recorded while a graceful stop is still saving the game. Apply the + // endpoint pin at zero replicas, then retain it on every future wake. + if actualReplicas == 0 || ss.ResourceVersion == "" || ss.Spec.Template.Annotations[hostportNodeAnnotation] != "" { + ss.Spec.Template.Spec.Affinity = hostportAffinity(gs) + if gs.Spec.Networking.Expose == "Hostport" && gs.Annotations[hostportNodeAnnotation] != "" { + if ss.Spec.Template.Annotations == nil { + ss.Spec.Template.Annotations = map[string]string{} + } + ss.Spec.Template.Annotations[hostportNodeAnnotation] = gs.Annotations[hostportNodeAnnotation] + } else { + delete(ss.Spec.Template.Annotations, hostportNodeAnnotation) + } + } else { + ss.Spec.Template.Spec.Affinity = gs.Spec.Affinity.DeepCopy() + } // Same unconditional-assign reasoning: removing template.spec.security // must clear a previously-set pod-level FSGroup, not leave it pinned. ss.Spec.Template.Spec.SecurityContext = gamePodSecurityContext(tmpl) diff --git a/operator/internal/controller/gameserver_hostport.go b/operator/internal/controller/gameserver_hostport.go new file mode 100644 index 000000000..621fa9cac --- /dev/null +++ b/operator/internal/controller/gameserver_hostport.go @@ -0,0 +1,186 @@ +package controller + +import ( + "context" + "fmt" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +const ( + // Unlike nodeAnnotation (the current Pod's dashboard location), these + // survive sleep, Pod deletion and operator restarts. Reusing a node name + // after replacement must not silently redirect the advertised endpoint. + hostportNodeAnnotation = "gameplane.local/hostport-node" + hostportNodeUIDAnnotation = "gameplane.local/hostport-node-uid" + hostportWakeCondition = "HostportWakeReady" +) + +// +kubebuilder:rbac:groups=core,resources=nodes,verbs=get + +func (r *GameServerReconciler) hostportReader() client.Reader { + if r.APIReader != nil { + return r.APIReader + } + return r.Client +} + +func hostportCondition(gs *gameplanev1alpha1.GameServer, reason, message string, ready bool) *metav1.Condition { + status := metav1.ConditionFalse + if ready { + status = metav1.ConditionTrue + } + return &metav1.Condition{Type: hostportWakeCondition, Status: status, Reason: reason, Message: message, ObservedGeneration: gs.Generation} +} + +// planHostportSentinel never transfers a port based on replica status or a +// cached Pod absence. Producers must first stop, then their actual Pods must +// disappear. All waits return a plan so the remaining reconciliation proceeds. +func (r *GameServerReconciler) planHostportSentinel(ctx context.Context, gs *gameplanev1alpha1.GameServer, idle idleState) (sentinelPlan, error) { + reader := r.hostportReader() + var gamePods, sentinelPods corev1.PodList + for _, entry := range []struct { + name string + pods *corev1.PodList + }{{"gameplane-game", &gamePods}, {sentinelValue, &sentinelPods}} { + if err := reader.List(ctx, entry.pods, client.InNamespace(gs.Namespace), client.MatchingLabels{"app.kubernetes.io/name": entry.name, "app.kubernetes.io/instance": gs.Name}); err != nil { + return sentinelPlan{}, err + } + } + plan := sentinelPlan{} + set := func(reason, message string, ready bool) { + plan.hostportCondition = hostportCondition(gs, reason, message, ready) + } + nodeName := gs.Annotations[hostportNodeAnnotation] + // Record the endpoint before the first sleep scales down the game. Doing + // this during sleep also avoids rolling an already-running game merely to + // add the new affinity after an operator upgrade. + if nodeName == "" && idle == idleAsleep { + for _, pod := range gamePods.Items { + if pod.Name == gs.Name+"-0" && pod.Spec.NodeName != "" { + nodeName = pod.Spec.NodeName + break + } + } + if nodeName != "" { + var node corev1.Node + if err := reader.Get(ctx, client.ObjectKey{Name: nodeName}, &node); err != nil { + if !apierrors.IsNotFound(err) { + return sentinelPlan{}, err + } + } + base := gs.DeepCopy() + if gs.Annotations == nil { + gs.Annotations = map[string]string{} + } + gs.Annotations[hostportNodeAnnotation] = nodeName + gs.Annotations[hostportNodeUIDAnnotation] = string(node.UID) + if err := r.Patch(ctx, gs, client.MergeFrom(base)); err != nil { + return sentinelPlan{}, err + } + } + } + if nodeName == "" && idle == idleAsleep { + set("EndpointNodeUnknown", "No game endpoint node has been recorded; start the game before enabling sleep.", false) + return plan, nil + } + if nodeName != "" { + var node corev1.Node + err := reader.Get(ctx, client.ObjectKey{Name: nodeName}, &node) + if err != nil && !apierrors.IsNotFound(err) { + return plan, err + } + ready := false + for _, condition := range node.Status.Conditions { + if condition.Type == corev1.NodeReady { + ready = condition.Status == corev1.ConditionTrue + } + } + if err != nil || !node.DeletionTimestamp.IsZero() || !ready || string(node.UID) != gs.Annotations[hostportNodeUIDAnnotation] { + plan.blockGame = len(gamePods.Items) == 0 + set("EndpointNodeUnavailable", fmt.Sprintf("Hostport endpoint node %q is missing, replaced or not Ready; restore that node or explicitly migrate the endpoint.", nodeName), false) + return plan, nil + } + } + if idle == idleAsleep { + var ss appsv1.StatefulSet + err := reader.Get(ctx, client.ObjectKeyFromObject(gs), &ss) + if err != nil && !apierrors.IsNotFound(err) { + return plan, err + } + if len(gamePods.Items) != 0 || (err == nil && (ss.Spec.Replicas == nil || *ss.Spec.Replicas != 0 || ss.Status.ObservedGeneration < ss.Generation || ss.Status.Replicas != 0)) { + set("GameDraining", "Waiting for the game Pod to disappear before acquiring its Hostport.", false) + return plan, nil + } + plan.wantSentinel = true + plan.hostportCondition = hostportListenerCondition(gs, sentinelPods.Items, nodeName) + plan.routeToSentinel = plan.hostportCondition.Status == metav1.ConditionTrue + return plan, nil + } + var dep appsv1.Deployment + err := reader.Get(ctx, client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-waker"}, &dep) + if err != nil && !apierrors.IsNotFound(err) { + return plan, err + } + if err == nil || len(sentinelPods.Items) != 0 { + plan.blockGame = true + set("SentinelDraining", "Waiting for the sentinel Deployment and Pods to disappear before scheduling the game on its endpoint node.", false) + return plan, nil + } + plan.hostportCondition = hostportListenerCondition(gs, gamePods.Items, nodeName) + return plan, nil +} + +func hostportListenerCondition(gs *gameplanev1alpha1.GameServer, pods []corev1.Pod, nodeName string) *metav1.Condition { + for _, pod := range pods { + if !pod.DeletionTimestamp.IsZero() { + continue + } + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodScheduled && condition.Status == corev1.ConditionFalse && condition.Reason == corev1.PodReasonUnschedulable { + return hostportCondition(gs, "EndpointUnschedulable", fmt.Sprintf("Hostport listener cannot schedule on endpoint node %q: %s", nodeName, condition.Message), false) + } + if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue && (nodeName == "" || pod.Spec.NodeName == nodeName) { + return hostportCondition(gs, "ListenerReady", "The listener is Ready on the Hostport endpoint node.", true) + } + } + } + return hostportCondition(gs, "ListenerPending", fmt.Sprintf("Waiting for a Ready listener on endpoint node %q.", nodeName), false) +} + +// hostportAffinity ANDs node identity into every OR alternative, retaining +// user selectors, pod affinity and anti-affinity. Empty selectors still match +// no nodes; setting PodSpec.NodeName would bypass the scheduler's port checks. +func hostportAffinity(gs *gameplanev1alpha1.GameServer) *corev1.Affinity { + affinity := gs.Spec.Affinity.DeepCopy() + node := gs.Annotations[hostportNodeAnnotation] + if gs.Spec.Networking.Expose != "Hostport" || node == "" { + return affinity + } + if affinity == nil { + affinity = &corev1.Affinity{} + } + if affinity.NodeAffinity == nil { + affinity.NodeAffinity = &corev1.NodeAffinity{} + } + if affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil { + affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = &corev1.NodeSelector{NodeSelectorTerms: []corev1.NodeSelectorTerm{{}}} + // This newly created term intentionally matches the endpoint node. + affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchFields = []corev1.NodeSelectorRequirement{{Key: "metadata.name", Operator: corev1.NodeSelectorOpIn, Values: []string{node}}} + return affinity + } + for i := range affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms { + term := &affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[i] + if len(term.MatchExpressions) == 0 && len(term.MatchFields) == 0 { + continue + } + term.MatchFields = append(term.MatchFields, corev1.NodeSelectorRequirement{Key: "metadata.name", Operator: corev1.NodeSelectorOpIn, Values: []string{node}}) + } + return affinity +} diff --git a/operator/internal/controller/gameserver_hostport_test.go b/operator/internal/controller/gameserver_hostport_test.go new file mode 100644 index 000000000..fc19f5116 --- /dev/null +++ b/operator/internal/controller/gameserver_hostport_test.go @@ -0,0 +1,132 @@ +package controller + +import ( + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +func hostportFixture(t *testing.T) (*GameServerReconciler, *gameplanev1alpha1.GameServer, *gameplanev1alpha1.GameTemplate) { + t.Helper() + s := wipeScheme(t) + if err := appsv1.AddToScheme(s); err != nil { + t.Fatal(err) + } + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "game", Namespace: "games", UID: "game-uid"}, Spec: gameplanev1alpha1.GameServerSpec{Networking: gameplanev1alpha1.GameServerNetworking{Expose: "Hostport"}, Idle: &gameplanev1alpha1.IdleSpec{Enabled: true, WakeOnConnect: true}}} + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-a", UID: "node-uid"}, Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}}} + zero := int32(0) + ss := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: gs.Name, Namespace: gs.Namespace}, Spec: appsv1.StatefulSetSpec{Replicas: &zero}} + c := fake.NewClientBuilder().WithScheme(s).WithObjects(gs, node, ss).Build() + return &GameServerReconciler{Client: c, APIReader: c, Scheme: s}, gs, singlePortTemplate(true, "generic") +} + +func TestHostportSleepCapturesNodeAndWaitsForActualPodDeletion(t *testing.T) { + r, gs, tmpl := hostportFixture(t) + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "game-0", Namespace: gs.Namespace, Labels: map[string]string{"app.kubernetes.io/name": "gameplane-game", "app.kubernetes.io/instance": gs.Name}, Finalizers: []string{"test/drain"}}, Spec: corev1.PodSpec{NodeName: "node-a"}} + if err := r.Create(t.Context(), pod); err != nil { + t.Fatal(err) + } + if err := r.Delete(t.Context(), pod); err != nil { + t.Fatal(err) + } + plan, err := r.planSentinel(t.Context(), gs, tmpl, idleAsleep) + if err != nil { + t.Fatal(err) + } + if plan.wantSentinel || plan.hostportCondition.Reason != "GameDraining" { + t.Fatalf("plan before actual deletion: %+v", plan) + } + var saved gameplanev1alpha1.GameServer + if err := r.Get(t.Context(), client.ObjectKeyFromObject(gs), &saved); err != nil { + t.Fatal(err) + } + if saved.Annotations[hostportNodeAnnotation] != "node-a" || saved.Annotations[hostportNodeUIDAnnotation] != "node-uid" { + t.Fatalf("durable endpoint not saved: %v", saved.Annotations) + } + if err := r.Get(t.Context(), client.ObjectKeyFromObject(pod), pod); err != nil { + t.Fatal(err) + } + pod.Finalizers = nil + if err := r.Update(t.Context(), pod); err != nil { + t.Fatal(err) + } + plan, err = r.planSentinel(t.Context(), &saved, tmpl, idleAsleep) + if err != nil || !plan.wantSentinel { + t.Fatalf("plan after actual deletion: %+v, %v", plan, err) + } +} + +func TestHostportWakeUsesUncachedPodsAndWaitsForProducerDeletion(t *testing.T) { + r, gs, tmpl := hostportFixture(t) + gs.Annotations = map[string]string{hostportNodeAnnotation: "node-a", hostportNodeUIDAnnotation: "node-uid"} + dep := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: gs.Name + "-waker", Namespace: gs.Namespace}} + if err := r.Create(t.Context(), dep); err != nil { + t.Fatal(err) + } + // A cached view with no Pods must not authorize transfer while the live + // API still has a terminating listener, even after its Deployment is gone. + live := fake.NewClientBuilder().WithScheme(r.Scheme).WithObjects(dep, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "old-waker", Namespace: gs.Namespace, Labels: map[string]string{sentinelLabel: sentinelValue, "app.kubernetes.io/instance": gs.Name}}}, &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-a", UID: "node-uid"}, Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}}}).Build() + r.APIReader = live + plan, err := r.planSentinel(t.Context(), gs, tmpl, idleAwake) + if err != nil || !plan.blockGame || plan.wantSentinel { + t.Fatalf("producer still live: %+v, %v", plan, err) + } + if err := live.Delete(t.Context(), dep); err != nil { + t.Fatal(err) + } + plan, err = r.planSentinel(t.Context(), gs, tmpl, idleAwake) + if err != nil || !plan.blockGame { + t.Fatalf("live Pod bypassed by cache: %+v, %v", plan, err) + } + if err := live.Delete(t.Context(), &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "old-waker", Namespace: gs.Namespace}}); err != nil { + t.Fatal(err) + } + plan, err = r.planSentinel(t.Context(), gs, tmpl, idleAwake) + if err != nil || plan.blockGame { + t.Fatalf("drained listener still blocks: %+v, %v", plan, err) + } +} + +func TestHostportNodeLossAndUnschedulableListenerStayUnavailable(t *testing.T) { + r, gs, tmpl := hostportFixture(t) + gs.Annotations = map[string]string{hostportNodeAnnotation: "node-a", hostportNodeUIDAnnotation: "node-uid"} + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "waker", Namespace: gs.Namespace, Labels: map[string]string{sentinelLabel: sentinelValue, "app.kubernetes.io/instance": gs.Name}}, Status: corev1.PodStatus{Conditions: []corev1.PodCondition{{Type: corev1.PodScheduled, Status: corev1.ConditionFalse, Reason: "Unschedulable", Message: "Insufficient memory"}}}} + if err := r.Create(t.Context(), pod); err != nil { + t.Fatal(err) + } + plan, err := r.planSentinel(t.Context(), gs, tmpl, idleAsleep) + if err != nil || plan.routeToSentinel || plan.hostportCondition.Reason != "EndpointUnschedulable" { + t.Fatalf("full node: %+v, %v", plan, err) + } + if err := r.Delete(t.Context(), &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-a"}}); err != nil { + t.Fatal(err) + } + plan, err = r.planSentinel(t.Context(), gs, tmpl, idleAsleep) + if err != nil || plan.wantSentinel || plan.hostportCondition.Reason != "EndpointNodeUnavailable" { + t.Fatalf("lost node: %+v, %v", plan, err) + } +} + +func TestHostportAffinityConjoinsEveryAlternativeAndPreservesOtherConstraints(t *testing.T) { + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{hostportNodeAnnotation: "node-a"}}, Spec: gameplanev1alpha1.GameServerSpec{Networking: gameplanev1alpha1.GameServerNetworking{Expose: "Hostport"}, Affinity: &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{NodeSelectorTerms: []corev1.NodeSelectorTerm{{MatchExpressions: []corev1.NodeSelectorRequirement{{Key: "pool", Operator: corev1.NodeSelectorOpIn, Values: []string{"a"}}}}, {MatchExpressions: []corev1.NodeSelectorRequirement{{Key: "pool", Operator: corev1.NodeSelectorOpIn, Values: []string{"b"}}}}}}}, PodAntiAffinity: &corev1.PodAntiAffinity{}}}} + got := hostportAffinity(gs) + for _, term := range got.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms { + if len(term.MatchFields) != 1 || term.MatchFields[0].Key != "metadata.name" || term.MatchFields[0].Values[0] != "node-a" { + t.Fatalf("unpinned alternative: %+v", term) + } + } + if got.PodAntiAffinity == nil || len(gs.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchFields) != 0 { + t.Fatal("lost constraints or mutated CR") + } + // An explicitly empty selector matches no nodes; pinning must not widen it. + gs.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = nil + if len(hostportAffinity(gs).NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms) != 0 { + t.Fatal("empty selector widened") + } +} diff --git a/operator/internal/controller/gameserver_sentinel.go b/operator/internal/controller/gameserver_sentinel.go index b2c8a9edc..bb145738e 100644 --- a/operator/internal/controller/gameserver_sentinel.go +++ b/operator/internal/controller/gameserver_sentinel.go @@ -102,8 +102,10 @@ func wakeOnConnectEligible(gs *gameplanev1alpha1.GameServer, tmpl *gameplanev1al // whether it should exist, and whether the game Service should route to it // instead of the game pod. type sentinelPlan struct { - wantSentinel bool - routeToSentinel bool + wantSentinel bool + routeToSentinel bool + blockGame bool + hostportCondition *metav1.Condition } // planSentinel is the single source of truth for the sentinel/game-Service @@ -132,9 +134,10 @@ type sentinelPlan struct { // terminationGracePeriodSeconds plus the sentinel's shutdown drain // (sentinelShutdownDrainTimeout) keep any handed-through session alive // until it ends on its own; only new connections stop reaching it. -// - Hostport: the sentinel and the game pod bind the exact same host -// port, so the sentinel cannot outlive the wake decision — the game -// pod cannot even schedule until the sentinel releases that port. The +// - Hostport: planHostportSentinel records the endpoint node and waits +// for actual game Pod deletion before creating a pinned sentinel. On +// wake it waits for foreground sentinel deletion and actual Pod drain +// before permitting the game to start on that same node. The // sentinel is expected to bounce any clients it is holding with a // protocol-native "waking up, try again shortly" response before the // operator tears it down here; players simply reconnect once the game @@ -142,6 +145,22 @@ type sentinelPlan struct { func (r *GameServerReconciler) planSentinel( ctx context.Context, gs *gameplanev1alpha1.GameServer, tmpl *gameplanev1alpha1.GameTemplate, idle idleState, ) (sentinelPlan, error) { + if gs.Spec.Networking.Expose == "Hostport" { + eligible := wakeOnConnectEligible(gs, tmpl) + if !eligible { + idle = idleAwake + } + plan, err := r.planHostportSentinel(ctx, gs, idle) + if err != nil { + return plan, err + } + // A Hostport server that has never used wake keeps its existing + // status behavior. Still drain a leftover sentinel when disabling it. + if !eligible && gs.Annotations[hostportNodeAnnotation] == "" && !plan.blockGame { + return sentinelPlan{}, nil + } + return plan, nil + } if !wakeOnConnectEligible(gs, tmpl) { return sentinelPlan{}, nil } @@ -157,10 +176,6 @@ func (r *GameServerReconciler) planSentinel( // idle == idleAwake: either a wake was just decided, or the server was // never asleep to begin with — sentinelExists is false in the latter // case, so we return immediately without the extra StatefulSet Get. - if gs.Spec.Networking.Expose == "Hostport" { - // See the Hostport bullet above: no hold-open window, ever. - return sentinelPlan{}, nil - } exists, err := r.sentinelExists(ctx, gs.Namespace, gs.Name) if err != nil || !exists { return sentinelPlan{}, err @@ -206,7 +221,10 @@ func (r *GameServerReconciler) reconcileSentinel( want bool, ) error { if !want { - return r.deleteSentinel(ctx, gs.Namespace, gs.Name) + return r.deleteSentinel(ctx, gs.Namespace, gs.Name, gs.Spec.Networking.Expose == "Hostport") + } + if gs.Spec.Networking.Expose == "Hostport" && gs.Annotations[hostportNodeAnnotation] == "" { + return fmt.Errorf("hostport sentinel requires a persisted endpoint node") } // Create or update the sentinel Deployment. @@ -277,6 +295,16 @@ func (r *GameServerReconciler) reconcileSentinel( dep.Spec.Template.Spec.TerminationGracePeriodSeconds = &graceSeconds dep.Spec.Template.Spec.ServiceAccountName = sentinelSAName(gs) + if hostPort { + dep.Spec.Template.Spec.NodeSelector = gs.Spec.NodeSelector + dep.Spec.Template.Spec.Tolerations = gs.Spec.Tolerations + dep.Spec.Template.Spec.Affinity = hostportAffinity(gs) + dep.Spec.Strategy = appsv1.DeploymentStrategy{Type: appsv1.RecreateDeploymentStrategyType} + } else { + dep.Spec.Template.Spec.NodeSelector = nil + dep.Spec.Template.Spec.Tolerations = nil + dep.Spec.Template.Spec.Affinity = nil + } dep.Spec.Template.Spec.Containers = []corev1.Container{{ Name: "sentinel", Image: image, @@ -406,12 +434,15 @@ func (r *GameServerReconciler) sentinelIsReady(ctx context.Context, namespace, g } // deleteSentinel removes the sentinel Deployment if it exists. -func (r *GameServerReconciler) deleteSentinel(ctx context.Context, namespace, gsName string) error { +func (r *GameServerReconciler) deleteSentinel(ctx context.Context, namespace, gsName string, foreground ...bool) error { dep, exists, err := r.getSentinel(ctx, namespace, gsName) if err != nil || !exists { return err } policy := metav1.DeletePropagationBackground + if len(foreground) != 0 && foreground[0] { + policy = metav1.DeletePropagationForeground + } return client.IgnoreNotFound(r.Delete(ctx, dep, &client.DeleteOptions{PropagationPolicy: &policy})) } diff --git a/operator/internal/controller/gameserver_sentinel_envtest_test.go b/operator/internal/controller/gameserver_sentinel_envtest_test.go index e805d58c8..f756311b6 100644 --- a/operator/internal/controller/gameserver_sentinel_envtest_test.go +++ b/operator/internal/controller/gameserver_sentinel_envtest_test.go @@ -12,6 +12,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" ) @@ -182,7 +183,7 @@ func TestSentinelReconciliation(t *testing.T) { deleteCleanup(t, tmpl) gs := &gameplanev1alpha1.GameServer{ - ObjectMeta: metav1.ObjectMeta{Name: "test-sentinel-hostport", Namespace: ns}, + ObjectMeta: metav1.ObjectMeta{Name: "test-sentinel-hostport", Namespace: ns, Annotations: map[string]string{hostportNodeAnnotation: "node-a"}}, Spec: gameplanev1alpha1.GameServerSpec{ TemplateRef: gameplanev1alpha1.GameTemplateRef{Name: tmpl.Name}, Idle: armedIdleSpec(), @@ -550,8 +551,40 @@ func TestGameServer_WakeOnConnectHostportNoHoldOpen(t *testing.T) { return ssReplicas(t, ns, "wakehp") == 1, "statefulset not scaled up yet" }) setSSStatus(t, ns, "wakehp", 1, 1) + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: uniqueName("hostport-node")}} + if err := k8sClient.Create(ctx, node); err != nil { + t.Fatal(err) + } + deleteCleanup(t, node) + node.Status.Conditions = []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}} + if err := k8sClient.Status().Update(ctx, node); err != nil { + t.Fatal(err) + } + gamePod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "wakehp-0", Namespace: ns, Labels: map[string]string{"app.kubernetes.io/name": "gameplane-game", "app.kubernetes.io/instance": gs.Name}}, Spec: corev1.PodSpec{NodeName: node.Name, Containers: []corev1.Container{{Name: "game", Image: "busybox"}}}} + if err := k8sClient.Create(ctx, gamePod); err != nil { + t.Fatal(err) + } setAsleep(t, ns, "wakehp") + eventually(t, func() (bool, string) { return ssReplicas(t, ns, gs.Name) == 0, "game not scaled down" }) + consistently(t, time.Second, func() (bool, string) { + return !deploymentExists(t, ns, "wakehp-waker"), "sentinel overlaps the live game Pod" + }) + zero := int64(0) + if err := k8sClient.Delete(ctx, gamePod, &client.DeleteOptions{GracePeriodSeconds: &zero}); err != nil { + t.Fatal(err) + } + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + var ss appsv1.StatefulSet + if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: gs.Name}, &ss); err != nil { + return err + } + ss.Status.Replicas, ss.Status.ReadyReplicas = 0, 0 + ss.Status.ObservedGeneration = ss.Generation + return k8sClient.Status().Update(ctx, &ss) + }); err != nil { + t.Fatal(err) + } eventually(t, func() (bool, string) { return deploymentExists(t, ns, "wakehp-waker"), "sentinel deployment not created" }) @@ -565,6 +598,23 @@ func TestGameServer_WakeOnConnectHostportNoHoldOpen(t *testing.T) { clearAsleep(t, ns, "wakehp") eventually(t, func() (bool, string) { - return !deploymentExists(t, ns, "wakehp-waker"), "sentinel not deleted immediately on Hostport wake" + var dep appsv1.Deployment + err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: "wakehp-waker"}, &dep) + return err == nil && !dep.DeletionTimestamp.IsZero(), "sentinel foreground deletion not requested" + }) + consistently(t, time.Second, func() (bool, string) { + return ssReplicas(t, ns, gs.Name) == 0, "game restarted before sentinel producer disappeared" + }) + // envtest has no garbage collector: finish foreground deletion explicitly. + var dep appsv1.Deployment + if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: "wakehp-waker"}, &dep); err != nil { + t.Fatal(err) + } + dep.Finalizers = nil + if err := k8sClient.Update(ctx, &dep); err != nil { + t.Fatal(err) + } + eventually(t, func() (bool, string) { + return ssReplicas(t, ns, gs.Name) == 1, "game did not resume after sentinel deletion" }) } diff --git a/operator/internal/controller/gameserver_status.go b/operator/internal/controller/gameserver_status.go index 54393de0b..4a470c122 100644 --- a/operator/internal/controller/gameserver_status.go +++ b/operator/internal/controller/gameserver_status.go @@ -43,6 +43,7 @@ func (r *GameServerReconciler) reconcileStatus( ctx context.Context, gs *gameplanev1alpha1.GameServer, idle idleState, idleStatus *gameplanev1alpha1.IdleStatus, idleScheduleErr error, tunnelPlan tunnelPlan, tmpl *gameplanev1alpha1.GameTemplate, svcEvents []corev1.Event, conflictingServer string, + hostportConditions ...*metav1.Condition, ) (time.Duration, error) { // base captures the object as fetched so we can issue a JSON merge // patch of only the fields this reconciler owns. The agent sidecar @@ -194,6 +195,16 @@ func (r *GameServerReconciler) reconcileStatus( // Compute tunnel conditions. gs.Status.Conditions = computeTunnelConditions(gs, tunnelPlan, tunnelDep) + if len(hostportConditions) != 0 && hostportConditions[0] != nil { + condition := *hostportConditions[0] + gs.Status.Conditions = upsertCondition(gs.Status.Conditions, condition) + if condition.Status != metav1.ConditionTrue { + condition.Type = "Ready" + gs.Status.Conditions = upsertCondition(gs.Status.Conditions, condition) + } + } else { + gs.Status.Conditions = removeCondition(gs.Status.Conditions, hostportWakeCondition) + } // Validate and merge playit tunnel endpoints if present. var playitErr string diff --git a/operator/specs.md b/operator/specs.md index c7aea8225..0fc1414e4 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -30,6 +30,24 @@ start/stop and expiry remain reconciler responsibilities; the gateway cannot bypass them or operate an arbitrary pod. Existing game resources and storage are not recreated to enable remote parity. +### Hostport wake handoff + +Hostport wake-on-connect keeps a durable endpoint node name and Node UID in +`gameplane.local/hostport-node` / `gameplane.local/hostport-node-uid`, captured from +the live game Pod before the first sleep. The current-node dashboard annotation +is independent. Sleep waits for the zero-replica StatefulSet generation to be +observed and for actual game Pod disappearance through the uncached API reader. +Wake foreground-deletes the sentinel and waits for its Deployment and all Pods +to disappear before scheduling the game. Both successor templates use required +node affinity for the recorded endpoint, combined with the game's scheduling +constraints; the sentinel carries its tolerations. Node gets are read-only RBAC. +`HostportWakeReady` reports listener readiness, each drain and unavailable or +unschedulable endpoints without blocking unrelated reconciliation. There is no +automatic endpoint migration. See the updated +[July handoff design](../docs/superpowers/specs/2026-07-24-wake-on-connect-design.md#sequencing-and-port-handover) +for recovery/migration steps. Service-backed wake retains its established-session +drain behavior. Auxiliary resource defaults are defined separately. + ## Non-goals / boundaries The operator **does not** handle: diff --git a/sentinel/main.go b/sentinel/main.go index edc563b4d..2eaec4afa 100644 --- a/sentinel/main.go +++ b/sentinel/main.go @@ -72,7 +72,8 @@ const ( // defaultPollInterval is how often waitForUpstream retries dialing the // game-direct Service while holding a connection open. - defaultPollInterval = 250 * time.Millisecond + defaultPollInterval = 250 * time.Millisecond + defaultWakePatchTimeout = 5 * time.Second // defaultShutdownDrainTimeout bounds how long the sentinel keeps // in-flight sessions alive after it is told to shut down (see @@ -354,44 +355,68 @@ type wakeRequester interface { // waker patches the GameServer's idle-wake-requested annotation to signal // the operator that this server should wake. // -// It coalesces bursts: while a patch is in flight, or within minInterval of -// the last one landing, RequestWake is a cheap no-op instead of issuing -// another apiserver call. This is what keeps a port scan, or a pile of +// It coalesces bursts: callers share the result of an in-flight patch, and +// successful patches suppress another call for minInterval. This keeps a port scan, or a pile of // players reconnecting at once, from hammering the apiserver with patches — // the operator only needs to see the annotation change once to act on it. type waker struct { - res dynamic.ResourceInterface - name string - minInterval time.Duration + res dynamic.ResourceInterface + name string + minInterval time.Duration + patchTimeout time.Duration mu sync.Mutex - inFlight bool + inFlight *wakeAttempt lastPatch time.Time } +type wakeAttempt struct { + done chan struct{} + err error +} + func newWaker(res dynamic.ResourceInterface, name string, minInterval time.Duration) *waker { - return &waker{res: res, name: name, minInterval: minInterval} + return &waker{res: res, name: name, minInterval: minInterval, patchTimeout: defaultWakePatchTimeout} } // RequestWake stamps a fresh token onto the GameServer's // idle-wake-requested annotation via a JSON merge patch (rather than a full // object update), so any other annotations already on the object survive. -func (w *waker) RequestWake(ctx context.Context) error { +func (w *waker) RequestWake(ctx context.Context) (result error) { + if err := ctx.Err(); err != nil { + return err + } w.mu.Lock() now := time.Now() - if w.inFlight || (!w.lastPatch.IsZero() && now.Sub(w.lastPatch) < w.minInterval) { + if attempt := w.inFlight; attempt != nil { + w.mu.Unlock() + select { + case <-ctx.Done(): + return ctx.Err() + case <-attempt.done: + return attempt.err + } + } + if !w.lastPatch.IsZero() && now.Sub(w.lastPatch) < w.minInterval { w.mu.Unlock() return nil } - w.inFlight = true + attempt := &wakeAttempt{done: make(chan struct{})} + w.inFlight = attempt w.mu.Unlock() defer func() { w.mu.Lock() - w.inFlight = false - w.lastPatch = time.Now() + attempt.err = result + w.inFlight = nil + if result == nil { + w.lastPatch = time.Now() + } + close(attempt.done) w.mu.Unlock() }() + patchCtx, cancel := context.WithTimeout(ctx, w.patchTimeout) + defer cancel() patch, err := json.Marshal(map[string]any{ "metadata": map[string]any{ @@ -404,7 +429,7 @@ func (w *waker) RequestWake(ctx context.Context) error { return fmt.Errorf("encode wake patch: %w", err) } - if _, err := w.res.Patch(ctx, w.name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + if _, err := w.res.Patch(patchCtx, w.name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { return fmt.Errorf("patch %s annotation: %w", idleWakeRequestedAnnotation, err) } return nil @@ -714,17 +739,20 @@ type joinParams struct { // bidirectionally; on timeout, the connection is bounced (if a bounce // function is set) and closed. func handleJoin(ctx context.Context, conn net.Conn, br *bufio.Reader, w wakeRequester, p joinParams) { - if err := w.RequestWake(ctx); err != nil { + joinCtx, cancelJoin := context.WithTimeout(ctx, p.deadline) + defer cancelJoin() + if err := w.RequestWake(joinCtx); err != nil { log.Printf("request wake: %v", err) } - upstream, err := waitForUpstream(ctx, p.upstreamAddr, p.deadline, p.pollInterval) + upstream, err := waitForUpstream(joinCtx, p.upstreamAddr, p.deadline, p.pollInterval) if err != nil { if p.bounce != nil { p.bounce(conn) } return } + cancelJoin() defer func() { if err := upstream.Close(); err != nil && !errors.Is(err, net.ErrClosed) { log.Printf("close upstream connection: %v", err) @@ -842,6 +870,11 @@ func serveUDP(ctx context.Context, port PortConfig, w wakeRequester, cfg Config, } return } + defer func() { _ = pc.Close() }() + serveUDPPackets(ctx, pc, port, w, cfg, errCh) +} + +func serveUDPPackets(ctx context.Context, pc net.PacketConn, port PortConfig, w wakeRequester, cfg Config, errCh chan<- error) { // Belt-and-suspenders shutdown: Close() unblocks a ReadFrom that's // already parked waiting for a packet, and the read deadline below @@ -858,6 +891,11 @@ func serveUDP(ctx context.Context, port PortConfig, w wakeRequester, cfg Config, sweepCtx, cancelSweep := context.WithCancel(ctx) defer cancelSweep() go heuristic.sweepLoop(sweepCtx, cfg.UDPWindow) + wakeWorker := newUDPWakeWorker(sweepCtx, w) + defer func() { + cancelSweep() + <-wakeWorker.done + }() buf := make([]byte, 2048) for { @@ -882,9 +920,7 @@ func serveUDP(ctx context.Context, port PortConfig, w wakeRequester, cfg Config, } if heuristic.shouldWake(addr.String(), time.Now()) { - if err := w.RequestWake(ctx); err != nil { - log.Printf("request wake: %v", err) - } + wakeWorker.request() } } } diff --git a/sentinel/specs.md b/sentinel/specs.md index e39166621..3d1a6305d 100644 --- a/sentinel/specs.md +++ b/sentinel/specs.md @@ -59,7 +59,7 @@ Single executable module; no subdirectories or packages. - `none`: Operator still advertises the port (container spec, Service), but the sentinel doesn't listen on it (port scan sink). - **Startup validation:** At startup, all configured `wakeProtocol` values (except `generic` and `none`) are validated against the gameproto registry. Unknown protocol names cause a fatal startup error listing available registered protocols. -- **`WAKE_DEADLINE`** (optional, default `25s`): How long to hold a join connection waiting for the game pod to come up. Must be less than the game client's own timeout (Minecraft ~30s, Terraria ~30s) or the client gives up first. +- **`WAKE_DEADLINE`** (optional, default `25s`): Total join budget, starting before the wake PATCH and including upstream dialing. The PATCH is bounded by the lesser of five seconds and the remaining join budget. Must be less than the game client's own timeout (Minecraft ~30s, Terraria ~30s) or the client gives up first. After a successful upstream connection, proxying uses the session/drain context and outlives this budget. - **`UDP_PACKET_THRESHOLD`** (optional, default `3`): Number of packets from one source within the window before triggering a wake. - **`UDP_PACKET_WINDOW`** (optional, default `10s`): Time window for counting UDP packets. - **`UDP_MAX_SOURCES`** (optional, default `4096`): Maximum number of distinct UDP sources (keyed by `ip:port`, not just IP) to track. Older sources are evicted when this limit is reached. @@ -246,7 +246,7 @@ Each accepted TCP connection runs as a *session* under its own context, separate 5. **No data duplication or loss.** The complete original stream is: `Consumed + remaining(bufio.Reader) + new-data-from-client`. Both classifiers and proxying preserve this boundary. -6. **Annotation patch coalesces bursts.** Bursts of players reconnecting at once don't hammer the apiserver; RequestWake is a no-op if a patch is already in flight or within `wakePatchInterval` of the last one. +6. **Annotation patch coalesces bursts.** Callers awaiting an in-flight PATCH observe that attempt's completion and error, unless their own context expires first. Only successful attempts start the `wakePatchInterval` cooldown; failures release the shared attempt so a later request can retry. One worker per UDP listener accepts a one-entry coalescing queue; packet reception and bounded source tracking continue while Kubernetes is stalled. Listener shutdown cancels and joins that worker. 7. **UDP sources are evicted fairly.** The least-recently-active source is evicted (LRU), not a random or oldest source, so brief bursts of scanners don't unfairly starve legitimate players trying to rejoin. @@ -293,6 +293,7 @@ Each accepted TCP connection runs as a *session* under its own context, separate - **Shutdown drain (F-179):** `TestRunDrainsProxiedSessionAfterContextCancel` (replaces `TestProxyBidirectionalStopsOnContextCancel`, which asserted the old cut-on-cancel behaviour) verifies that after cancel new connections are refused while a handed-through session keeps working in both directions and `run` returns only once it ends; `TestRunForceClosesSessionsAfterDrainTimeout` verifies the drain is bounded; `TestLoadConfigShutdownDrainTimeout` covers `SHUTDOWN_DRAIN_TIMEOUT` parsing. - **Listener failure (F-180):** `TestRunSurfacesUDPListenFailureWhileTCPHealthy` verifies a UDP bind failure is returned promptly while a TCP listener is still healthy. - **Waker (annotation patching):** `TestWakerRequestWakeSetsAnnotation`, `TestWakerRequestWakePreservesOtherAnnotations`, `TestWakerRequestWakeCoalescesBursts` verify merge patch isolation, burst coalescing. +- **Wake availability:** `wake_budget_test.go` covers a hanging HTTP API, shared failure and retry, the total join deadline, worker cancellation/coalescing, successful gameplay beyond the join deadline, and receiving a 101-datagram burst while wake is stalled. CI logs elapsed receive time; this fixture does not establish a production capacity ceiling. - **Registry dispatch:** `TestRegistryProtocolDispatchMinecraftJoinWakes`, `TestRegistryProtocolDispatchMinecraftStatusPingNoWake`, `TestRegistryProtocolDispatchTerrariaJoinWakes`, `TestRegistryProtocolDispatchUnknownNoWake`, `TestRegistryProtocolReplayConsumedBytes` verify registry lookup, Classifier.Classify invocation, protocol-agnostic join/status/unknown handling, and consumed-byte replay. - **Startup validation:** `TestParsePortsConfigRejectsUnknownProtocol` verifies that unknown protocol names cause a fatal startup error with a clear message listing available protocols. - **Generic handlers:** `TestHandleGenericWakesOnConnect`, `TestHandleGenericClosesSilentlyOnDeadline` verify generic TCP (always join), generic UDP bounce (no message). diff --git a/sentinel/udp_wake.go b/sentinel/udp_wake.go new file mode 100644 index 000000000..3b8a9b29a --- /dev/null +++ b/sentinel/udp_wake.go @@ -0,0 +1,41 @@ +package main + +import ( + "context" + "log" +) + +type udpWakeWorker struct { + requests chan struct{} + done chan struct{} +} + +func newUDPWakeWorker(ctx context.Context, w wakeRequester) *udpWakeWorker { + worker := &udpWakeWorker{requests: make(chan struct{}, 1), done: make(chan struct{})} + go func() { + defer close(worker.done) + for { + select { + case <-ctx.Done(): + return + case <-worker.requests: + if ctx.Err() != nil { + return + } + wakeCtx, cancel := context.WithTimeout(ctx, defaultWakePatchTimeout) + if err := w.RequestWake(wakeCtx); err != nil { + log.Printf("request UDP wake: %v", err) + } + cancel() + } + } + }() + return worker +} + +func (w *udpWakeWorker) request() { + select { + case w.requests <- struct{}{}: + default: + } +} diff --git a/sentinel/wake_budget_test.go b/sentinel/wake_budget_test.go new file mode 100644 index 000000000..e7b5646d0 --- /dev/null +++ b/sentinel/wake_budget_test.go @@ -0,0 +1,264 @@ +package main + +import ( + "bufio" + "context" + "errors" + "io" + "net" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" + ktesting "k8s.io/client-go/testing" +) + +func TestWakeSharedFailureAndImmediateRetry(t *testing.T) { + w, client := newFakeGameServerWaker(t, "alpha", "games", time.Minute, nil) + started, release := make(chan struct{}), make(chan struct{}) + var patches atomic.Int32 + failure := errors.New("API unavailable") + client.PrependReactor("patch", "gameservers", func(ktesting.Action) (bool, runtime.Object, error) { + if patches.Add(1) == 1 { + close(started) + <-release + return true, nil, failure + } + return false, nil, nil + }) + first, second := make(chan error, 1), make(chan error, 1) + go func() { first <- w.RequestWake(t.Context()) }() + <-started + waiting := &observedWaitContext{Context: t.Context(), observed: make(chan struct{})} + go func() { second <- w.RequestWake(waiting) }() + select { + case err := <-second: + close(release) + t.Fatalf("in-flight waiter returned before wake completed: %v", err) + case <-waiting.observed: + case <-time.After(time.Second): + close(release) + t.Fatal("second wake request never reached its wait") + } + close(release) + for _, result := range []<-chan error{first, second} { + select { + case err := <-result: + if !errors.Is(err, failure) { + t.Fatalf("got %v, want shared failure", err) + } + case <-time.After(time.Second): + t.Fatal("wake waiter did not finish") + } + } + if err := w.RequestWake(t.Context()); err != nil { + t.Fatalf("retry: %v", err) + } + if patches.Load() != 2 { + t.Fatalf("got %d patches, want failed attempt and retry", patches.Load()) + } +} + +// Done is evaluated when RequestWake waits on the shared result. This handshake +// avoids relying on the scheduler running the second goroutine within a sleep. +type observedWaitContext struct { + context.Context + observed chan struct{} + once sync.Once +} + +func (c *observedWaitContext) Done() <-chan struct{} { + c.once.Do(func() { close(c.observed) }) + return c.Context.Done() +} + +func TestWakePatchHasIndependentTimeout(t *testing.T) { + release := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, req *http.Request) { + _, _ = io.Copy(io.Discard, req.Body) + select { + case <-req.Context().Done(): + case <-release: + } + })) + defer server.Close() + defer close(release) + client, err := dynamic.NewForConfig(&rest.Config{Host: server.URL}) + if err != nil { + t.Fatal(err) + } + w := newWaker(client.Resource(gameServerGVR).Namespace("games"), "alpha", time.Minute) + w.patchTimeout = 25 * time.Millisecond + started := time.Now() + if err := w.RequestWake(t.Context()); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("got %v, want deadline exceeded", err) + } + if time.Since(started) > time.Second { + t.Fatal("wake PATCH exceeded its own timeout") + } +} + +type contextBoundWake struct{ started chan struct{} } + +func (w *contextBoundWake) RequestWake(ctx context.Context) error { + close(w.started) + <-ctx.Done() + return ctx.Err() +} + +func TestJoinBudgetIncludesWakeRequest(t *testing.T) { + server, client := net.Pipe() + defer server.Close() + defer client.Close() + wake := &contextBoundWake{started: make(chan struct{})} + done := make(chan struct{}) + go func() { + defer close(done) + handleJoin(t.Context(), server, bufio.NewReader(server), wake, joinParams{upstreamAddr: "127.0.0.1:1", deadline: 25 * time.Millisecond, pollInterval: time.Millisecond}) + }() + <-wake.started + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("join deadline did not bound wake request") + } +} + +func TestUDPWakeWorkerCoalescesAndStops(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + wake := &contextBoundWake{started: make(chan struct{})} + worker := newUDPWakeWorker(ctx, wake) + worker.request() + <-wake.started + for range 1000 { + worker.request() + } + if len(worker.requests) > 1 { + t.Fatal("unbounded UDP wake backlog") + } + cancel() + select { + case <-worker.done: + case <-time.After(time.Second): + t.Fatal("UDP wake worker did not stop") + } +} + +func TestSuccessfulGameplayOutlivesJoinBudget(t *testing.T) { + upstream, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer upstream.Close() + accepted := make(chan struct{}) + go func() { + conn, acceptErr := upstream.Accept() + if acceptErr != nil { + return + } + defer conn.Close() + close(accepted) + _, _ = io.Copy(conn, conn) + }() + server, player := tcpPipe(t) + defer player.Close() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + done := make(chan struct{}) + go func() { + defer close(done) + defer server.Close() + handleJoin(ctx, server, bufio.NewReader(server), newCountingWaker(nil), joinParams{upstreamAddr: upstream.Addr().String(), deadline: 100 * time.Millisecond, pollInterval: time.Millisecond}) + }() + select { + case <-accepted: + case <-time.After(time.Second): + t.Fatal("join did not reach game") + } + // The session must retain the drain context after the short join context + // is cancelled, including after its original deadline has elapsed. + time.Sleep(200 * time.Millisecond) + _ = player.SetDeadline(time.Now().Add(time.Second)) + if _, err := player.Write([]byte("still playing")); err != nil { + t.Fatal(err) + } + got := make([]byte, len("still playing")) + if _, err := io.ReadFull(player, got); err != nil || string(got) != "still playing" { + t.Fatalf("gameplay after join budget: %q, %v", got, err) + } + cancel() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("session did not stop on drain cancellation") + } +} + +type observedPacketConn struct { + net.PacketConn + reads chan struct{} +} + +func (c observedPacketConn) ReadFrom(p []byte) (int, net.Addr, error) { + n, addr, err := c.PacketConn.ReadFrom(p) + if err == nil { + c.reads <- struct{}{} + } + return n, addr, err +} + +func TestUDPReceiveContinuesWhileWakeIsStalled(t *testing.T) { + listener, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + client, err := net.Dial("udp", listener.LocalAddr().String()) + if err != nil { + t.Fatal(err) + } + defer client.Close() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + wake := &contextBoundWake{started: make(chan struct{})} + reads := make(chan struct{}, 128) + done := make(chan struct{}) + go func() { + defer close(done) + serveUDPPackets(ctx, observedPacketConn{listener, reads}, PortConfig{}, wake, Config{UDPThreshold: 1, UDPWindow: time.Minute, UDPMaxSources: 10}, make(chan error, 1)) + }() + if _, err := client.Write([]byte("wake")); err != nil { + t.Fatal(err) + } + select { + case <-wake.started: + case <-time.After(time.Second): + t.Fatal("wake did not start") + } + started := time.Now() + for range 100 { + if _, err := client.Write([]byte("game packet")); err != nil { + t.Fatal(err) + } + } + for range 101 { + select { + case <-reads: + case <-time.After(time.Second): + t.Fatal("UDP receive blocked behind stalled wake PATCH") + } + } + t.Logf("received 101 datagrams while wake blocked in %s", time.Since(started)) + cancel() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("UDP listener/worker did not stop") + } +} diff --git a/test/e2e/buckets.sh b/test/e2e/buckets.sh index d60a26f14..cf09c7ae3 100755 --- a/test/e2e/buckets.sh +++ b/test/e2e/buckets.sh @@ -392,10 +392,15 @@ EOF # Tests that exist in the suite but deliberately run in NO bucket. Every # entry needs a reason; `verify` fails on any unlisted stray so additions # here are a conscious, reviewed act. Currently empty. +bucket_wake_hostport() { cat <<'EOF' +TestGameServer_HostportWakeStableEndpoint +EOF +} + unbucketed() { :; } bucket_names() { - printf '%s\n' operator api-auth api-roles api-rbac api-agent api-mods ratelimit bot-fast bot-heavy multicluster upgrade telemetry capture-overhead optional-components + printf '%s\n' operator api-auth api-roles api-rbac api-agent api-mods ratelimit bot-fast bot-heavy multicluster upgrade telemetry capture-overhead optional-components wake-hostport } list_bucket() { @@ -414,6 +419,7 @@ list_bucket() { telemetry) bucket_telemetry ;; capture-overhead) bucket_capture_overhead ;; optional-components) bucket_optional_components ;; + wake-hostport) bucket_wake_hostport ;; *) echo "unknown bucket: $1 (known: $(bucket_names | tr '\n' ' '))" >&2 exit 2 diff --git a/test/e2e/hostport_wake_e2e_test.go b/test/e2e/hostport_wake_e2e_test.go new file mode 100644 index 000000000..d0461dd76 --- /dev/null +++ b/test/e2e/hostport_wake_e2e_test.go @@ -0,0 +1,238 @@ +//go:build e2e + +package e2e + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" +) + +// This bucket owns its two-node cluster. It deliberately retains terminating +// Pods and changes node scheduling; it must not run with other game tests. +func TestGameServer_HostportWakeStableEndpoint(t *testing.T) { + ctx := t.Context() + const ns, name, tmpl, port = "gameplane-games", "hostport-wake", "hostport-wake-template", 12345 + nodes, err := envInstance.K8s.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + t.Fatal(err) + } + if len(nodes.Items) < 2 { + t.Fatal("wake-hostport requires the two-node kind profile") + } + var endpoint corev1.Node + for _, node := range nodes.Items { + if _, control := node.Labels["node-role.kubernetes.io/control-plane"]; !control { + endpoint = node + break + } + } + if endpoint.Name == "" { + t.Fatal("no worker node") + } + for _, node := range nodes.Items { + patch := []byte(`{"metadata":{"labels":{"gameplane.local/wake-pool":"test"}}}`) + if _, err := envInstance.K8s.CoreV1().Nodes().Patch(ctx, node.Name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + } + // The game and sentinel both need this dedicated-node toleration. + taints := append(append([]corev1.Taint{}, endpoint.Spec.Taints...), corev1.Taint{Key: "gameplane.local/dedicated", Value: "games", Effect: corev1.TaintEffectNoSchedule}) + patchHostportNode(t, endpoint.Name, map[string]any{"taints": taints}) + t.Cleanup(func() { + patchHostportNode(t, endpoint.Name, map[string]any{"unschedulable": false, "taints": endpoint.Spec.Taints}) + }) + address := "" + for _, addr := range endpoint.Status.Addresses { + if addr.Type == corev1.NodeInternalIP { + address = net.JoinHostPort(addr.Address, fmt.Sprint(port)) + break + } + } + if address == "" { + t.Fatal("worker has no InternalIP") + } + + applyBusyboxTemplate(t, tmpl) + // During deletion the shell keeps httpd alive for 12 seconds. This makes + // the old port genuinely occupied while the sleep handoff is observed. + templatePatch := map[string]any{"spec": map[string]any{ + "command": []string{"sh", "-c", "mkdir -p /tmp/site; echo endpoint-ok >/tmp/site/index.html; httpd -p 12345 -h /tmp/site; trap 'sleep 12; exit 0' TERM; while :; do sleep 1; done"}, + "ports": []any{map[string]any{"name": "game", "containerPort": port, "advertise": true, "protocol": "TCP", "wakeProtocol": "generic"}}, + }} + data, err := json.Marshal(templatePatch) + if err != nil { + t.Fatal(err) + } + if _, err := envInstance.Dyn.Resource(gameTemplateGVR).Patch(ctx, tmpl, types.MergePatchType, data, metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + gs := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "gameplane.local/v1alpha1", "kind": "GameServer", + "metadata": map[string]any{"name": name, "namespace": ns}, + "spec": map[string]any{"templateRef": map[string]any{"name": tmpl}, "networking": map[string]any{"expose": "Hostport"}, + "nodeSelector": map[string]any{"kubernetes.io/hostname": endpoint.Labels["kubernetes.io/hostname"]}, + "tolerations": []any{map[string]any{"key": "gameplane.local/dedicated", "operator": "Equal", "value": "games", "effect": "NoSchedule"}, map[string]any{"key": "node-role.kubernetes.io/control-plane", "operator": "Exists", "effect": "NoSchedule"}}, + "idle": map[string]any{"enabled": true, "afterMinutes": int64(60), "wakeOnConnect": true}}, + }} + if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Create(ctx, gs, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Delete(context.Background(), name, metav1.DeleteOptions{}) + }) + waitPVCBound(t, ns, name+"-data", 90*time.Second) + httpClient := &http.Client{Timeout: 2 * time.Second, Transport: &http.Transport{DisableKeepAlives: true}} + defer httpClient.CloseIdleConnections() + gameReachable := func() (bool, string) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://"+address, nil) + if err != nil { + return false, err.Error() + } + resp, err := httpClient.Do(request) + if err != nil { + return false, err.Error() + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + return err == nil && strings.Contains(string(body), "endpoint-ok"), string(body) + } + for cycle := range 2 { + envInstance.Eventually(t, 2*time.Minute, gameReachable) + pod, err := envInstance.K8s.CoreV1().Pods(ns).Get(ctx, name+"-0", metav1.GetOptions{}) + if err != nil || pod.Spec.NodeName != endpoint.Name { + t.Fatalf("game moved: pod=%v err=%v", pod, err) + } + patchHostportServer(t, ns, name, map[string]any{"metadata": map[string]any{"annotations": map[string]any{"gameplane.local/idle-asleep-since": time.Now().UTC().Format(time.RFC3339)}}}) + envInstance.Eventually(t, 20*time.Second, func() (bool, string) { + pod, err := envInstance.K8s.CoreV1().Pods(ns).Get(ctx, name+"-0", metav1.GetOptions{}) + return err == nil && !pod.DeletionTimestamp.IsZero(), fmt.Sprint(err) + }) + if ok, msg := gameReachable(); !ok { + t.Fatalf("fixture did not retain occupied old port during termination: %s", msg) + } + if _, err := envInstance.K8s.AppsV1().Deployments(ns).Get(ctx, name+"-waker", metav1.GetOptions{}); !apierrors.IsNotFound(err) { + t.Fatalf("sentinel created before old game released port: %v", err) + } + assertHostportCondition(t, ns, name, "GameDraining", false) + if cycle == 0 { + // Once the endpoint is recorded, broaden the worker pool to both + // nodes. Only the persisted required affinity can retain identity. + patchHostportServer(t, ns, name, map[string]any{"spec": map[string]any{"nodeSelector": map[string]any{"kubernetes.io/hostname": nil, "gameplane.local/wake-pool": "test"}}}) + patchHostportNode(t, endpoint.Name, map[string]any{"unschedulable": true}) + envInstance.Eventually(t, 90*time.Second, func() (bool, string) { return hostportConditionIs(t, ns, name, "EndpointUnschedulable", false) }) + // The alternate node is available; the listener must stay Pending + // on the pinned endpoint instead of silently moving there. + assertHostportPodsOnNode(t, ns, name, endpoint.Name) + patchHostportNode(t, endpoint.Name, map[string]any{"unschedulable": false}) + } + envInstance.Eventually(t, 90*time.Second, func() (bool, string) { return hostportConditionIs(t, ns, name, "ListenerReady", true) }) + assertHostportPodsOnNode(t, ns, name, endpoint.Name) + wakers, err := envInstance.K8s.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{LabelSelector: "app.kubernetes.io/name=gameplane-waker,app.kubernetes.io/instance=" + name}) + if err != nil || len(wakers.Items) != 1 { + t.Fatalf("sentinel Pods: %v, %v", wakers, err) + } + waker := wakers.Items[0] + finalizer := []byte(`{"metadata":{"finalizers":["test.gameplane.local/hold-handoff"]}}`) + if _, err := envInstance.K8s.CoreV1().Pods(ns).Patch(ctx, waker.Name, types.MergePatchType, finalizer, metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _, _ = envInstance.K8s.CoreV1().Pods(ns).Patch(context.Background(), waker.Name, types.MergePatchType, []byte(`{"metadata":{"finalizers":[]}}`), metav1.PatchOptions{}) + }) + // A connection to the same node address must reach the sentinel and + // wake the game. Hostport clients reconnect after this intentional bounce. + conn, err := (&net.Dialer{Timeout: 3 * time.Second}).DialContext(ctx, "tcp", address) + if err != nil { + t.Fatalf("sleeping endpoint unreachable: %v", err) + } + _ = conn.Close() + envInstance.Eventually(t, 30*time.Second, func() (bool, string) { return hostportConditionIs(t, ns, name, "SentinelDraining", false) }) + envInstance.Consistently(t, 3*time.Second, 200*time.Millisecond, func() (bool, string) { + _, err := envInstance.K8s.CoreV1().Pods(ns).Get(ctx, name+"-0", metav1.GetOptions{}) + return apierrors.IsNotFound(err), "game scheduled before terminating sentinel Pod disappeared" + }) + if _, err := envInstance.K8s.CoreV1().Pods(ns).Patch(ctx, waker.Name, types.MergePatchType, []byte(`{"metadata":{"finalizers":[]}}`), metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + envInstance.Eventually(t, 2*time.Minute, gameReachable) + assertHostportPodsOnNode(t, ns, name, endpoint.Name) + t.Logf("cycle %d kept reachable endpoint %s on %s", cycle+1, address, endpoint.Name) + } + // Real node deletion (including replacement under the same name) must + // report unavailable instead of migrating to the otherwise eligible node. + patchHostportServer(t, ns, name, map[string]any{"metadata": map[string]any{"annotations": map[string]any{"gameplane.local/idle-asleep-since": time.Now().UTC().Format(time.RFC3339)}}}) + if err := envInstance.K8s.CoreV1().Nodes().Delete(ctx, endpoint.Name, metav1.DeleteOptions{}); err != nil { + t.Fatal(err) + } + envInstance.Eventually(t, 45*time.Second, func() (bool, string) { return hostportConditionIs(t, ns, name, "EndpointNodeUnavailable", false) }) + assertHostportPodsOnNode(t, ns, name, endpoint.Name) +} + +func patchHostportServer(t *testing.T, ns, name string, patch map[string]any) { + t.Helper() + data, err := json.Marshal(patch) + if err != nil { + t.Fatal(err) + } + if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Patch(context.Background(), name, types.MergePatchType, data, metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } +} + +func patchHostportNode(t *testing.T, name string, spec map[string]any) { + t.Helper() + data, err := json.Marshal(map[string]any{"spec": spec}) + if err != nil { + t.Fatal(err) + } + if _, err := envInstance.K8s.CoreV1().Nodes().Patch(context.Background(), name, types.MergePatchType, data, metav1.PatchOptions{}); err != nil && !apierrors.IsNotFound(err) { + t.Error(err) + } +} + +func hostportConditionIs(t *testing.T, ns, name, reason string, ready bool) (bool, string) { + t.Helper() + gs, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Get(t.Context(), name, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + conditions, _, _ := unstructured.NestedSlice(gs.Object, "status", "conditions") + for _, item := range conditions { + condition, ok := item.(map[string]any) + if ok && condition["type"] == "HostportWakeReady" { + return condition["reason"] == reason && (condition["status"] == "True") == ready, fmt.Sprint(condition) + } + } + return false, "HostportWakeReady not yet present" +} + +func assertHostportCondition(t *testing.T, ns, name, reason string, ready bool) { + t.Helper() + envInstance.Eventually(t, 15*time.Second, func() (bool, string) { return hostportConditionIs(t, ns, name, reason, ready) }) +} + +func assertHostportPodsOnNode(t *testing.T, ns, name, node string) { + t.Helper() + pods, err := envInstance.K8s.CoreV1().Pods(ns).List(t.Context(), metav1.ListOptions{LabelSelector: "app.kubernetes.io/instance=" + name}) + if err != nil { + t.Fatal(err) + } + for _, pod := range pods.Items { + if pod.Spec.NodeName != "" && pod.Spec.NodeName != node { + t.Fatalf("listener %s moved to %s; endpoint remains %s", pod.Name, pod.Spec.NodeName, node) + } + } +} diff --git a/test/e2e/specs.md b/test/e2e/specs.md index abf7cda91..c3a640e66 100644 --- a/test/e2e/specs.md +++ b/test/e2e/specs.md @@ -19,6 +19,17 @@ It was one of three `go.work` modules without CI lint coverage until this featur - Validate multi-cluster dispatch, upgrade-from-previous-release, wake-on-connect, module signing/SSRF rejection, and backup/restore round trips end to end. - Enforce, via `buckets.sh`, that every test in the suite is assigned to exactly one CI bucket so nothing silently drops out of coverage. +### Hostport wake acceptance + +The dedicated `wake-hostport` bucket runs on the `two-node` kind profile. It uses +a real Hostport HTTP listener and dedicated-node taint, broadens the scheduling +pool after recording the endpoint, and exercises two sleep/wake cycles at the +same node address. It observes the old game still serving during termination, +holds a terminating sentinel Pod to prove the reverse handoff waits, checks a +cordoned endpoint remains unavailable rather than relocating, and deletes the +node to exercise the explicit unavailable/replaced-node condition. This bucket +runs alone because it changes node scheduling. Runtime results come from CI. + ### Remote gateway parity acceptance The multicluster bucket runs `TestMultiCluster_GatewayParity` against two real From 875b28cf76c687bd95ee7ed6377db209e0ee025a Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 21 Oct 2026 11:33:06 -0400 Subject: [PATCH 02/20] fix(operator): clarify Hostport waits and localize their conditions Signed-off-by: Baylem --- i18n/locales/en/app.json | 56 +++++++++++++++++++ operator/hack/condition-ids/main.go | 11 ++++ .../controller/gameserver_hostport.go | 17 +++--- 3 files changed, 76 insertions(+), 8 deletions(-) diff --git a/i18n/locales/en/app.json b/i18n/locales/en/app.json index 059470fd2..80079e373 100644 --- a/i18n/locales/en/app.json +++ b/i18n/locales/en/app.json @@ -263,6 +263,34 @@ "defaultMessage": "The state of the server is not known yet.", "description": "Healthy condition before the operator has observed the server." }, + "condition.hostportwakeready.endpointnodeunavailable": { + "defaultMessage": "The Hostport endpoint node is missing, replaced or not Ready; restore that node or explicitly migrate the endpoint.", + "description": "Hostport endpoint handoff status: EndpointNodeUnavailable." + }, + "condition.hostportwakeready.endpointnodeunknown": { + "defaultMessage": "No game endpoint node has been recorded; start the game before enabling sleep.", + "description": "Hostport endpoint handoff status: EndpointNodeUnknown." + }, + "condition.hostportwakeready.endpointunschedulable": { + "defaultMessage": "The Hostport listener cannot schedule on its endpoint node. Inspect the Pod scheduling events for details.", + "description": "Hostport endpoint handoff status: EndpointUnschedulable." + }, + "condition.hostportwakeready.gamedraining": { + "defaultMessage": "Waiting for the game Pod to disappear before acquiring its Hostport.", + "description": "Hostport endpoint handoff status: GameDraining." + }, + "condition.hostportwakeready.listenerpending": { + "defaultMessage": "Waiting for a Ready listener on the Hostport endpoint node.", + "description": "Hostport endpoint handoff status: ListenerPending." + }, + "condition.hostportwakeready.listenerready": { + "defaultMessage": "The listener is Ready on the Hostport endpoint node.", + "description": "Hostport endpoint handoff status: ListenerReady." + }, + "condition.hostportwakeready.sentineldraining": { + "defaultMessage": "Waiting for the sentinel Deployment and Pods to disappear before scheduling the game on its endpoint node.", + "description": "Hostport endpoint handoff status: SentinelDraining." + }, "condition.idlescheduleinvalid.wakewindowunparseable": { "defaultMessage": "A wake window in the idle schedule cannot be parsed: {details}", "description": "Condition when spec.idle has an invalid wake window. details is the English parse error." @@ -423,10 +451,26 @@ "defaultMessage": "The module content does not match its pinned digest.", "description": "Module condition when the digest differs from the pin." }, + "condition.ready.endpointnodeunavailable": { + "defaultMessage": "The Hostport endpoint node is missing, replaced or not Ready; restore that node or explicitly migrate the endpoint.", + "description": "Hostport endpoint handoff status: EndpointNodeUnavailable." + }, + "condition.ready.endpointnodeunknown": { + "defaultMessage": "No game endpoint node has been recorded; start the game before enabling sleep.", + "description": "Hostport endpoint handoff status: EndpointNodeUnknown." + }, + "condition.ready.endpointunschedulable": { + "defaultMessage": "The Hostport listener cannot schedule on its endpoint node. Inspect the Pod scheduling events for details.", + "description": "Hostport endpoint handoff status: EndpointUnschedulable." + }, "condition.ready.failed": { "defaultMessage": "{detail}", "description": "Ready condition when the server cannot start. detail is the English reason from the operator, for example a missing template or an invalid configuration." }, + "condition.ready.gamedraining": { + "defaultMessage": "Waiting for the game Pod to disappear before acquiring its Hostport.", + "description": "Hostport endpoint handoff status: GameDraining." + }, "condition.ready.idleasleep": { "defaultMessage": "The server is asleep because no players are online.", "description": "Ready condition for a server put to sleep by idle auto-sleep." @@ -443,6 +487,14 @@ "defaultMessage": "The module cannot be removed while game servers use its template.", "description": "Ready condition when deletion is blocked." }, + "condition.ready.listenerpending": { + "defaultMessage": "Waiting for a Ready listener on the Hostport endpoint node.", + "description": "Hostport endpoint handoff status: ListenerPending." + }, + "condition.ready.listenerready": { + "defaultMessage": "The listener is Ready on the Hostport endpoint node.", + "description": "Hostport endpoint handoff status: ListenerReady." + }, "condition.ready.noversionavailable": { "defaultMessage": "The module source has no available versions.", "description": "Module condition when no version can be chosen." @@ -463,6 +515,10 @@ "defaultMessage": "The server is running and the agent is reporting heartbeats.", "description": "Ready condition while the game server is up." }, + "condition.ready.sentineldraining": { + "defaultMessage": "Waiting for the sentinel Deployment and Pods to disappear before scheduling the game on its endpoint node.", + "description": "Hostport endpoint handoff status: SentinelDraining." + }, "condition.ready.servingstalecatalog": { "defaultMessage": "Serving the cached catalog. The last index attempt failed.", "description": "Ready condition while an older catalog stays available." diff --git a/operator/hack/condition-ids/main.go b/operator/hack/condition-ids/main.go index e644cf041..0fb74217b 100644 --- a/operator/hack/condition-ids/main.go +++ b/operator/hack/condition-ids/main.go @@ -46,6 +46,8 @@ type row struct { // guardCalls maps a helper that takes a condition reason as a string literal // to the index of that argument. var guardCalls = map[string]int{ + "hostportCondition": 1, + "hostportPending": 0, "markUnhealthy": 2, "markFailed": 2, "markPending": 2, @@ -120,6 +122,15 @@ var rows = []row{ {types: []string{"AddressAssignment"}, reason: "AssignmentPending", english: "Requested $REQ: the address manager has not assigned an address yet.", description: "AddressAssignment condition while the address manager works."}, {types: []string{"AddressAssignment"}, reason: "Assigned", english: "{assignedFrom, select, pool {Address {address} assigned from pool ''{pool}''.} other {Address {address} assigned.}}", description: "AddressAssignment condition once an address is assigned. assignedFrom is pool or default."}, + // Hostport endpoint handoff; Ready can mirror the same reason. + {types: []string{"HostportWakeReady", "Ready"}, reason: "EndpointNodeUnknown", english: "No game endpoint node has been recorded; start the game before enabling sleep.", description: "Hostport endpoint handoff status: EndpointNodeUnknown."}, + {types: []string{"HostportWakeReady", "Ready"}, reason: "EndpointNodeUnavailable", english: "The Hostport endpoint node is missing, replaced or not Ready; restore that node or explicitly migrate the endpoint.", description: "Hostport endpoint handoff status: EndpointNodeUnavailable."}, + {types: []string{"HostportWakeReady", "Ready"}, reason: "GameDraining", english: "Waiting for the game Pod to disappear before acquiring its Hostport.", description: "Hostport endpoint handoff status: GameDraining."}, + {types: []string{"HostportWakeReady", "Ready"}, reason: "SentinelDraining", english: "Waiting for the sentinel Deployment and Pods to disappear before scheduling the game on its endpoint node.", description: "Hostport endpoint handoff status: SentinelDraining."}, + {types: []string{"HostportWakeReady", "Ready"}, reason: "EndpointUnschedulable", english: "The Hostport listener cannot schedule on its endpoint node. Inspect the Pod scheduling events for details.", description: "Hostport endpoint handoff status: EndpointUnschedulable."}, + {types: []string{"HostportWakeReady", "Ready"}, reason: "ListenerReady", english: "The listener is Ready on the Hostport endpoint node.", description: "Hostport endpoint handoff status: ListenerReady."}, + {types: []string{"HostportWakeReady", "Ready"}, reason: "ListenerPending", english: "Waiting for a Ready listener on the Hostport endpoint node.", description: "Hostport endpoint handoff status: ListenerPending."}, + // GameServer: tunnel and other {types: []string{"TunnelReady"}, reason: "InvalidConfig", english: "{provider, select, none {The tunnel is enabled but the server has no tunnel configuration.} other {The ''{provider}'' tunnel provider is selected but its configuration is missing.}}", description: "TunnelReady condition for an incomplete tunnel setup. provider is the selected provider, or none."}, {types: []string{"TunnelReady"}, reason: "DeploymentNotReady", english: "Waiting for the tunnel deployment to become ready.", description: "TunnelReady condition while the tunnel pod is not ready."}, diff --git a/operator/internal/controller/gameserver_hostport.go b/operator/internal/controller/gameserver_hostport.go index 621fa9cac..0ff7464a8 100644 --- a/operator/internal/controller/gameserver_hostport.go +++ b/operator/internal/controller/gameserver_hostport.go @@ -54,8 +54,8 @@ func (r *GameServerReconciler) planHostportSentinel(ctx context.Context, gs *gam } } plan := sentinelPlan{} - set := func(reason, message string, ready bool) { - plan.hostportCondition = hostportCondition(gs, reason, message, ready) + hostportPending := func(reason, message string) { + plan.hostportCondition = hostportCondition(gs, reason, message, false) } nodeName := gs.Annotations[hostportNodeAnnotation] // Record the endpoint before the first sleep scales down the game. Doing @@ -87,13 +87,14 @@ func (r *GameServerReconciler) planHostportSentinel(ctx context.Context, gs *gam } } if nodeName == "" && idle == idleAsleep { - set("EndpointNodeUnknown", "No game endpoint node has been recorded; start the game before enabling sleep.", false) + hostportPending("EndpointNodeUnknown", "No game endpoint node has been recorded; start the game before enabling sleep.") return plan, nil } if nodeName != "" { var node corev1.Node err := reader.Get(ctx, client.ObjectKey{Name: nodeName}, &node) - if err != nil && !apierrors.IsNotFound(err) { + nodeMissing := apierrors.IsNotFound(err) + if err != nil && !nodeMissing { return plan, err } ready := false @@ -102,9 +103,9 @@ func (r *GameServerReconciler) planHostportSentinel(ctx context.Context, gs *gam ready = condition.Status == corev1.ConditionTrue } } - if err != nil || !node.DeletionTimestamp.IsZero() || !ready || string(node.UID) != gs.Annotations[hostportNodeUIDAnnotation] { + if nodeMissing || !node.DeletionTimestamp.IsZero() || !ready || string(node.UID) != gs.Annotations[hostportNodeUIDAnnotation] { plan.blockGame = len(gamePods.Items) == 0 - set("EndpointNodeUnavailable", fmt.Sprintf("Hostport endpoint node %q is missing, replaced or not Ready; restore that node or explicitly migrate the endpoint.", nodeName), false) + hostportPending("EndpointNodeUnavailable", fmt.Sprintf("Hostport endpoint node %q is missing, replaced or not Ready; restore that node or explicitly migrate the endpoint.", nodeName)) return plan, nil } } @@ -115,7 +116,7 @@ func (r *GameServerReconciler) planHostportSentinel(ctx context.Context, gs *gam return plan, err } if len(gamePods.Items) != 0 || (err == nil && (ss.Spec.Replicas == nil || *ss.Spec.Replicas != 0 || ss.Status.ObservedGeneration < ss.Generation || ss.Status.Replicas != 0)) { - set("GameDraining", "Waiting for the game Pod to disappear before acquiring its Hostport.", false) + hostportPending("GameDraining", "Waiting for the game Pod to disappear before acquiring its Hostport.") return plan, nil } plan.wantSentinel = true @@ -130,7 +131,7 @@ func (r *GameServerReconciler) planHostportSentinel(ctx context.Context, gs *gam } if err == nil || len(sentinelPods.Items) != 0 { plan.blockGame = true - set("SentinelDraining", "Waiting for the sentinel Deployment and Pods to disappear before scheduling the game on its endpoint node.", false) + hostportPending("SentinelDraining", "Waiting for the sentinel Deployment and Pods to disappear before scheduling the game on its endpoint node.") return plan, nil } plan.hostportCondition = hostportListenerCondition(gs, gamePods.Items, nodeName) From 640fbf3a96e2a537394c352bc6df335696ad4271 Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 21 Oct 2026 12:43:16 -0400 Subject: [PATCH 03/20] ci: require and report every audit E2E bucket Signed-off-by: Baylem --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 967f99c60..1b26692f0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -2516,7 +2516,7 @@ jobs: 'go-e2e-unit': (n) => n === 'go (test/e2e)', 'api-postgres': (n) => n === 'api (postgres)', 'e2e-buckets': (n) => n === 'e2e bucket coverage', - 'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//.test(n) && n.includes('/ amd64 '), + 'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|wake-hostport) \//.test(n) && n.includes('/ amd64 '), 'e2e-go-arm64': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//.test(n) && n.includes('/ arm64 '), 'e2e-multicluster': (n) => n.startsWith('e2e multicluster /') && n.includes('/ amd64 '), 'e2e-multicluster-arm64': (n) => n.startsWith('e2e multicluster /') && n.includes('/ arm64 '), @@ -2669,7 +2669,7 @@ jobs: const bucketSet = new Set(); for (const j of apiJobs) { const n = j.name || ''; - const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//); + const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|wake-hostport) \//); if (m) bucketSet.add(m[1]); if (/^e2e multicluster \//.test(n)) bucketSet.add('multicluster'); if (/^e2e upgrade \//.test(n)) bucketSet.add('upgrade'); From b31a26b236cc126ec22c6d9dcfe20a8d774e5260 Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 21 Oct 2026 17:15:54 -0400 Subject: [PATCH 04/20] test(sentinel): bind wake-budget network fixtures to context Signed-off-by: Baylem --- sentinel/wake_budget_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sentinel/wake_budget_test.go b/sentinel/wake_budget_test.go index e7b5646d0..1b518742f 100644 --- a/sentinel/wake_budget_test.go +++ b/sentinel/wake_budget_test.go @@ -151,7 +151,7 @@ func TestUDPWakeWorkerCoalescesAndStops(t *testing.T) { } func TestSuccessfulGameplayOutlivesJoinBudget(t *testing.T) { - upstream, err := net.Listen("tcp", "127.0.0.1:0") + upstream, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) } @@ -214,12 +214,12 @@ func (c observedPacketConn) ReadFrom(p []byte) (int, net.Addr, error) { } func TestUDPReceiveContinuesWhileWakeIsStalled(t *testing.T) { - listener, err := net.ListenPacket("udp", "127.0.0.1:0") + listener, err := (&net.ListenConfig{}).ListenPacket(t.Context(), "udp", "127.0.0.1:0") if err != nil { t.Fatal(err) } defer listener.Close() - client, err := net.Dial("udp", listener.LocalAddr().String()) + client, err := (&net.Dialer{}).DialContext(t.Context(), "udp", listener.LocalAddr().String()) if err != nil { t.Fatal(err) } From 5978bd13666f8ebdec37a9497b96ca964e5c5895 Mon Sep 17 00:00:00 2001 From: Baylem Date: Thu, 22 Oct 2026 20:26:44 -0400 Subject: [PATCH 05/20] test(ci): cover audit buckets in full selection fixtures Signed-off-by: Baylem --- test/e2e/ci_selection_e2e_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/ci_selection_e2e_test.go b/test/e2e/ci_selection_e2e_test.go index d2d6edd00..1775ef3d2 100644 --- a/test/e2e/ci_selection_e2e_test.go +++ b/test/e2e/ci_selection_e2e_test.go @@ -22,7 +22,7 @@ import ( // ciSelectionBuckets mirrors E2E_BUCKETS in hack/ci_scope.py (the e2e-go matrix). var ciSelectionBuckets = []string{ "operator", "api-auth", "api-roles", "api-rbac", "api-agent", "api-mods", - "telemetry", "optional-components", + "telemetry", "optional-components", "wake-hostport", } // ciSelection is the decoded output of one `ci_scope.py e2e` run. From 6a03b3f2dab6b13c88750131dacf1ee251871476 Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 23 Oct 2026 00:19:17 -0400 Subject: [PATCH 06/20] test: synchronize closed RCON response fixtures Signed-off-by: Baylem --- agent/internal/rcon/nuclearoption_test.go | 29 +++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/agent/internal/rcon/nuclearoption_test.go b/agent/internal/rcon/nuclearoption_test.go index 248c6a7fb..9061d2c94 100644 --- a/agent/internal/rcon/nuclearoption_test.go +++ b/agent/internal/rcon/nuclearoption_test.go @@ -1275,18 +1275,47 @@ type pipeFrameCase struct { forbidErr string // substring the error must NOT contain (e.g. a detail suffix) } +type responseDeadlineConn struct { + net.Conn + deadlineSet chan<- struct{} +} + +func (c *responseDeadlineConn) SetDeadline(deadline time.Time) error { + if err := c.Conn.SetDeadline(deadline); err != nil { + return err + } + select { + case c.deadlineSet <- struct{}{}: + default: + } + return nil +} + // runPipeFrameCases sends each case's raw frame back to the client over net.Pipe // and checks the body or the error. func runPipeFrameCases(t *testing.T, tests []pipeFrameCase) { t.Helper() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + deadlineSet := make(chan struct{}, 1) client := newPipeClient(t, func(remote net.Conn) { if _, _, err := readNuclearOptionRequest(remote); err != nil { return } + if len(tt.frame) == 0 { + // Exec sets its response deadline after sending the request. + // Closing first can fail SetDeadline instead of the status read. + select { + case <-deadlineSet: + case <-t.Context().Done(): + return + } + } writeFrameSegments(remote, tt.frame, tt.oneByte) }) + if len(tt.frame) == 0 { + client.conn = &responseDeadlineConn{Conn: client.conn, deadlineSet: deadlineSet} + } body, err := client.Exec("test-command") if tt.wantErr != "" { From 92b7ab07743beb6271516d9fb1c3bfe3ef422692 Mon Sep 17 00:00:00 2001 From: Baylem Date: Sun, 25 Oct 2026 19:10:32 -0400 Subject: [PATCH 07/20] fix: preserve HostPort status without healthy polling Signed-off-by: Baylem --- .../2026-07-24-wake-on-connect-design.md | 40 +--------------- .../controller/gameserver_controller.go | 2 +- .../controller/gameserver_hostport.go | 9 ++++ .../gameserver_hostport_feedback_test.go | 46 +++++++++++++++++++ .../internal/controller/gameserver_status.go | 20 +++++--- operator/specs.md | 24 ++++++++-- 6 files changed, 91 insertions(+), 50 deletions(-) create mode 100644 operator/internal/controller/gameserver_hostport_feedback_test.go diff --git a/docs/archive/superpowers/specs/2026-07-24-wake-on-connect-design.md b/docs/archive/superpowers/specs/2026-07-24-wake-on-connect-design.md index 51f5288e6..c8632d253 100644 --- a/docs/archive/superpowers/specs/2026-07-24-wake-on-connect-design.md +++ b/docs/archive/superpowers/specs/2026-07-24-wake-on-connect-design.md @@ -79,45 +79,7 @@ A new Service `-game-direct` — an always-ClusterIP Service selecting the g 4. Game Service selector flips back to `gameserver: ` 5. Sentinel Deployment deleted last -Connections already held in the proxy survive the flip-back because they connect through `-game-direct`, not the re-pointed game Service. - -**Hostport correction (#730, October 2026):** Hostport uses a separate handoff. -Before its first sleep, the operator saves the live game's node name and Node UID -in `gameplane.local/hostport-node` and `gameplane.local/hostport-node-uid`. These -annotations survive Pod deletion and operator restart; the dashboard's current -`gameplane.local/node` annotation remains a separate, transient observation. -The game completes its normal graceful stop and scales to zero. Only after the -StatefulSet acknowledges that generation and a direct API list confirms all game -Pods have disappeared does the operator create the sentinel. The sentinel has -required node affinity for that identity, the game's tolerations and scheduling -constraints, and a Recreate strategy. It never bypasses the scheduler with -`spec.nodeName`. - -On wake, foreground deletion removes the sentinel Deployment and its children. -The game remains at zero until a direct API read confirms the Deployment is gone -and no sentinel Pods remain, including terminating Pods. The game then starts -with the same required endpoint-node affinity. Hostport clients still reconnect; -the old "briefly Pending, self-heals" sequencing is superseded. Service-backed -exposure retains the hold-and-proxy drain described above. - -`HostportWakeReady=False` explains `GameDraining`, `SentinelDraining`, -`EndpointNodeUnknown`, `EndpointNodeUnavailable`, `EndpointUnschedulable`, or -`ListenerPending`. A full node stays pinned and reports the scheduler's reason; -a missing, replaced or unready node makes wake unavailable. The operator never -silently changes the advertised node. Unrelated child reconciliation continues -during these waits. After restoring node capacity/readiness, reconciliation -resumes automatically. For intentional migration, suspend the server, wait for -both game and sentinel Pods to disappear, clear both durable endpoint annotations, -update scheduling/storage placement and the client-facing node address, then -resume the game before sleeping it again. Do not clear the annotations while a -listener remains active. An installation upgraded while already asleep without -a recorded game node must first start its game to establish placement. - -**Wake budget correction (#729):** The join deadline starts before the Kubernetes -PATCH, which has a separate five-second maximum within that budget. Concurrent -callers share the actual result of the in-flight PATCH. UDP packet reads enqueue -coalesced requests to one bounded worker and never wait on the PATCH. Successful -TCP proxy sessions use the longer session/drain context after upstream connects. +Connections already held in the proxy survive the flip-back because they connect through `-game-direct`, not the re-pointed game Service. In Hostport mode the sentinel relinquishes the host port on deletion; the game pod briefly Pending self-heals via scheduler retry. ### Connection handling state machine diff --git a/operator/internal/controller/gameserver_controller.go b/operator/internal/controller/gameserver_controller.go index 5d501afba..00e4b8fa5 100644 --- a/operator/internal/controller/gameserver_controller.go +++ b/operator/internal/controller/gameserver_controller.go @@ -457,7 +457,7 @@ func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) // or nothing fires the transition until an unrelated event happens to // wake the reconciler. hints := []time.Duration{stopRequeue, idleRequeue} - if plan.wantSentinel || plan.hostportCondition != nil { + if sentinelBackstopPending(plan) { // Bounded backstop for planSentinel's wait on the sentinel or the // game pod. Both are owned (Owns(&appsv1.Deployment{}), // Owns(&appsv1.StatefulSet{})) so a readiness transition normally diff --git a/operator/internal/controller/gameserver_hostport.go b/operator/internal/controller/gameserver_hostport.go index 0ff7464a8..fbd34a8da 100644 --- a/operator/internal/controller/gameserver_hostport.go +++ b/operator/internal/controller/gameserver_hostport.go @@ -31,6 +31,15 @@ func (r *GameServerReconciler) hostportReader() client.Reader { return r.Client } +// sentinelBackstopPending polls Hostport only until the listener is ready. +// Service-backed sentinel handoffs retain their existing watch backstop. +func sentinelBackstopPending(plan sentinelPlan) bool { + if plan.hostportCondition != nil { + return plan.hostportCondition.Status != metav1.ConditionTrue + } + return plan.wantSentinel +} + func hostportCondition(gs *gameplanev1alpha1.GameServer, reason, message string, ready bool) *metav1.Condition { status := metav1.ConditionFalse if ready { diff --git a/operator/internal/controller/gameserver_hostport_feedback_test.go b/operator/internal/controller/gameserver_hostport_feedback_test.go new file mode 100644 index 000000000..db7b078c5 --- /dev/null +++ b/operator/internal/controller/gameserver_hostport_feedback_test.go @@ -0,0 +1,46 @@ +package controller + +import ( + "testing" + + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestHostportBackstopStopsOnceListenerReady(t *testing.T) { + t.Parallel() + for _, asleep := range []bool{false, true} { + for _, status := range []metav1.ConditionStatus{metav1.ConditionTrue, metav1.ConditionFalse, metav1.ConditionUnknown} { + plan := sentinelPlan{wantSentinel: asleep, hostportCondition: &metav1.Condition{Status: status}} + if got := sentinelBackstopPending(plan); got != (status != metav1.ConditionTrue) { + t.Fatalf("asleep=%v status=%s backstop=%v", asleep, status, got) + } + } + } + if !sentinelBackstopPending(sentinelPlan{wantSentinel: true}) || sentinelBackstopPending(sentinelPlan{}) { + t.Fatal("service-backed sentinel backstop changed") + } +} + +func TestHostportConditionPreservesExistingReadyFailure(t *testing.T) { + t.Parallel() + hostport := &metav1.Condition{Type: hostportWakeCondition, Status: metav1.ConditionFalse, Reason: "ListenerPending", Message: "Waiting for a Ready listener"} + for _, existing := range []metav1.ConditionStatus{metav1.ConditionTrue, metav1.ConditionFalse, metav1.ConditionUnknown} { + conditions := []metav1.Condition{{Type: "Ready", Status: existing, Reason: "CrashLoopBackOff", Message: "game keeps crashing"}} + got := applyHostportCondition(conditions, hostport) + ready := meta.FindStatusCondition(got, "Ready") + if existing == metav1.ConditionFalse { + if ready.Reason != "CrashLoopBackOff" || ready.Message != "game keeps crashing" { + t.Fatalf("lost startup failure: %+v", ready) + } + } else if ready.Reason != hostport.Reason || ready.Status != metav1.ConditionFalse { + t.Fatalf("endpoint failure did not block readiness: %+v", ready) + } + if meta.FindStatusCondition(got, hostportWakeCondition) == nil { + t.Fatal("hostport diagnostic missing") + } + } + if meta.FindStatusCondition(applyHostportCondition(nil, hostport), "Ready") == nil { + t.Fatal("missing Ready condition was not added") + } +} diff --git a/operator/internal/controller/gameserver_status.go b/operator/internal/controller/gameserver_status.go index 4a470c122..dcfc1cfd8 100644 --- a/operator/internal/controller/gameserver_status.go +++ b/operator/internal/controller/gameserver_status.go @@ -196,12 +196,7 @@ func (r *GameServerReconciler) reconcileStatus( // Compute tunnel conditions. gs.Status.Conditions = computeTunnelConditions(gs, tunnelPlan, tunnelDep) if len(hostportConditions) != 0 && hostportConditions[0] != nil { - condition := *hostportConditions[0] - gs.Status.Conditions = upsertCondition(gs.Status.Conditions, condition) - if condition.Status != metav1.ConditionTrue { - condition.Type = "Ready" - gs.Status.Conditions = upsertCondition(gs.Status.Conditions, condition) - } + gs.Status.Conditions = applyHostportCondition(gs.Status.Conditions, hostportConditions[0]) } else { gs.Status.Conditions = removeCondition(gs.Status.Conditions, hostportWakeCondition) } @@ -319,6 +314,19 @@ func (r *GameServerReconciler) reconcileStatus( return 15 * time.Second, nil } +// applyHostportCondition preserves the more specific failure already computed +// for the game while still exposing the independent endpoint diagnostic. +func applyHostportCondition(conditions []metav1.Condition, hostport *metav1.Condition) []metav1.Condition { + conditions = upsertCondition(conditions, *hostport) + ready := meta.FindStatusCondition(conditions, "Ready") + if hostport.Status != metav1.ConditionTrue && (ready == nil || ready.Status != metav1.ConditionFalse) { + condition := *hostport + condition.Type = "Ready" + conditions = upsertCondition(conditions, condition) + } + return conditions +} + func derivePhase( gs *gameplanev1alpha1.GameServer, ssExists, ssReady, hbFresh bool, idle idleState, ) gameplanev1alpha1.GameServerPhase { diff --git a/operator/specs.md b/operator/specs.md index 4d5124ef3..720131222 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -43,10 +43,26 @@ node affinity for the recorded endpoint, combined with the game's scheduling constraints; the sentinel carries its tolerations. Node gets are read-only RBAC. `HostportWakeReady` reports listener readiness, each drain and unavailable or unschedulable endpoints without blocking unrelated reconciliation. There is no -automatic endpoint migration. See the updated -[July handoff design](../docs/superpowers/specs/2026-07-24-wake-on-connect-design.md#sequencing-and-port-handover) -for recovery/migration steps. Service-backed wake retains its established-session -drain behavior. Auxiliary resource defaults are defined separately. +automatic endpoint migration. Healthy Hostport listeners rely on watches; the +five-second backstop runs only while `HostportWakeReady` is not True. Endpoint +diagnostics never replace an existing `Ready=False` game failure. + +After restoring node capacity/readiness, reconciliation resumes automatically. +For intentional migration, suspend the server, wait for both game and sentinel +Pods to disappear, clear both durable endpoint annotations, update scheduling, +storage placement and the client-facing node address, then resume the game before +sleeping it again. Do not clear the annotations while a listener remains active. +An installation upgraded while already asleep without a recorded game node must +first start its game to establish placement. Hostport clients reconnect after +handoff; the historical "briefly Pending, self-heals" sequence in the archived +[July design](../docs/archive/superpowers/specs/2026-07-24-wake-on-connect-design.md#sequencing-and-port-handover) +is superseded by this specification. Service-backed wake retains its +established-session drain behavior. Auxiliary resource defaults are defined separately. + +The sentinel starts the join deadline before its Kubernetes wake PATCH, whose +separate five-second maximum fits inside that budget. Concurrent callers share +the in-flight PATCH result. UDP reads enqueue coalesced requests to one bounded +worker, and established TCP sessions use the longer session/drain context. ## Non-goals / boundaries From 644edc8e3cada833beee46f7fb0318a2df1478f8 Mon Sep 17 00:00:00 2001 From: Baylem Date: Tue, 27 Oct 2026 11:56:21 -0400 Subject: [PATCH 08/20] fix: clarify pending Hostport status and isolate CI fixtures Signed-off-by: Baylem --- .github/workflows/ci.yaml | 2 +- operator/internal/controller/gameserver_hostport.go | 6 +++++- operator/specs.md | 3 ++- test/e2e/hostport_wake_e2e_test.go | 5 ++++- test/e2e/specs.md | 6 ++++-- 5 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index aff475c98..680d34e0f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1832,7 +1832,7 @@ jobs: - bucket: wake-hostport parallel: 1 test_timeout: 15m - job_timeout: 60 + job_timeout: 60 # Boots two-node kind, installs the chart and exercises repeated Hostport sleep/wake handoffs. - bucket: api-auth parallel: 4 test_timeout: 15m diff --git a/operator/internal/controller/gameserver_hostport.go b/operator/internal/controller/gameserver_hostport.go index fbd34a8da..c0362d4e4 100644 --- a/operator/internal/controller/gameserver_hostport.go +++ b/operator/internal/controller/gameserver_hostport.go @@ -161,7 +161,11 @@ func hostportListenerCondition(gs *gameplanev1alpha1.GameServer, pods []corev1.P } } } - return hostportCondition(gs, "ListenerPending", fmt.Sprintf("Waiting for a Ready listener on endpoint node %q.", nodeName), false) + message := "Waiting for a Ready Hostport listener." + if nodeName != "" { + message = fmt.Sprintf("Waiting for a Ready listener on endpoint node %q.", nodeName) + } + return hostportCondition(gs, "ListenerPending", message, false) } // hostportAffinity ANDs node identity into every OR alternative, retaining diff --git a/operator/specs.md b/operator/specs.md index 720131222..2d52d03dc 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -45,7 +45,8 @@ constraints; the sentinel carries its tolerations. Node gets are read-only RBAC. unschedulable endpoints without blocking unrelated reconciliation. There is no automatic endpoint migration. Healthy Hostport listeners rely on watches; the five-second backstop runs only while `HostportWakeReady` is not True. Endpoint -diagnostics never replace an existing `Ready=False` game failure. +diagnostics never replace an existing `Ready=False` game failure. Pending-listener +messages omit the endpoint node name until it has been recorded. After restoring node capacity/readiness, reconciliation resumes automatically. For intentional migration, suspend the server, wait for both game and sentinel diff --git a/test/e2e/hostport_wake_e2e_test.go b/test/e2e/hostport_wake_e2e_test.go index d0461dd76..4cef3451b 100644 --- a/test/e2e/hostport_wake_e2e_test.go +++ b/test/e2e/hostport_wake_e2e_test.go @@ -24,7 +24,10 @@ import ( // Pods and changes node scheduling; it must not run with other game tests. func TestGameServer_HostportWakeStableEndpoint(t *testing.T) { ctx := t.Context() - const ns, name, tmpl, port = "gameplane-games", "hostport-wake", "hostport-wake-template", 12345 + // The chart owns this namespace and scopes operator Secret access to it. + const ns, port = "gameplane-games", 12345 + name := fmt.Sprintf("hostport-wake-%d", time.Now().UnixNano()) + tmpl := name + "-template" nodes, err := envInstance.K8s.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) if err != nil { t.Fatal(err) diff --git a/test/e2e/specs.md b/test/e2e/specs.md index d44b742de..666e0ef3b 100644 --- a/test/e2e/specs.md +++ b/test/e2e/specs.md @@ -29,7 +29,9 @@ same node address. It observes the old game still serving during termination, holds a terminating sentinel Pod to prove the reverse handoff waits, checks a cordoned endpoint remains unavailable rather than relocating, and deletes the node to exercise the explicit unavailable/replaced-node condition. This bucket -runs alone because it changes node scheduling. Runtime results come from CI. +runs alone because it changes node scheduling. Its GameServer and template names +are unique per run; it uses the chart-owned game namespace so the operator's +configured Secret scope and RBAC apply. Runtime results come from CI. ### Remote gateway parity acceptance @@ -91,7 +93,7 @@ This is a name-matching contract, not a structural one: ## Conventions (already normative via `CLAUDE.md`, restated here for the module's own record) -- **`t.Parallel()`**: tests call it unless shared infrastructure requires serialization. The `ratelimit` bucket deliberately drains the shared login limiter; gateway parity temporarily changes the shared remote Helm release and trust. These are documented exceptions. +- **`t.Parallel()`**: tests call it unless shared infrastructure requires serialization. The `ratelimit` bucket deliberately drains the shared login limiter; gateway parity temporarily changes the shared remote Helm release and trust; `wake-hostport` changes shared node scheduling and deletes the endpoint node. These are documented exceptions. - **Per-test unique resource names**: CRs, namespaces, and users a test creates get a name unique to that test run (commonly time-based, e.g. `CreateUser`'s `fmt.Sprintf("%s-%d", prefix, time.Now().UnixNano())`) so parallel tests never collide on a shared cluster. - **`ociPushMu`**: a package-level mutex guarding tests that push through the shared, fixed-name `oras-push` Job (`module_e2e_test.go`, `module_verify_e2e_test.go`). Because the Job's name and namespace are fixed fixtures, two tests racing to apply/wait on it would stomp each other; `ociPushMu.Lock()`/`Unlock()` serializes them. - **`ensureResticRepo(t)`**: guards tests that run a backup against the shared restic repository fixture (`backup_e2e_test.go`, `backupschedule_e2e_test.go`, `failure_paths_e2e_test.go`) — restic repo initialization is not safe to race. From 60ec371009a456f93a6e543d82ee6a0c0e734b59 Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 4 Nov 2026 00:18:40 -0500 Subject: [PATCH 09/20] fix(operator): fence Hostport workloads after node replacement Signed-off-by: Baylem --- .../controller/gameserver_controller.go | 16 ++ .../controller/gameserver_hostport.go | 40 ++++- .../gameserver_hostport_identity_test.go | 170 ++++++++++++++++++ operator/specs.md | 7 + 4 files changed, 226 insertions(+), 7 deletions(-) create mode 100644 operator/internal/controller/gameserver_hostport_identity_test.go diff --git a/operator/internal/controller/gameserver_controller.go b/operator/internal/controller/gameserver_controller.go index 76ec50599..cae36b95e 100644 --- a/operator/internal/controller/gameserver_controller.go +++ b/operator/internal/controller/gameserver_controller.go @@ -1520,6 +1520,22 @@ func (r *GameServerReconciler) reconcileStatefulSet( if owner := metav1.GetControllerOf(ss); owner != nil && owner.UID != live.UID { return fmt.Errorf("StatefulSet belongs to a different GameServer identity") } + if live.Spec.Networking.Expose == "Hostport" && live.Annotations[hostportNodeAnnotation] != "" { + node, err := r.hostportEndpointNode(ctx, &live) + if err != nil { + return err + } + if node == nil { + // StatefulSet replacement and Node deletion run independently + // of the earlier sentinel plan. Identity loss must also fence + // stale wake decisions and lifecycle owners that retain replicas. + actualReplicas, replicas = 0, 0 + ss.Spec.Replicas = &actualReplicas + if ss.ResourceVersion != "" { + return nil // Stop without rolling or changing the game template. + } + } + } op, held, err := lifecycleOwner(&live) if err != nil { return err diff --git a/operator/internal/controller/gameserver_hostport.go b/operator/internal/controller/gameserver_hostport.go index c0362d4e4..296c92ad7 100644 --- a/operator/internal/controller/gameserver_hostport.go +++ b/operator/internal/controller/gameserver_hostport.go @@ -31,6 +31,23 @@ func (r *GameServerReconciler) hostportReader() client.Reader { return r.Client } +// hostportEndpointNode returns the recorded node only while its identity is +// intact. Readiness is separate: a temporarily NotReady node may still host a +// running game, but a missing, deleting or replacement node cannot authorize it. +func (r *GameServerReconciler) hostportEndpointNode(ctx context.Context, gs *gameplanev1alpha1.GameServer) (*corev1.Node, error) { + var node corev1.Node + if err := r.hostportReader().Get(ctx, client.ObjectKey{Name: gs.Annotations[hostportNodeAnnotation]}, &node); err != nil { + if apierrors.IsNotFound(err) { + return nil, nil + } + return nil, fmt.Errorf("read Hostport endpoint node: %w", err) + } + if !node.DeletionTimestamp.IsZero() || gs.Annotations[hostportNodeUIDAnnotation] == "" || string(node.UID) != gs.Annotations[hostportNodeUIDAnnotation] { + return nil, nil + } + return &node, nil +} + // sentinelBackstopPending polls Hostport only until the listener is ready. // Service-backed sentinel handoffs retain their existing watch backstop. func sentinelBackstopPending(plan sentinelPlan) bool { @@ -100,21 +117,30 @@ func (r *GameServerReconciler) planHostportSentinel(ctx context.Context, gs *gam return plan, nil } if nodeName != "" { - var node corev1.Node - err := reader.Get(ctx, client.ObjectKey{Name: nodeName}, &node) - nodeMissing := apierrors.IsNotFound(err) - if err != nil && !nodeMissing { + node, err := r.hostportEndpointNode(ctx, gs) + if err != nil { return plan, err } + if node == nil { + plan.blockGame = true + hostportPending("EndpointNodeUnavailable", fmt.Sprintf("Hostport endpoint node %q is missing, replaced or deleting; restore that node or explicitly migrate the endpoint.", nodeName)) + return plan, nil + } ready := false for _, condition := range node.Status.Conditions { if condition.Type == corev1.NodeReady { ready = condition.Status == corev1.ConditionTrue } } - if nodeMissing || !node.DeletionTimestamp.IsZero() || !ready || string(node.UID) != gs.Annotations[hostportNodeUIDAnnotation] { - plan.blockGame = len(gamePods.Items) == 0 - hostportPending("EndpointNodeUnavailable", fmt.Sprintf("Hostport endpoint node %q is missing, replaced or not Ready; restore that node or explicitly migrate the endpoint.", nodeName)) + if !ready { + plan.blockGame = true + for _, pod := range gamePods.Items { + if pod.Name == gs.Name+"-0" && pod.Spec.NodeName == nodeName && pod.DeletionTimestamp.IsZero() && pod.Status.Phase == corev1.PodRunning { + plan.blockGame = false + break + } + } + hostportPending("EndpointNodeUnavailable", fmt.Sprintf("Hostport endpoint node %q is not Ready; restore its readiness before starting a listener.", nodeName)) return plan, nil } } diff --git a/operator/internal/controller/gameserver_hostport_identity_test.go b/operator/internal/controller/gameserver_hostport_identity_test.go new file mode 100644 index 000000000..7faaf8e96 --- /dev/null +++ b/operator/internal/controller/gameserver_hostport_identity_test.go @@ -0,0 +1,170 @@ +package controller + +import ( + "context" + "errors" + "reflect" + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +func hostportIdentityFixture(t *testing.T, node *corev1.Node, phase corev1.PodPhase, operation *lifecycleOperation) (*GameServerReconciler, *gameplanev1alpha1.GameServer, *gameplanev1alpha1.GameTemplate) { + t.Helper() + r, gs, tmpl := hostportFixture(t) + gs.Annotations = map[string]string{hostportNodeAnnotation: "node-a", hostportNodeUIDAnnotation: "original-node"} + if operation != nil { + operation.TargetUID = gs.UID + gs.Annotations[lifecycleOperationAnnotation] = lifecycleOperationJSON(*operation) + } + one := int32(1) + ss := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: gs.Name, Namespace: gs.Namespace, UID: "workload", OwnerReferences: []metav1.OwnerReference{{APIVersion: "gameplane.local/v1alpha1", Kind: "GameServer", Name: gs.Name, UID: gs.UID, Controller: ownerBoolPtr(true)}}}, Spec: appsv1.StatefulSetSpec{Replicas: &one, Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{"preserve": "template"}}, Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "game", Image: "existing-image"}}}}}} + objects := []client.Object{gs, ss} + if node != nil { + objects = append(objects, node) + } + if phase != "" { + objects = append(objects, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: gs.Name + "-0", Namespace: gs.Namespace, Labels: map[string]string{"app.kubernetes.io/name": "gameplane-game", "app.kubernetes.io/instance": gs.Name}}, Spec: corev1.PodSpec{NodeName: "node-a"}, Status: corev1.PodStatus{Phase: phase}}) + } + c := fake.NewClientBuilder().WithScheme(r.Scheme).WithObjects(objects...).Build() + r.Client, r.APIReader = c, c + return r, gs, tmpl +} + +func hostportIdentityNode(uid types.UID, ready corev1.ConditionStatus) *corev1.Node { + return &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-a", UID: uid}, Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: ready}}}} +} + +func TestHostportNodeIdentityFencesExistingReplacementPods(t *testing.T) { + t.Parallel() + deleting := hostportIdentityNode("original-node", corev1.ConditionTrue) + now := metav1.Now() + deleting.DeletionTimestamp, deleting.Finalizers = &now, []string{"test/hold"} + for _, tc := range []struct { + name string + node *corev1.Node + }{ + {"replaced", hostportIdentityNode("replacement-node", corev1.ConditionTrue)}, + {"missing", nil}, + {"deleting", deleting}, + } { + for _, phase := range []corev1.PodPhase{corev1.PodPending, corev1.PodRunning} { + t.Run(tc.name+"/"+string(phase), func(t *testing.T) { + r, gs, tmpl := hostportIdentityFixture(t, tc.node, phase, nil) + plan, err := r.planSentinel(t.Context(), gs, tmpl, idleAwake) + if err != nil { + t.Fatal(err) + } + if !plan.blockGame || plan.wantSentinel || plan.hostportCondition == nil || plan.hostportCondition.Reason != "EndpointNodeUnavailable" { + t.Fatalf("replacement Pod bypassed endpoint identity: %+v", plan) + } + }) + } + } +} + +func TestHostportNotReadyPreservesOnlyAnExistingRunningGame(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + phase corev1.PodPhase + block bool + }{{corev1.PodRunning, false}, {corev1.PodPending, true}, {corev1.PodSucceeded, true}, {"", true}} { + t.Run(string(tc.phase), func(t *testing.T) { + r, gs, tmpl := hostportIdentityFixture(t, hostportIdentityNode("original-node", corev1.ConditionFalse), tc.phase, nil) + plan, err := r.planSentinel(t.Context(), gs, tmpl, idleAwake) + if err != nil || plan.blockGame != tc.block || plan.hostportCondition == nil || plan.hostportCondition.Reason != "EndpointNodeUnavailable" { + t.Fatalf("NotReady plan = %+v, %v; want block=%v", plan, err, tc.block) + } + }) + } +} + +func TestHostportIdentityFenceSurvivesLifecycleReplicaPreservation(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + name string + op *lifecycleOperation + }{ + {"ordinary", nil}, + {"backup", &lifecycleOperation{lifecycleIdentity: lifecycleIdentity{Kind: "Backup", Name: "backup", UID: "backup-uid"}}}, + {"restore-startup", &lifecycleOperation{lifecycleIdentity: lifecycleIdentity{Kind: "Restore", Name: "restore", UID: "restore-uid", InitialStartup: true}}}, + } { + t.Run(tc.name, func(t *testing.T) { + r, gs, tmpl := hostportIdentityFixture(t, hostportIdentityNode("original-node", corev1.ConditionFalse), corev1.PodRunning, tc.op) + // A transient readiness loss must not stop an existing listener, + // including while a backup owns its replica count. + plan, err := r.planSentinel(t.Context(), gs, tmpl, idleAwake) + if err != nil || plan.blockGame { + t.Fatalf("same-node running listener was blocked: %+v %v", plan, err) + } + replicas, _, err := r.desiredReplicas(t.Context(), gs, tmpl, idleAwake) + if err != nil || replicas != 1 { + t.Fatalf("running listener replicas=%d, err=%v", replicas, err) + } + var original appsv1.StatefulSet + if err := r.Get(t.Context(), client.ObjectKeyFromObject(gs), &original); err != nil { + t.Fatal(err) + } + // Node replacement races the prior plan and desiredReplicas read. + // The mutation must recheck identity instead of trusting replicas=1. + // Keep the cache's old Node available; only the uncached reader + // sees the replacement that must govern the mutation. + r.APIReader = fake.NewClientBuilder().WithScheme(r.Scheme).WithObjects(gs, hostportIdentityNode("replacement-node", corev1.ConditionTrue)).Build() + if err := r.reconcileStatefulSet(t.Context(), gs, tmpl, nil, &materializedConfig{}, replicas); err != nil { + t.Fatal(err) + } + var current appsv1.StatefulSet + if err := r.Get(t.Context(), client.ObjectKeyFromObject(gs), ¤t); err != nil { + t.Fatal(err) + } + if current.Spec.Replicas == nil || *current.Spec.Replicas != 0 || !reflect.DeepEqual(original.Spec.Template, current.Spec.Template) { + t.Fatalf("endpoint loss did not stop the existing workload without changing its template: %+v", current.Spec) + } + }) + } +} + +func TestHostportMutationKeepsRunningGameDuringNotReady(t *testing.T) { + t.Parallel() + r, gs, tmpl := hostportIdentityFixture(t, hostportIdentityNode("original-node", corev1.ConditionUnknown), corev1.PodRunning, nil) + if err := r.reconcileStatefulSet(t.Context(), gs, tmpl, nil, &materializedConfig{}, 1); err != nil { + t.Fatal(err) + } + var ss appsv1.StatefulSet + if err := r.Get(t.Context(), client.ObjectKeyFromObject(gs), &ss); err != nil { + t.Fatal(err) + } + if ss.Spec.Replicas == nil || *ss.Spec.Replicas != 1 { + t.Fatalf("transient NotReady stopped an existing game: %+v", ss.Spec.Replicas) + } +} + +func TestHostportNodeReadErrorDoesNotMutateWorkload(t *testing.T) { + t.Parallel() + r, gs, tmpl := hostportIdentityFixture(t, hostportIdentityNode("original-node", corev1.ConditionTrue), corev1.PodRunning, nil) + readErr := errors.New("API unavailable") + r.APIReader = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, object client.Object, options ...client.GetOption) error { + if _, ok := object.(*corev1.Node); ok { + return readErr + } + return cl.Get(ctx, key, object, options...) + }}) + if err := r.reconcileStatefulSet(t.Context(), gs, tmpl, nil, &materializedConfig{}, 1); !errors.Is(err, readErr) { + t.Fatalf("node read failure lost: %v", err) + } + var ss appsv1.StatefulSet + if err := r.Get(t.Context(), client.ObjectKeyFromObject(gs), &ss); err != nil { + t.Fatal(err) + } + if ss.Spec.Replicas == nil || *ss.Spec.Replicas != 1 || ss.Spec.Template.Spec.Containers[0].Image != "existing-image" { + t.Fatalf("failed node read changed the workload: %+v", ss.Spec) + } +} diff --git a/operator/specs.md b/operator/specs.md index 084677ae5..5272befc0 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -60,6 +60,13 @@ five-second backstop runs only while `HostportWakeReady` is not True. Endpoint diagnostics never replace an existing `Ready=False` game failure. Pending-listener messages omit the endpoint node name until it has been recorded. +A missing, deleting or replaced endpoint Node forces the game StatefulSet to +zero replicas even if a replacement game Pod already exists. The live Node +identity is checked again immediately before workload mutation, including +backup replica preservation and restore startup; the existing pod template is +left intact. A transient NotReady condition on the same Node UID preserves an +existing running game, while Pending or terminating Pods do not authorize a wake. + After restoring node capacity/readiness, reconciliation resumes automatically. For intentional migration, suspend the server, wait for both game and sentinel Pods to disappear, clear both durable endpoint annotations, update scheduling, From 5ced4f9559970582d906a330a0766c5e986b3706 Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 4 Nov 2026 22:37:48 -0500 Subject: [PATCH 10/20] fix(operator): fence stale Hostport sentinel mutations Signed-off-by: Baylem --- .../gameserver_hostport_identity_test.go | 121 ++++++++++++++++++ .../controller/gameserver_sentinel.go | 30 +++++ .../gameserver_sentinel_envtest_test.go | 13 +- operator/specs.md | 4 +- 4 files changed, 166 insertions(+), 2 deletions(-) diff --git a/operator/internal/controller/gameserver_hostport_identity_test.go b/operator/internal/controller/gameserver_hostport_identity_test.go index 7faaf8e96..03e7ad401 100644 --- a/operator/internal/controller/gameserver_hostport_identity_test.go +++ b/operator/internal/controller/gameserver_hostport_identity_test.go @@ -168,3 +168,124 @@ func TestHostportNodeReadErrorDoesNotMutateWorkload(t *testing.T) { t.Fatalf("failed node read changed the workload: %+v", ss.Spec) } } + +func hostportSentinelMutationFixture(t *testing.T, existing bool) (*GameServerReconciler, *gameplanev1alpha1.GameServer, *gameplanev1alpha1.GameTemplate, sentinelPlan, *appsv1.Deployment) { + t.Helper() + r, gs, tmpl := hostportFixture(t) + gs.Annotations = map[string]string{hostportNodeAnnotation: "node-a", hostportNodeUIDAnnotation: "node-uid"} + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + plan, err := r.planSentinel(t.Context(), gs, tmpl, idleAsleep) + if err != nil || !plan.wantSentinel { + t.Fatalf("sleep plan did not authorize the healthy endpoint: %+v, %v", plan, err) + } + var original appsv1.Deployment + if existing { + if err := r.reconcileSentinel(t.Context(), gs, tmpl, plan.wantSentinel); err != nil { + t.Fatal(err) + } + if err := r.Get(t.Context(), client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-waker"}, &original); err != nil { + t.Fatal(err) + } + } + return r, gs, tmpl, plan, &original +} + +func TestHostportSentinelMutationFencesNodeIdentityLoss(t *testing.T) { + t.Parallel() + deleting := hostportIdentityNode("node-uid", corev1.ConditionTrue) + now := metav1.Now() + deleting.DeletionTimestamp, deleting.Finalizers = &now, []string{"test/hold"} + for _, tc := range []struct { + name string + node *corev1.Node + }{ + {"replacement", hostportIdentityNode("replacement-node", corev1.ConditionTrue)}, + {"missing", nil}, + {"deleting", deleting}, + } { + for _, existing := range []bool{false, true} { + mode := "create" + if existing { + mode = "update" + } + t.Run(tc.name+"/"+mode, func(t *testing.T) { + r, gs, tmpl, plan, original := hostportSentinelMutationFixture(t, existing) + objects := []client.Object{gs} + if tc.node != nil { + objects = append(objects, tc.node) + } + // The cache retains the healthy node that authorized the plan. + // Only the uncached reader observes its subsequent identity loss. + r.APIReader = fake.NewClientBuilder().WithScheme(r.Scheme).WithObjects(objects...).Build() + if err := r.reconcileSentinel(t.Context(), gs, tmpl, plan.wantSentinel); err != nil { + t.Fatal(err) + } + var current appsv1.Deployment + if err := r.Get(t.Context(), client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-waker"}, ¤t); err != nil { + t.Fatal(err) + } + if current.Spec.Replicas == nil || *current.Spec.Replicas != 0 { + t.Fatalf("stale sleep plan authorized a sentinel after node identity loss: %+v", current.Spec.Replicas) + } + if existing && !reflect.DeepEqual(original.Spec.Template, current.Spec.Template) { + t.Fatal("identity fence rolled the existing sentinel template") + } + }) + } + } +} + +func TestHostportSentinelMutationRejectsChangedGameServer(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + name string + change func(*gameplanev1alpha1.GameServer) + }{ + {"identity", func(gs *gameplanev1alpha1.GameServer) { gs.UID = "replacement-game" }}, + {"endpoint-name", func(gs *gameplanev1alpha1.GameServer) { gs.Annotations[hostportNodeAnnotation] = "node-b" }}, + {"endpoint-uid", func(gs *gameplanev1alpha1.GameServer) { gs.Annotations[hostportNodeUIDAnnotation] = "replacement-node" }}, + {"expose-mode", func(gs *gameplanev1alpha1.GameServer) { gs.Spec.Networking.Expose = "NodePort" }}, + {"deleting", func(gs *gameplanev1alpha1.GameServer) { + now := metav1.Now() + gs.DeletionTimestamp, gs.Finalizers = &now, []string{"test/hold"} + }}, + } { + t.Run(tc.name, func(t *testing.T) { + r, gs, tmpl, plan, original := hostportSentinelMutationFixture(t, true) + live := gs.DeepCopy() + tc.change(live) + r.APIReader = fake.NewClientBuilder().WithScheme(r.Scheme).WithObjects(live, hostportIdentityNode("node-uid", corev1.ConditionTrue)).Build() + if err := r.reconcileSentinel(t.Context(), gs, tmpl, plan.wantSentinel); err == nil { + t.Fatal("stale GameServer authorized sentinel mutation") + } + var current appsv1.Deployment + if err := r.Get(t.Context(), client.ObjectKeyFromObject(original), ¤t); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(original.Spec, current.Spec) { + t.Fatal("stale GameServer changed the sentinel") + } + }) + } +} + +func TestHostportSentinelMutationRetainsListenerOnNotReadyNode(t *testing.T) { + t.Parallel() + r, gs, tmpl, plan, original := hostportSentinelMutationFixture(t, true) + r.APIReader = fake.NewClientBuilder().WithScheme(r.Scheme).WithObjects(gs, hostportIdentityNode("node-uid", corev1.ConditionFalse)).Build() + if err := r.reconcileSentinel(t.Context(), gs, tmpl, plan.wantSentinel); err != nil { + t.Fatal(err) + } + var current appsv1.Deployment + if err := r.Get(t.Context(), client.ObjectKeyFromObject(original), ¤t); err != nil { + t.Fatal(err) + } + if current.Spec.Replicas == nil || *current.Spec.Replicas != 1 { + t.Fatal("transient NotReady stopped the listener on the original node") + } + if !reflect.DeepEqual(original.Spec.Template, current.Spec.Template) { + t.Fatal("transient NotReady rolled the listener template") + } +} diff --git a/operator/internal/controller/gameserver_sentinel.go b/operator/internal/controller/gameserver_sentinel.go index 69ba7390e..0804f3b18 100644 --- a/operator/internal/controller/gameserver_sentinel.go +++ b/operator/internal/controller/gameserver_sentinel.go @@ -237,6 +237,36 @@ func (r *GameServerReconciler) reconcileSentinel( _, err := controllerutil.CreateOrUpdate(ctx, r.Client, dep, func() error { replicas := int32(1) + if gs.Spec.Networking.Expose == "Hostport" { + var live gameplanev1alpha1.GameServer + if err := r.hostportReader().Get(ctx, client.ObjectKeyFromObject(gs), &live); err != nil { + return err + } + if live.UID != gs.UID || !live.DeletionTimestamp.IsZero() { + return fmt.Errorf("GameServer changed identity or is deleting before sentinel mutation") + } + if live.Spec.Networking.Expose != "Hostport" || + live.Annotations[hostportNodeAnnotation] != gs.Annotations[hostportNodeAnnotation] || + live.Annotations[hostportNodeUIDAnnotation] != gs.Annotations[hostportNodeUIDAnnotation] { + return fmt.Errorf("Hostport endpoint changed before sentinel mutation") + } + if dep.ResourceVersion != "" && !metav1.IsControlledBy(dep, &live) { + return fmt.Errorf("sentinel Deployment is not owned by this GameServer") + } + node, err := r.hostportEndpointNode(ctx, &live) + if err != nil { + return err + } + if node == nil { + // A node can be replaced after the sleep plan. Fence this + // producer before it creates a listener on the reused name. + replicas = 0 + dep.Spec.Replicas = &replicas + if dep.ResourceVersion != "" { + return nil // Stop without rolling the existing pod template. + } + } + } dep.Spec.Replicas = &replicas dep.Spec.Selector = &metav1.LabelSelector{ MatchLabels: map[string]string{ diff --git a/operator/internal/controller/gameserver_sentinel_envtest_test.go b/operator/internal/controller/gameserver_sentinel_envtest_test.go index f756311b6..f400cabb1 100644 --- a/operator/internal/controller/gameserver_sentinel_envtest_test.go +++ b/operator/internal/controller/gameserver_sentinel_envtest_test.go @@ -182,8 +182,16 @@ func TestSentinelReconciliation(t *testing.T) { } deleteCleanup(t, tmpl) + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: uniqueName("sentinel-node")}} + if err := k8sClient.Create(ctx, node); err != nil { + t.Fatalf("create node: %v", err) + } + deleteCleanup(t, node) + gs := &gameplanev1alpha1.GameServer{ - ObjectMeta: metav1.ObjectMeta{Name: "test-sentinel-hostport", Namespace: ns, Annotations: map[string]string{hostportNodeAnnotation: "node-a"}}, + ObjectMeta: metav1.ObjectMeta{Name: "test-sentinel-hostport", Namespace: ns, Annotations: map[string]string{ + hostportNodeAnnotation: node.Name, hostportNodeUIDAnnotation: string(node.UID), + }}, Spec: gameplanev1alpha1.GameServerSpec{ TemplateRef: gameplanev1alpha1.GameTemplateRef{Name: tmpl.Name}, Idle: armedIdleSpec(), @@ -208,6 +216,9 @@ func TestSentinelReconciliation(t *testing.T) { if len(dep.Spec.Template.Spec.Containers) == 0 { t.Fatal("no containers in sentinel deployment") } + if dep.Spec.Replicas == nil || *dep.Spec.Replicas != 1 { + t.Fatal("valid endpoint did not start the sentinel") + } container := dep.Spec.Template.Spec.Containers[0] if len(container.Ports) == 0 { t.Fatal("no ports in sentinel container") diff --git a/operator/specs.md b/operator/specs.md index 5272befc0..467cfea6e 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -64,7 +64,9 @@ A missing, deleting or replaced endpoint Node forces the game StatefulSet to zero replicas even if a replacement game Pod already exists. The live Node identity is checked again immediately before workload mutation, including backup replica preservation and restore startup; the existing pod template is -left intact. A transient NotReady condition on the same Node UID preserves an +left intact. The sentinel Deployment also rechecks the live GameServer endpoint +and Node UID before mutation; a stale sleep plan cannot start a listener on a +replacement node. A transient NotReady condition on the same Node UID preserves an existing running game, while Pending or terminating Pods do not authorize a wake. After restoring node capacity/readiness, reconciliation resumes automatically. From 25bea13b7a15dabc2068158f9e41772a2fcb4b93 Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 4 Nov 2026 22:53:56 -0500 Subject: [PATCH 11/20] fix(operator): preserve sleeping Hostport listeners while nodes recover Signed-off-by: Baylem --- .../controller/gameserver_hostport.go | 31 +++++-- .../gameserver_hostport_identity_test.go | 85 +++++++++++++++++++ .../controller/gameserver_sentinel.go | 8 ++ .../gameserver_sentinel_envtest_test.go | 4 + operator/specs.md | 5 +- 5 files changed, 124 insertions(+), 9 deletions(-) diff --git a/operator/internal/controller/gameserver_hostport.go b/operator/internal/controller/gameserver_hostport.go index 296c92ad7..13e3ebe44 100644 --- a/operator/internal/controller/gameserver_hostport.go +++ b/operator/internal/controller/gameserver_hostport.go @@ -48,6 +48,15 @@ func (r *GameServerReconciler) hostportEndpointNode(ctx context.Context, gs *gam return &node, nil } +func hostportNodeReady(node *corev1.Node) bool { + for _, condition := range node.Status.Conditions { + if condition.Type == corev1.NodeReady { + return condition.Status == corev1.ConditionTrue + } + } + return false +} + // sentinelBackstopPending polls Hostport only until the listener is ready. // Service-backed sentinel handoffs retain their existing watch backstop. func sentinelBackstopPending(plan sentinelPlan) bool { @@ -126,14 +135,16 @@ func (r *GameServerReconciler) planHostportSentinel(ctx context.Context, gs *gam hostportPending("EndpointNodeUnavailable", fmt.Sprintf("Hostport endpoint node %q is missing, replaced or deleting; restore that node or explicitly migrate the endpoint.", nodeName)) return plan, nil } - ready := false - for _, condition := range node.Status.Conditions { - if condition.Type == corev1.NodeReady { - ready = condition.Status == corev1.ConditionTrue - } - } - if !ready { + if !hostportNodeReady(node) { plan.blockGame = true + if idle == idleAsleep { + for _, pod := range sentinelPods.Items { + if pod.Spec.NodeName == nodeName && pod.DeletionTimestamp.IsZero() && pod.Status.Phase == corev1.PodRunning { + plan.wantSentinel = true + break + } + } + } for _, pod := range gamePods.Items { if pod.Name == gs.Name+"-0" && pod.Spec.NodeName == nodeName && pod.DeletionTimestamp.IsZero() && pod.Status.Phase == corev1.PodRunning { plan.blockGame = false @@ -180,7 +191,11 @@ func hostportListenerCondition(gs *gameplanev1alpha1.GameServer, pods []corev1.P } for _, condition := range pod.Status.Conditions { if condition.Type == corev1.PodScheduled && condition.Status == corev1.ConditionFalse && condition.Reason == corev1.PodReasonUnschedulable { - return hostportCondition(gs, "EndpointUnschedulable", fmt.Sprintf("Hostport listener cannot schedule on endpoint node %q: %s", nodeName, condition.Message), false) + message := "Hostport listener cannot schedule" + if nodeName != "" { + message += fmt.Sprintf(" on endpoint node %q", nodeName) + } + return hostportCondition(gs, "EndpointUnschedulable", message+": "+condition.Message, false) } if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue && (nodeName == "" || pod.Spec.NodeName == nodeName) { return hostportCondition(gs, "ListenerReady", "The listener is Ready on the Hostport endpoint node.", true) diff --git a/operator/internal/controller/gameserver_hostport_identity_test.go b/operator/internal/controller/gameserver_hostport_identity_test.go index 03e7ad401..f1d06cc8f 100644 --- a/operator/internal/controller/gameserver_hostport_identity_test.go +++ b/operator/internal/controller/gameserver_hostport_identity_test.go @@ -3,11 +3,13 @@ package controller import ( "context" "errors" + "fmt" "reflect" "testing" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -274,6 +276,7 @@ func TestHostportSentinelMutationRejectsChangedGameServer(t *testing.T) { func TestHostportSentinelMutationRetainsListenerOnNotReadyNode(t *testing.T) { t.Parallel() r, gs, tmpl, plan, original := hostportSentinelMutationFixture(t, true) + r.SentinelImage = "sentinel:pending-upgrade" r.APIReader = fake.NewClientBuilder().WithScheme(r.Scheme).WithObjects(gs, hostportIdentityNode("node-uid", corev1.ConditionFalse)).Build() if err := r.reconcileSentinel(t.Context(), gs, tmpl, plan.wantSentinel); err != nil { t.Fatal(err) @@ -289,3 +292,85 @@ func TestHostportSentinelMutationRetainsListenerOnNotReadyNode(t *testing.T) { t.Fatal("transient NotReady rolled the listener template") } } + +func TestHostportNotReadyPlanRetainsOnlyRunningSleepListener(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + name string + idle idleState + pod string + retain bool + }{ + {"sleeping-running", idleAsleep, "running", true}, + {"awake-running", idleAwake, "running", false}, + {"sleeping-missing", idleAsleep, "missing", false}, + {"sleeping-pending", idleAsleep, "pending", false}, + {"sleeping-deleting", idleAsleep, "deleting", false}, + {"sleeping-other-node", idleAsleep, "other-node", false}, + } { + t.Run(tc.name, func(t *testing.T) { + r, gs, tmpl, _, original := hostportSentinelMutationFixture(t, true) + objects := []client.Object{gs, hostportIdentityNode("node-uid", corev1.ConditionFalse)} + if tc.pod != "missing" { + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "game-waker-pod", Namespace: gs.Namespace, Labels: map[string]string{sentinelLabel: sentinelValue, "app.kubernetes.io/instance": gs.Name}}, Spec: corev1.PodSpec{NodeName: "node-a"}, Status: corev1.PodStatus{Phase: corev1.PodRunning}} + switch tc.pod { + case "pending": + pod.Status.Phase = corev1.PodPending + case "deleting": + now := metav1.Now() + pod.DeletionTimestamp, pod.Finalizers = &now, []string{"test/hold"} + case "other-node": + pod.Spec.NodeName = "node-b" + } + objects = append(objects, pod) + } + r.APIReader = fake.NewClientBuilder().WithScheme(r.Scheme).WithObjects(objects...).Build() + plan, err := r.planSentinel(t.Context(), gs, tmpl, tc.idle) + if err != nil || plan.wantSentinel != tc.retain || !plan.blockGame { + t.Fatalf("NotReady plan = %+v, %v; retain=%t", plan, err, tc.retain) + } + r.SentinelImage = "sentinel:pending-upgrade" + if err := r.reconcileSentinel(t.Context(), gs, tmpl, plan.wantSentinel); err != nil { + t.Fatal(err) + } + var current appsv1.Deployment + err = r.Get(t.Context(), client.ObjectKeyFromObject(original), ¤t) + if !tc.retain { + if !apierrors.IsNotFound(err) { + t.Fatalf("plan retained a listener that cannot be preserved: %v", err) + } + return + } + if err != nil || !reflect.DeepEqual(original.Spec, current.Spec) { + t.Fatalf("NotReady plan changed the running listener: %v", err) + } + }) + } +} + +func TestHostportSentinelMutationDoesNotStartOnNotReadyNode(t *testing.T) { + t.Parallel() + for _, existing := range []bool{false, true} { + t.Run(fmt.Sprintf("existing=%t", existing), func(t *testing.T) { + r, gs, tmpl, plan, original := hostportSentinelMutationFixture(t, existing) + if existing { + zero := int32(0) + original.Spec.Replicas = &zero + if err := r.Update(t.Context(), original); err != nil { + t.Fatal(err) + } + } + r.APIReader = fake.NewClientBuilder().WithScheme(r.Scheme).WithObjects(gs, hostportIdentityNode("node-uid", corev1.ConditionFalse)).Build() + if err := r.reconcileSentinel(t.Context(), gs, tmpl, plan.wantSentinel); err != nil { + t.Fatal(err) + } + var current appsv1.Deployment + if err := r.Get(t.Context(), client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-waker"}, ¤t); err != nil { + t.Fatal(err) + } + if current.Spec.Replicas == nil || *current.Spec.Replicas != 0 { + t.Fatal("stale healthy plan started a listener on a NotReady node") + } + }) + } +} diff --git a/operator/internal/controller/gameserver_sentinel.go b/operator/internal/controller/gameserver_sentinel.go index 0804f3b18..86467b714 100644 --- a/operator/internal/controller/gameserver_sentinel.go +++ b/operator/internal/controller/gameserver_sentinel.go @@ -265,6 +265,14 @@ func (r *GameServerReconciler) reconcileSentinel( if dep.ResourceVersion != "" { return nil // Stop without rolling the existing pod template. } + } else if !hostportNodeReady(node) { + // Retain an existing listener's exact desired state while the + // original node recovers. Do not roll its template or start a + // stopped/new producer from a stale healthy-node plan. + if dep.ResourceVersion != "" { + return nil + } + replicas = 0 } } dep.Spec.Replicas = &replicas diff --git a/operator/internal/controller/gameserver_sentinel_envtest_test.go b/operator/internal/controller/gameserver_sentinel_envtest_test.go index f400cabb1..bf79d3c8a 100644 --- a/operator/internal/controller/gameserver_sentinel_envtest_test.go +++ b/operator/internal/controller/gameserver_sentinel_envtest_test.go @@ -187,6 +187,10 @@ func TestSentinelReconciliation(t *testing.T) { t.Fatalf("create node: %v", err) } deleteCleanup(t, node) + node.Status.Conditions = []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}} + if err := k8sClient.Status().Update(ctx, node); err != nil { + t.Fatalf("mark node Ready: %v", err) + } gs := &gameplanev1alpha1.GameServer{ ObjectMeta: metav1.ObjectMeta{Name: "test-sentinel-hostport", Namespace: ns, Annotations: map[string]string{ diff --git a/operator/specs.md b/operator/specs.md index 467cfea6e..39a474d37 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -67,7 +67,10 @@ backup replica preservation and restore startup; the existing pod template is left intact. The sentinel Deployment also rechecks the live GameServer endpoint and Node UID before mutation; a stale sleep plan cannot start a listener on a replacement node. A transient NotReady condition on the same Node UID preserves an -existing running game, while Pending or terminating Pods do not authorize a wake. +existing running game or sleeping sentinel. Sentinel planning requires a running, +non-terminating Pod on that endpoint, and mutation preserves its Deployment without +rolling the template or raising replicas while the node is NotReady. Pending or +terminating Pods do not authorize a wake or a new sleep listener. After restoring node capacity/readiness, reconciliation resumes automatically. For intentional migration, suspend the server, wait for both game and sentinel From 018d6a8cef0201c5ad89fbdb9271ae4ea2542fae Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 6 Nov 2026 01:55:54 -0500 Subject: [PATCH 12/20] fix(operator): lowercase sentinel endpoint error Signed-off-by: Baylem --- operator/internal/controller/gameserver_sentinel.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operator/internal/controller/gameserver_sentinel.go b/operator/internal/controller/gameserver_sentinel.go index 86467b714..08a4ea49d 100644 --- a/operator/internal/controller/gameserver_sentinel.go +++ b/operator/internal/controller/gameserver_sentinel.go @@ -248,7 +248,7 @@ func (r *GameServerReconciler) reconcileSentinel( if live.Spec.Networking.Expose != "Hostport" || live.Annotations[hostportNodeAnnotation] != gs.Annotations[hostportNodeAnnotation] || live.Annotations[hostportNodeUIDAnnotation] != gs.Annotations[hostportNodeUIDAnnotation] { - return fmt.Errorf("Hostport endpoint changed before sentinel mutation") + return fmt.Errorf("hostport endpoint changed before sentinel mutation") } if dep.ResourceVersion != "" && !metav1.IsControlledBy(dep, &live) { return fmt.Errorf("sentinel Deployment is not owned by this GameServer") From 22762f5bc8d4f11d25617c4fc8deccd7f1d1467d Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 6 Nov 2026 13:34:22 -0500 Subject: [PATCH 13/20] fix(wake): retry shared wake after caller cancellation Signed-off-by: Baylem --- sentinel/main.go | 52 +++++++++++++--------- sentinel/specs.md | 2 +- sentinel/wake_budget_test.go | 85 ++++++++++++++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 21 deletions(-) diff --git a/sentinel/main.go b/sentinel/main.go index 2eaec4afa..954962e98 100644 --- a/sentinel/main.go +++ b/sentinel/main.go @@ -371,8 +371,9 @@ type waker struct { } type wakeAttempt struct { - done chan struct{} - err error + done chan struct{} + err error + callerCanceled bool } func newWaker(res dynamic.ResourceInterface, name string, minInterval time.Duration) *waker { @@ -383,31 +384,42 @@ func newWaker(res dynamic.ResourceInterface, name string, minInterval time.Durat // idle-wake-requested annotation via a JSON merge patch (rather than a full // object update), so any other annotations already on the object survive. func (w *waker) RequestWake(ctx context.Context) (result error) { - if err := ctx.Err(); err != nil { - return err - } - w.mu.Lock() - now := time.Now() - if attempt := w.inFlight; attempt != nil { - w.mu.Unlock() - select { - case <-ctx.Done(): - return ctx.Err() - case <-attempt.done: - return attempt.err + var attempt *wakeAttempt + for { + if err := ctx.Err(); err != nil { + return err } - } - if !w.lastPatch.IsZero() && now.Sub(w.lastPatch) < w.minInterval { + w.mu.Lock() + now := time.Now() + if shared := w.inFlight; shared != nil { + w.mu.Unlock() + select { + case <-ctx.Done(): + return ctx.Err() + case <-shared.done: + // A live caller still owes a wake when the first caller's + // context canceled the shared PATCH. Compete for a new shared + // attempt within this caller's own remaining budget. + if shared.callerCanceled { + continue + } + return shared.err + } + } + if !w.lastPatch.IsZero() && now.Sub(w.lastPatch) < w.minInterval { + w.mu.Unlock() + return nil + } + attempt = &wakeAttempt{done: make(chan struct{})} + w.inFlight = attempt w.mu.Unlock() - return nil + break } - attempt := &wakeAttempt{done: make(chan struct{})} - w.inFlight = attempt - w.mu.Unlock() defer func() { w.mu.Lock() attempt.err = result + attempt.callerCanceled = result != nil && ctx.Err() != nil w.inFlight = nil if result == nil { w.lastPatch = time.Now() diff --git a/sentinel/specs.md b/sentinel/specs.md index 3d1a6305d..d74e9ade8 100644 --- a/sentinel/specs.md +++ b/sentinel/specs.md @@ -64,7 +64,7 @@ Single executable module; no subdirectories or packages. - **`UDP_PACKET_WINDOW`** (optional, default `10s`): Time window for counting UDP packets. - **`UDP_MAX_SOURCES`** (optional, default `4096`): Maximum number of distinct UDP sources (keyed by `ip:port`, not just IP) to track. Older sources are evicted when this limit is reached. - **`MAX_CONNECTIONS`** (optional, default `256`): Maximum concurrent TCP connection handlers. Excess connections queue in the OS accept backlog. -- **`WAKE_PATCH_INTERVAL`** (optional, default `2s`): Minimum time between GameServer annotation patches. Bursts of joining clients within this window are coalesced into one apiserver call. +- **`WAKE_PATCH_INTERVAL`** (optional, default `2s`): Minimum time between successful GameServer annotation patches. Concurrent callers share an in-flight PATCH and can cancel their own waits. If the first caller's context cancels that PATCH, live waiters retry through one new shared attempt within their own remaining budgets. API failures and the independent five-second PATCH timeout are returned to all waiters without an automatic retry. - **`SHUTDOWN_DRAIN_TIMEOUT`** (optional, default `4h`, must be `>= 0`): How long in-flight sessions are kept alive after SIGTERM before they are force-closed. `0s` closes them immediately. The operator always sets it: `4h0m0s` in the ClusterIP/NodePort/LoadBalancer expose modes (with `terminationGracePeriodSeconds` = drain + 30s, so the kubelet does not SIGKILL the pod mid-drain), and `0s` in Hostport mode (grace 30s), where the terminating sentinel would otherwise keep holding the host port the game pod needs. ### Kubernetes RBAC diff --git a/sentinel/wake_budget_test.go b/sentinel/wake_budget_test.go index 1b518742f..25e67a424 100644 --- a/sentinel/wake_budget_test.go +++ b/sentinel/wake_budget_test.go @@ -65,6 +65,91 @@ func TestWakeSharedFailureAndImmediateRetry(t *testing.T) { } } +func TestWakeLiveWaiterRetriesCanceledOwner(t *testing.T) { + started := make(chan struct{}) + var patches atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + _, _ = io.Copy(io.Discard, req.Body) + if patches.Add(1) == 1 { + close(started) + <-req.Context().Done() + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"apiVersion":"gameplane.local/v1alpha1","kind":"GameServer","metadata":{"name":"alpha"}}`) + })) + defer server.Close() + client, err := dynamic.NewForConfig(&rest.Config{Host: server.URL}) + if err != nil { + t.Fatal(err) + } + w := newWaker(client.Resource(gameServerGVR).Namespace("games"), "alpha", time.Minute) + ownerCtx, cancelOwner := context.WithCancel(t.Context()) + defer cancelOwner() + ownerDone := make(chan error, 1) + go func() { ownerDone <- w.RequestWake(ownerCtx) }() + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("owning wake did not reach the API") + } + waiting := &observedWaitContext{Context: t.Context(), observed: make(chan struct{})} + waiterDone := make(chan error, 1) + go func() { waiterDone <- w.RequestWake(waiting) }() + select { + case <-waiting.observed: + case <-time.After(time.Second): + t.Fatal("live caller did not join the shared wake") + } + // Canceling a different waiter must release only that caller, without + // canceling the owning PATCH or the live caller's shared wait. + canceledCtx, cancelWaiter := context.WithCancel(t.Context()) + defer cancelWaiter() + canceled := &observedWaitContext{Context: canceledCtx, observed: make(chan struct{})} + canceledDone := make(chan error, 1) + go func() { canceledDone <- w.RequestWake(canceled) }() + select { + case <-canceled.observed: + case <-time.After(time.Second): + t.Fatal("cancelable caller did not join the shared wake") + } + cancelWaiter() + select { + case err := <-canceledDone: + if !errors.Is(err, context.Canceled) { + t.Fatalf("canceled waiter: %v", err) + } + case <-time.After(time.Second): + t.Fatal("canceled waiter remained blocked") + } + if patches.Load() != 1 { + t.Fatal("canceling a waiter started another PATCH") + } + cancelOwner() + select { + case err := <-ownerDone: + if !errors.Is(err, context.Canceled) { + t.Fatalf("owning caller: %v", err) + } + case <-time.After(time.Second): + t.Fatal("owning caller remained blocked after cancellation") + } + select { + case err := <-waiterDone: + if err != nil { + t.Fatalf("live waiter did not complete its own wake: %v", err) + } + case <-time.After(time.Second): + t.Fatal("live waiter did not retry the canceled shared wake") + } + if err := w.RequestWake(t.Context()); err != nil { + t.Fatalf("coalesced successful wake: %v", err) + } + if patches.Load() != 2 { + t.Fatalf("got %d PATCHes, want canceled owner and one successful retry", patches.Load()) + } +} + // Done is evaluated when RequestWake waits on the shared result. This handshake // avoids relying on the scheduler running the second goroutine within a sleep. type observedWaitContext struct { From d86b7c723a729f5f9dc8b59e95034ea1ce76eac1 Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 6 Nov 2026 13:56:02 -0500 Subject: [PATCH 14/20] fix(hostport): fence replacements before graceful endpoint shutdown Signed-off-by: Baylem --- .../controller/gameserver_controller.go | 17 +- .../controller/gameserver_hostport_stop.go | 116 +++++++ .../gameserver_hostport_stop_test.go | 294 ++++++++++++++++++ operator/specs.md | 16 +- 4 files changed, 432 insertions(+), 11 deletions(-) create mode 100644 operator/internal/controller/gameserver_hostport_stop.go create mode 100644 operator/internal/controller/gameserver_hostport_stop_test.go diff --git a/operator/internal/controller/gameserver_controller.go b/operator/internal/controller/gameserver_controller.go index cae36b95e..a857f5c28 100644 --- a/operator/internal/controller/gameserver_controller.go +++ b/operator/internal/controller/gameserver_controller.go @@ -398,7 +398,11 @@ func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, err } - replicas, stopRequeue, err := r.desiredReplicas(ctx, &gs, &tmpl, idle, &wipeState) + var replicas int32 + var stopRequeue time.Duration + if !plan.blockGame { + replicas, stopRequeue, err = r.desiredReplicas(ctx, &gs, &tmpl, idle, &wipeState) + } if err != nil { if apierrors.IsConflict(err) { return ctrl.Result{RequeueAfter: lifecycleConflictRetry}, nil @@ -406,9 +410,6 @@ func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) logger.Error(err, "compute desired replicas") return ctrl.Result{}, err } - if plan.blockGame { - replicas = 0 - } if err := r.reconcileStatefulSet(ctx, &gs, &tmpl, ver, mc, replicas, &wipeState); err != nil { if apierrors.IsConflict(err) { return requeueOnConflict(ctrl.Result{}, err) @@ -1529,13 +1530,15 @@ func (r *GameServerReconciler) reconcileStatefulSet( // StatefulSet replacement and Node deletion run independently // of the earlier sentinel plan. Identity loss must also fence // stale wake decisions and lifecycle owners that retain replicas. - actualReplicas, replicas = 0, 0 - ss.Spec.Replicas = &actualReplicas if ss.ResourceVersion != "" { - return nil // Stop without rolling or changing the game template. + return r.stopHostportOnDeletingNode(ctx, &live, tmpl, ss) } + actualReplicas, replicas = 0, 0 } } + if err := clearHostportStopFence(ss); err != nil { + return err + } op, held, err := lifecycleOwner(&live) if err != nil { return err diff --git a/operator/internal/controller/gameserver_hostport_stop.go b/operator/internal/controller/gameserver_hostport_stop.go new file mode 100644 index 000000000..efe132a51 --- /dev/null +++ b/operator/internal/controller/gameserver_hostport_stop.go @@ -0,0 +1,116 @@ +package controller + +import ( + "context" + "encoding/json" + "fmt" + "slices" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +const hostportStopFenceAnnotation = "gameplane.local/hostport-stop-fence" +const hostportStopSchedulingGate = "gameplane.local/hostport-endpoint-draining" + +type hostportStopFence struct { + PodUID types.UID `json:"podUID"` + NodeUID types.UID `json:"nodeUID"` + Strategy appsv1.StatefulSetUpdateStrategy `json:"strategy"` +} + +// A deleting original node may still run the game's save sequence. Retaining +// replicas alone would also permit an independently recreated Pod to start. +// Persist OnDelete and a scheduling gate first: the original Pod is untouched, +// while every replacement remains unschedulable, even after operator restart. +// Called only inside the StatefulSet mutation after a live identity read. +func (r *GameServerReconciler) stopHostportOnDeletingNode(ctx context.Context, gs *gameplanev1alpha1.GameServer, tmpl *gameplanev1alpha1.GameTemplate, ss *appsv1.StatefulSet) error { + zero := int32(0) + if ss.ResourceVersion == "" || ss.Spec.Replicas == nil || *ss.Spec.Replicas == 0 { + ss.Spec.Replicas = &zero + return nil + } + var node corev1.Node + err := r.hostportReader().Get(ctx, client.ObjectKey{Name: gs.Annotations[hostportNodeAnnotation]}, &node) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + if err != nil || node.DeletionTimestamp.IsZero() || string(node.UID) != gs.Annotations[hostportNodeUIDAnnotation] { + ss.Spec.Replicas = &zero + return nil + } + var pod corev1.Pod + err = r.hostportReader().Get(ctx, client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-0"}, &pod) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + if err != nil || !metav1.IsControlledBy(ss, gs) || !metav1.IsControlledBy(&pod, ss) || pod.UID == "" || pod.Spec.NodeName != node.Name || !pod.DeletionTimestamp.IsZero() || pod.Status.Phase != corev1.PodRunning { + ss.Spec.Replicas = &zero + return nil + } + fence := hostportStopFence{PodUID: pod.UID, NodeUID: node.UID, Strategy: *ss.Spec.UpdateStrategy.DeepCopy()} + if raw := ss.Annotations[hostportStopFenceAnnotation]; raw != "" { + if err := json.Unmarshal([]byte(raw), &fence); err != nil || fence.PodUID != pod.UID || fence.NodeUID != node.UID { + ss.Spec.Replicas = &zero + return nil + } + if ss.Spec.UpdateStrategy.Type == appsv1.OnDeleteStatefulSetStrategyType && slices.ContainsFunc(ss.Spec.Template.Spec.SchedulingGates, func(gate corev1.PodSchedulingGate) bool { return gate.Name == hostportStopSchedulingGate }) { + if ss.Status.ObservedGeneration < ss.Generation { + return nil // Wait until the StatefulSet controller has observed the fence. + } + op, held, err := lifecycleOwner(gs) + if err != nil { + return err + } + if held && (op.Kind == "Backup" || len(op.Recovery) != 0) { + // A save/stop RPC must not write while another copy owns the + // world. Preserve that owner's cleanup contract; orphaned + // Restore recovery still requires the existing zero-replica stop. + if lifecycleRecoveryNeedsStop(op) { + ss.Spec.Replicas = &zero + } + return nil + } + replicas, _, err := r.softStop(ctx, gs, tmpl) + if err != nil { + return err + } + ss.Spec.Replicas = &replicas + return nil + } + } + data, err := json.Marshal(fence) + if err != nil { + return fmt.Errorf("encode Hostport stop fence: %w", err) + } + if ss.Annotations == nil { + ss.Annotations = map[string]string{} + } + ss.Annotations[hostportStopFenceAnnotation] = string(data) + ss.Spec.UpdateStrategy = appsv1.StatefulSetUpdateStrategy{Type: appsv1.OnDeleteStatefulSetStrategyType} + if !slices.ContainsFunc(ss.Spec.Template.Spec.SchedulingGates, func(gate corev1.PodSchedulingGate) bool { return gate.Name == hostportStopSchedulingGate }) { + ss.Spec.Template.Spec.SchedulingGates = append(ss.Spec.Template.Spec.SchedulingGates, corev1.PodSchedulingGate{Name: hostportStopSchedulingGate}) + } + return nil // Commit the replacement fence before issuing the stop RPC. +} + +func clearHostportStopFence(ss *appsv1.StatefulSet) error { + raw := ss.Annotations[hostportStopFenceAnnotation] + if raw == "" { + return nil + } + var fence hostportStopFence + if err := json.Unmarshal([]byte(raw), &fence); err != nil { + return fmt.Errorf("decode Hostport stop fence: %w", err) + } + ss.Spec.UpdateStrategy = fence.Strategy + ss.Spec.Template.Spec.SchedulingGates = slices.DeleteFunc(ss.Spec.Template.Spec.SchedulingGates, func(gate corev1.PodSchedulingGate) bool { return gate.Name == hostportStopSchedulingGate }) + delete(ss.Annotations, hostportStopFenceAnnotation) + return nil +} diff --git a/operator/internal/controller/gameserver_hostport_stop_test.go b/operator/internal/controller/gameserver_hostport_stop_test.go new file mode 100644 index 000000000..f3db432d2 --- /dev/null +++ b/operator/internal/controller/gameserver_hostport_stop_test.go @@ -0,0 +1,294 @@ +package controller + +import ( + "reflect" + "slices" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +func hostportStopFixture(t *testing.T) (*GameServerReconciler, *gameplanev1alpha1.GameServer, *gameplanev1alpha1.GameTemplate, *lifecycleStopper) { + t.Helper() + node := hostportIdentityNode("original-node", corev1.ConditionTrue) + now := metav1.Now() + node.DeletionTimestamp, node.Finalizers = &now, []string{"test/hold"} + r, gs, tmpl := hostportIdentityFixture(t, node, corev1.PodRunning, nil) + grace := int32(60) + gs.Spec.StopGracePeriodSeconds = &grace + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + tmpl.Spec.RCON = &gameplanev1alpha1.RCONSpec{Protocol: "source"} + tmpl.Spec.Capabilities = &gameplanev1alpha1.CapabilitiesSpec{Lifecycle: &gameplanev1alpha1.LifecycleSpec{Stop: []string{"save", "stop"}}} + var ss appsv1.StatefulSet + if err := r.Get(t.Context(), client.ObjectKeyFromObject(gs), &ss); err != nil { + t.Fatal(err) + } + ss.Generation = 1 + ss.Spec.UpdateStrategy = appsv1.StatefulSetUpdateStrategy{Type: appsv1.RollingUpdateStatefulSetStrategyType} + if err := r.Update(t.Context(), &ss); err != nil { + t.Fatal(err) + } + ss.Status.ReadyReplicas = 1 + if err := r.Status().Update(t.Context(), &ss); err != nil { + t.Fatal(err) + } + var pod corev1.Pod + if err := r.Get(t.Context(), client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-0"}, &pod); err != nil { + t.Fatal(err) + } + pod.UID = "original-pod" + pod.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(&ss, appsv1.SchemeGroupVersion.WithKind("StatefulSet"))} + if err := r.Update(t.Context(), &pod); err != nil { + t.Fatal(err) + } + stopper := &lifecycleStopper{} + r.AgentClient = stopper + return r, gs, tmpl, stopper +} + +func hostportStopSet(t *testing.T, r *GameServerReconciler, gs *gameplanev1alpha1.GameServer) *appsv1.StatefulSet { + t.Helper() + var ss appsv1.StatefulSet + if err := r.Get(t.Context(), client.ObjectKeyFromObject(gs), &ss); err != nil { + t.Fatal(err) + } + return &ss +} + +func TestHostportDeletingOriginalNodeFencesReplacementsBeforeGracefulStop(t *testing.T) { + r, gs, tmpl, stopper := hostportStopFixture(t) + original := hostportStopSet(t, r, gs).Spec.Template.DeepCopy() + for pass := range 3 { + if err := r.Get(t.Context(), client.ObjectKeyFromObject(gs), gs); err != nil { + t.Fatal(err) + } + // replicas=0 models an already-blocked plan. The same mutation gate + // must also protect a stale healthy plan passing replicas=1. + replicas := int32(0) + if pass == 0 { + replicas = 1 + } + if err := r.reconcileStatefulSet(t.Context(), gs, tmpl, nil, &materializedConfig{}, replicas); err != nil { + t.Fatal(err) + } + ss := hostportStopSet(t, r, gs) + if *ss.Spec.Replicas != 1 || ss.Spec.UpdateStrategy.Type != appsv1.OnDeleteStatefulSetStrategyType || !slices.ContainsFunc(ss.Spec.Template.Spec.SchedulingGates, func(g corev1.PodSchedulingGate) bool { return g.Name == hostportStopSchedulingGate }) { + t.Fatal("graceful stop lost original Pod or replacement scheduling fence") + } + withoutGate := ss.Spec.Template.DeepCopy() + withoutGate.Spec.SchedulingGates = nil + if !reflect.DeepEqual(original, withoutGate) { + t.Fatal("endpoint drain changed the running game's template") + } + wantCalls := 1 + if pass == 0 { + wantCalls = 0 + } + if stopper.calls != wantCalls { + t.Fatalf("pass %d: stop calls=%d, want=%d", pass, stopper.calls, wantCalls) + } + if pass == 0 { + ss.Status.ObservedGeneration = ss.Generation + if err := r.Status().Update(t.Context(), ss); err != nil { + t.Fatal(err) + } + } + // Recreate the reconciler: durable gate and grace timestamp must be + // sufficient to continue without another stop command. + r = &GameServerReconciler{Client: r.Client, APIReader: r.APIReader, Scheme: r.Scheme, AgentClient: stopper} + } + if err := r.Get(t.Context(), client.ObjectKeyFromObject(gs), gs); err != nil { + t.Fatal(err) + } + gs.Annotations[stopRequestedAtAnnotation] = time.Now().Add(-2 * time.Minute).UTC().Format(time.RFC3339) + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + if err := r.reconcileStatefulSet(t.Context(), gs, tmpl, nil, &materializedConfig{}, 0); err != nil { + t.Fatal(err) + } + if *hostportStopSet(t, r, gs).Spec.Replicas != 0 { + t.Fatal("grace deadline did not stop original workload") + } +} + +func TestHostportStopFenceRejectsReplacementPodAndRestoresAfterMigration(t *testing.T) { + r, gs, tmpl, _ := hostportStopFixture(t) + originalSet := hostportStopSet(t, r, gs) + originalSet.Spec.Template.Spec.SchedulingGates = []corev1.PodSchedulingGate{{Name: "example.invalid/existing-gate"}} + if err := r.Update(t.Context(), originalSet); err != nil { + t.Fatal(err) + } + if err := r.reconcileStatefulSet(t.Context(), gs, tmpl, nil, &materializedConfig{}, 1); err != nil { + t.Fatal(err) + } + var pod corev1.Pod + if err := r.Get(t.Context(), client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-0"}, &pod); err != nil { + t.Fatal(err) + } + pod.UID = "replacement-pod" + if err := r.Update(t.Context(), &pod); err != nil { + t.Fatal(err) + } + if err := r.reconcileStatefulSet(t.Context(), gs, tmpl, nil, &materializedConfig{}, 1); err != nil { + t.Fatal(err) + } + if *hostportStopSet(t, r, gs).Spec.Replicas != 0 { + t.Fatal("replacement Pod inherited original Pod's save window") + } + // Explicit endpoint migration follows the documented stopped/drained + // procedure. Ordinary rendering removes only our temporary scheduling gate. + if err := r.Delete(t.Context(), &pod); err != nil { + t.Fatal(err) + } + delete(gs.Annotations, hostportNodeAnnotation) + delete(gs.Annotations, hostportNodeUIDAnnotation) + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + if err := r.reconcileStatefulSet(t.Context(), gs, tmpl, nil, &materializedConfig{}, 0); err != nil { + t.Fatal(err) + } + ss := hostportStopSet(t, r, gs) + if ss.Annotations[hostportStopFenceAnnotation] != "" || ss.Spec.UpdateStrategy.Type != appsv1.RollingUpdateStatefulSetStrategyType || !reflect.DeepEqual(ss.Spec.Template.Spec.SchedulingGates, originalSet.Spec.Template.Spec.SchedulingGates) { + t.Fatal("migration retained temporary drain fence") + } +} + +func TestHostportStopWaitsForFenceObservation(t *testing.T) { + r, gs, tmpl, stopper := hostportStopFixture(t) + ss := hostportStopSet(t, r, gs) + ss.Generation = 9 + if err := r.Update(t.Context(), ss); err != nil { + t.Fatal(err) + } + ss.Status.ObservedGeneration = 8 + if err := r.Status().Update(t.Context(), ss); err != nil { + t.Fatal(err) + } + for range 2 { + if err := r.reconcileStatefulSet(t.Context(), gs, tmpl, nil, &materializedConfig{}, 1); err != nil { + t.Fatal(err) + } + } + if stopper.calls != 0 || *hostportStopSet(t, r, gs).Spec.Replicas != 1 { + t.Fatal("stop started before replacement fence was observed") + } + ss = hostportStopSet(t, r, gs) + ss.Status.ObservedGeneration = ss.Generation + if err := r.Status().Update(t.Context(), ss); err != nil { + t.Fatal(err) + } + if err := r.reconcileStatefulSet(t.Context(), gs, tmpl, nil, &materializedConfig{}, 1); err != nil { + t.Fatal(err) + } + if stopper.calls != 1 { + t.Fatal("observed replacement fence did not permit graceful stop") + } +} + +func TestHostportValidEndpointRestoresFencedStrategy(t *testing.T) { + r, gs, tmpl, _ := hostportStopFixture(t) + if err := r.reconcileStatefulSet(t.Context(), gs, tmpl, nil, &materializedConfig{}, 1); err != nil { + t.Fatal(err) + } + // A fresh healthy endpoint read governs cleanup, independently of the + // previous cached deleting-node plan. + r.APIReader = fake.NewClientBuilder().WithScheme(r.Scheme).WithObjects(gs, hostportIdentityNode("original-node", corev1.ConditionTrue)).Build() + if err := r.reconcileStatefulSet(t.Context(), gs, tmpl, nil, &materializedConfig{}, 1); err != nil { + t.Fatal(err) + } + ss := hostportStopSet(t, r, gs) + if ss.Annotations[hostportStopFenceAnnotation] != "" || ss.Spec.UpdateStrategy.Type != appsv1.RollingUpdateStatefulSetStrategyType || slices.ContainsFunc(ss.Spec.Template.Spec.SchedulingGates, func(g corev1.PodSchedulingGate) bool { return g.Name == hostportStopSchedulingGate }) { + t.Fatal("valid endpoint retained temporary stop fence") + } +} + +func TestHostportDeletingNodeDoesNotGrantGraceToUntrustedPod(t *testing.T) { + for _, mode := range []string{"pending", "unowned", "other-node"} { + t.Run(mode, func(t *testing.T) { + r, gs, tmpl, stopper := hostportStopFixture(t) + var pod corev1.Pod + if err := r.Get(t.Context(), client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-0"}, &pod); err != nil { + t.Fatal(err) + } + switch mode { + case "pending": + pod.Status.Phase = corev1.PodPending + case "unowned": + pod.OwnerReferences = nil + case "other-node": + pod.Spec.NodeName = "node-b" + } + if err := r.Update(t.Context(), &pod); err != nil { + t.Fatal(err) + } + if mode == "pending" { + pod.Status.Phase = corev1.PodPending + if err := r.Status().Update(t.Context(), &pod); err != nil { + t.Fatal(err) + } + } + if err := r.reconcileStatefulSet(t.Context(), gs, tmpl, nil, &materializedConfig{}, 1); err != nil { + t.Fatal(err) + } + if *hostportStopSet(t, r, gs).Spec.Replicas != 0 || stopper.calls != 0 { + t.Fatal("untrusted Pod bypassed endpoint identity fence") + } + }) + } +} + +func TestHostportDeletingNodeWaitsForCopyOwnerBeforeSaveStop(t *testing.T) { + for _, recovery := range []bool{false, true} { + t.Run(map[bool]string{false: "backup", true: "recovery"}[recovery], func(t *testing.T) { + r, gs, tmpl, stopper := hostportStopFixture(t) + op := lifecycleOperation{lifecycleIdentity: lifecycleIdentity{Kind: "Backup", Name: "copy", UID: "copy-uid", TargetUID: gs.UID}} + if recovery { + op.Kind, op.Name, op.UID = "Wipe", gs.Name, gs.UID + op.Token = "wipe" + op.Recovery = []lifecycleIdentity{{Kind: "Backup", Name: "old-copy", UID: "old-copy-uid", TargetUID: gs.UID}} + } + gs.Annotations[lifecycleOperationAnnotation] = lifecycleOperationJSON(op) + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + if err := r.reconcileStatefulSet(t.Context(), gs, tmpl, nil, &materializedConfig{}, 0); err != nil { + t.Fatal(err) + } + ss := hostportStopSet(t, r, gs) + ss.Status.ObservedGeneration = ss.Generation + if err := r.Status().Update(t.Context(), ss); err != nil { + t.Fatal(err) + } + if err := r.reconcileStatefulSet(t.Context(), gs, tmpl, nil, &materializedConfig{}, 0); err != nil { + t.Fatal(err) + } + if err := r.Get(t.Context(), client.ObjectKeyFromObject(gs), gs); err != nil { + t.Fatal(err) + } + if stopper.calls != 0 || gs.Annotations[stopRequestedAtAnnotation] != "" || *hostportStopSet(t, r, gs).Spec.Replicas != 1 { + t.Fatal("endpoint stop RPC overlapped copy/save-off ownership") + } + delete(gs.Annotations, lifecycleOperationAnnotation) + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + if err := r.reconcileStatefulSet(t.Context(), gs, tmpl, nil, &materializedConfig{}, 0); err != nil { + t.Fatal(err) + } + if stopper.calls != 1 { + t.Fatal("copy cleanup did not permit graceful stop") + } + }) + } +} diff --git a/operator/specs.md b/operator/specs.md index 39a474d37..48ddf789a 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -60,11 +60,19 @@ five-second backstop runs only while `HostportWakeReady` is not True. Endpoint diagnostics never replace an existing `Ready=False` game failure. Pending-listener messages omit the endpoint node name until it has been recorded. -A missing, deleting or replaced endpoint Node forces the game StatefulSet to -zero replicas even if a replacement game Pod already exists. The live Node +A missing or replaced endpoint Node forces the game StatefulSet to zero replicas +even if a replacement game Pod already exists. A deleting Node with the original +UID permits only its owned original running game Pod to finish a bounded graceful +stop. Before issuing the stop, the operator durably switches the StatefulSet to +OnDelete and gates scheduling of replacement Pods, waiting for that generation to +be observed. The running Pod is not rolled. The save deadline/readiness check then +scales to zero; a Backup or inherited copy recovery retains its cleanup precedence +before any save/stop RPC. Pending, foreign or replacement Pods receive no save window. The +temporary gate and previous update strategy are restored during valid endpoint +reconciliation or the documented explicit migration. The live Node identity is checked again immediately before workload mutation, including -backup replica preservation and restore startup; the existing pod template is -left intact. The sentinel Deployment also rechecks the live GameServer endpoint +backup replica preservation and restore startup; existing container and scheduling +settings are otherwise retained. The sentinel Deployment also rechecks the live GameServer endpoint and Node UID before mutation; a stale sleep plan cannot start a listener on a replacement node. A transient NotReady condition on the same Node UID preserves an existing running game or sleeping sentinel. Sentinel planning requires a running, From 3d86dd906e72acc2b80df2a0060ec4736dad83aa Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 6 Nov 2026 14:53:03 -0500 Subject: [PATCH 15/20] fix(deps): carry x/net image security update Apply canonical dependency fix 09831c3978fa57bd4d2954db1fd2392653d755e1 for GO-2026-6860 (CVE-2026-78661), including the isolated frpc module and required graph minimums. Retain existing image security gates. Signed-off-by: Baylem --- agent/go.mod | 6 +++--- agent/go.sum | 6 ++++++ api/go.mod | 2 +- api/go.sum | 2 ++ mcp-server/go.mod | 10 +++++----- mcp-server/go.sum | 11 +++++++++++ operator/go.mod | 2 +- operator/go.sum | 2 ++ sentinel/go.mod | 8 ++++---- sentinel/go.sum | 8 ++++++++ tunnel/frpc/go.mod | 2 +- tunnel/frpc/go.sum | 2 ++ tunnel/go.mod | 2 +- tunnel/go.sum | 2 ++ 14 files changed, 49 insertions(+), 16 deletions(-) diff --git a/agent/go.mod b/agent/go.mod index 426be7f19..122b11f87 100644 --- a/agent/go.mod +++ b/agent/go.mod @@ -53,10 +53,10 @@ require ( github.com/x448/float16 v0.8.4 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect - golang.org/x/net v0.60.0 // indirect + golang.org/x/net v0.62.0 // indirect golang.org/x/oauth2 v0.37.0 // indirect - golang.org/x/term v0.46.0 // indirect - golang.org/x/text v0.42.0 // indirect + golang.org/x/term v0.47.0 // indirect + golang.org/x/text v0.43.0 // indirect golang.org/x/time v0.15.0 // indirect google.golang.org/protobuf v1.36.12 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect diff --git a/agent/go.sum b/agent/go.sum index d98b2a11e..5972cf8a8 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -100,14 +100,20 @@ go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= +golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58= golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= +golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM= +golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY= golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= +golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= diff --git a/api/go.mod b/api/go.mod index 21e3fc1de..56ba6a627 100644 --- a/api/go.mod +++ b/api/go.mod @@ -110,7 +110,7 @@ require ( go.uber.org/zap v1.28.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect - golang.org/x/net v0.60.0 // indirect + golang.org/x/net v0.62.0 // indirect golang.org/x/term v0.47.0 // indirect golang.org/x/time v0.15.0 // indirect gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect diff --git a/api/go.sum b/api/go.sum index 9060b639a..5c9bca2c6 100644 --- a/api/go.sum +++ b/api/go.sum @@ -193,6 +193,8 @@ golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg= golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs= golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= +golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58= golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= diff --git a/mcp-server/go.mod b/mcp-server/go.mod index 7147c09c2..9ce5f3f88 100644 --- a/mcp-server/go.mod +++ b/mcp-server/go.mod @@ -57,12 +57,12 @@ require ( go.uber.org/zap v1.28.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect - golang.org/x/net v0.60.0 // indirect + golang.org/x/net v0.62.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sync v0.23.0 // indirect - golang.org/x/sys v0.48.0 // indirect - golang.org/x/term v0.46.0 // indirect - golang.org/x/text v0.42.0 // indirect + golang.org/x/sync v0.24.0 // indirect + golang.org/x/sys v0.49.0 // indirect + golang.org/x/term v0.47.0 // indirect + golang.org/x/text v0.43.0 // indirect golang.org/x/time v0.15.0 // indirect gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect diff --git a/mcp-server/go.sum b/mcp-server/go.sum index 8172723c7..6539daf59 100644 --- a/mcp-server/go.sum +++ b/mcp-server/go.sum @@ -136,20 +136,31 @@ golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= +golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= +golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= +golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= +golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM= +golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY= golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= +golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= +golang.org/x/tools v0.51.0 h1:k4Xc/1Om9jwkBJBo4NVLMSARBoWtK10mx+W5BnXCeAI= gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0= gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= diff --git a/operator/go.mod b/operator/go.mod index ed1f8736c..413a23fa1 100644 --- a/operator/go.mod +++ b/operator/go.mod @@ -152,7 +152,7 @@ require ( go.uber.org/zap v1.28.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect - golang.org/x/net v0.60.0 // indirect + golang.org/x/net v0.62.0 // indirect golang.org/x/oauth2 v0.37.0 // indirect golang.org/x/sync v0.24.0 // indirect golang.org/x/term v0.47.0 // indirect diff --git a/operator/go.sum b/operator/go.sum index 3a83a8a3f..dc7b5700b 100644 --- a/operator/go.sum +++ b/operator/go.sum @@ -583,6 +583,8 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= +golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/sentinel/go.mod b/sentinel/go.mod index f0a24254b..023341121 100644 --- a/sentinel/go.mod +++ b/sentinel/go.mod @@ -38,11 +38,11 @@ require ( github.com/x448/float16 v0.8.4 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect - golang.org/x/net v0.60.0 // indirect + golang.org/x/net v0.62.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sys v0.48.0 // indirect - golang.org/x/term v0.46.0 // indirect - golang.org/x/text v0.42.0 // indirect + golang.org/x/sys v0.49.0 // indirect + golang.org/x/term v0.47.0 // indirect + golang.org/x/text v0.43.0 // indirect golang.org/x/time v0.15.0 // indirect google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect diff --git a/sentinel/go.sum b/sentinel/go.sum index 926892929..6b2bfd7a7 100644 --- a/sentinel/go.sum +++ b/sentinel/go.sum @@ -78,14 +78,22 @@ go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= +golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= +golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= +golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM= +golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY= golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= +golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= diff --git a/tunnel/frpc/go.mod b/tunnel/frpc/go.mod index cdc68e414..b7319cbd0 100644 --- a/tunnel/frpc/go.mod +++ b/tunnel/frpc/go.mod @@ -38,7 +38,7 @@ require ( github.com/vishvananda/netns v0.0.4 // indirect github.com/xtaci/kcp-go/v5 v5.6.13 // indirect golang.org/x/crypto v0.58.0 // indirect - golang.org/x/net v0.60.0 // indirect + golang.org/x/net v0.62.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sync v0.24.0 // indirect golang.org/x/sys v0.49.0 // indirect diff --git a/tunnel/frpc/go.sum b/tunnel/frpc/go.sum index 73d2adc13..dab154d3d 100644 --- a/tunnel/frpc/go.sum +++ b/tunnel/frpc/go.sum @@ -111,6 +111,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= +golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= diff --git a/tunnel/go.mod b/tunnel/go.mod index fc3027b00..3e492ebc3 100644 --- a/tunnel/go.mod +++ b/tunnel/go.mod @@ -38,7 +38,7 @@ require ( go4.org/netipx v0.0.0-20260823151212-3075585bcbeb // indirect golang.org/x/crypto v0.58.0 // indirect golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba // indirect - golang.org/x/net v0.61.0 // indirect + golang.org/x/net v0.62.0 // indirect golang.org/x/oauth2 v0.37.0 // indirect golang.org/x/sync v0.24.0 // indirect golang.org/x/sys v0.49.0 // indirect diff --git a/tunnel/go.sum b/tunnel/go.sum index ab6f5885c..eed4f09ba 100644 --- a/tunnel/go.sum +++ b/tunnel/go.sum @@ -204,6 +204,8 @@ golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= golang.org/x/net v0.61.0 h1:RR+6j/BTBCrggOLugxa+66R8FvoN6MHlzDlZIbN9YFw= golang.org/x/net v0.61.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= +golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= +golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From 50c40c7bc4a37ddc9f177f2984301f0d93a292f7 Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 6 Nov 2026 15:32:43 -0500 Subject: [PATCH 16/20] fix(deps): carry normalized source module checksums Apply reviewed canonical checksum normalization 0cc80c120a23e405e58c8a10790301275e7732a7 after the x/net image security update. Keep the isolated frpc module and all dependency selections unchanged. Signed-off-by: Baylem --- agent/go.sum | 6 ------ api/go.sum | 2 -- mcp-server/go.sum | 13 +------------ operator/go.sum | 2 -- sentinel/go.sum | 8 -------- tunnel/go.sum | 2 -- 6 files changed, 1 insertion(+), 32 deletions(-) diff --git a/agent/go.sum b/agent/go.sum index 5972cf8a8..4e4caf01e 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -98,20 +98,14 @@ go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= -golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= -golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58= golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= -golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= -golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM= golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY= -golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= -golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= diff --git a/api/go.sum b/api/go.sum index 5c9bca2c6..1eb60b7e3 100644 --- a/api/go.sum +++ b/api/go.sum @@ -191,8 +191,6 @@ golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w= golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg= golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs= -golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= -golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= diff --git a/mcp-server/go.sum b/mcp-server/go.sum index 6539daf59..df19a5b8c 100644 --- a/mcp-server/go.sum +++ b/mcp-server/go.sum @@ -134,33 +134,22 @@ go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= -golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= -golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= -golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= -golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= -golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= -golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= -golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= -golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM= golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY= -golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= -golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= -golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= -golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= golang.org/x/tools v0.51.0 h1:k4Xc/1Om9jwkBJBo4NVLMSARBoWtK10mx+W5BnXCeAI= +golang.org/x/tools v0.51.0/go.mod h1:9eEncMayCV6zRMGhR5eZEC2iBx98qWcF1HZ9Z7wJOoA= gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0= gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= diff --git a/operator/go.sum b/operator/go.sum index dc7b5700b..9025bd04e 100644 --- a/operator/go.sum +++ b/operator/go.sum @@ -581,8 +581,6 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= -golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= diff --git a/sentinel/go.sum b/sentinel/go.sum index 6b2bfd7a7..85e3e603a 100644 --- a/sentinel/go.sum +++ b/sentinel/go.sum @@ -76,22 +76,14 @@ go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= -golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= -golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= -golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= -golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= -golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= -golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM= golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY= -golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= -golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= diff --git a/tunnel/go.sum b/tunnel/go.sum index eed4f09ba..df5b45492 100644 --- a/tunnel/go.sum +++ b/tunnel/go.sum @@ -202,8 +202,6 @@ golang.org/x/image v0.46.0 h1:b1+oYj0Jbp6K5MDT4i4/eZpYlk3V8SJhhDKh6LBHAyQ= golang.org/x/image v0.46.0/go.mod h1:3B3W05VGVQyuXucLINLjXKrqISASfi4Xj+iCVkLMwew= golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= -golang.org/x/net v0.61.0 h1:RR+6j/BTBCrggOLugxa+66R8FvoN6MHlzDlZIbN9YFw= -golang.org/x/net v0.61.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= From 424ddadd72b1dd9c8418f625815975afef6a33d8 Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 6 Nov 2026 16:40:03 -0500 Subject: [PATCH 17/20] fix(deps): carry workspace module graph checksums Apply reviewed canonical workspace metadata fix cfb7258c179b0235b6216e288229e31bf6c160ab. Retain historical x/net v0.60.0 and x/tools v0.49.0 go.mod checksums needed by workspace generation without changing selected source versions. Signed-off-by: Baylem --- go.work.sum | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go.work.sum b/go.work.sum index c733fe949..8e5e543d3 100644 --- a/go.work.sum +++ b/go.work.sum @@ -893,6 +893,7 @@ golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= @@ -983,6 +984,7 @@ golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0= golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0= golang.org/x/tools/go/expect v0.1.0-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA= From c986a69219d056208ad0cf77f35523a66e8f10e8 Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 6 Nov 2026 17:05:52 -0500 Subject: [PATCH 18/20] fix(hostport): distinguish missing resources from invalid stop fences Signed-off-by: Baylem --- .../controller/gameserver_hostport_stop.go | 34 ++++++++++++++----- .../gameserver_hostport_stop_test.go | 25 ++++++++++++++ 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/operator/internal/controller/gameserver_hostport_stop.go b/operator/internal/controller/gameserver_hostport_stop.go index efe132a51..9501b9e6d 100644 --- a/operator/internal/controller/gameserver_hostport_stop.go +++ b/operator/internal/controller/gameserver_hostport_stop.go @@ -37,29 +37,35 @@ func (r *GameServerReconciler) stopHostportOnDeletingNode(ctx context.Context, g return nil } var node corev1.Node - err := r.hostportReader().Get(ctx, client.ObjectKey{Name: gs.Annotations[hostportNodeAnnotation]}, &node) - if err != nil && !apierrors.IsNotFound(err) { - return err + if err := r.hostportReader().Get(ctx, client.ObjectKey{Name: gs.Annotations[hostportNodeAnnotation]}, &node); err != nil { + if apierrors.IsNotFound(err) { + ss.Spec.Replicas = &zero + } + return client.IgnoreNotFound(err) } - if err != nil || node.DeletionTimestamp.IsZero() || string(node.UID) != gs.Annotations[hostportNodeUIDAnnotation] { + if node.DeletionTimestamp.IsZero() || string(node.UID) != gs.Annotations[hostportNodeUIDAnnotation] { ss.Spec.Replicas = &zero return nil } var pod corev1.Pod - err = r.hostportReader().Get(ctx, client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-0"}, &pod) - if err != nil && !apierrors.IsNotFound(err) { - return err + if err := r.hostportReader().Get(ctx, client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-0"}, &pod); err != nil { + if apierrors.IsNotFound(err) { + ss.Spec.Replicas = &zero + } + return client.IgnoreNotFound(err) } - if err != nil || !metav1.IsControlledBy(ss, gs) || !metav1.IsControlledBy(&pod, ss) || pod.UID == "" || pod.Spec.NodeName != node.Name || !pod.DeletionTimestamp.IsZero() || pod.Status.Phase != corev1.PodRunning { + if !metav1.IsControlledBy(ss, gs) || !metav1.IsControlledBy(&pod, ss) || pod.UID == "" || pod.Spec.NodeName != node.Name || !pod.DeletionTimestamp.IsZero() || pod.Status.Phase != corev1.PodRunning { ss.Spec.Replicas = &zero return nil } fence := hostportStopFence{PodUID: pod.UID, NodeUID: node.UID, Strategy: *ss.Spec.UpdateStrategy.DeepCopy()} if raw := ss.Annotations[hostportStopFenceAnnotation]; raw != "" { - if err := json.Unmarshal([]byte(raw), &fence); err != nil || fence.PodUID != pod.UID || fence.NodeUID != node.UID { + saved, valid := decodeHostportStopFence(raw) + if !valid || saved.PodUID != pod.UID || saved.NodeUID != node.UID { ss.Spec.Replicas = &zero return nil } + fence = saved if ss.Spec.UpdateStrategy.Type == appsv1.OnDeleteStatefulSetStrategyType && slices.ContainsFunc(ss.Spec.Template.Spec.SchedulingGates, func(gate corev1.PodSchedulingGate) bool { return gate.Name == hostportStopSchedulingGate }) { if ss.Status.ObservedGeneration < ss.Generation { return nil // Wait until the StatefulSet controller has observed the fence. @@ -100,6 +106,16 @@ func (r *GameServerReconciler) stopHostportOnDeletingNode(ctx context.Context, g return nil // Commit the replacement fence before issuing the stop RPC. } +// Malformed persisted state denies graceful-stop permission. It is a validation +// failure, rather than an operational error that could prevent scaling to zero. +func decodeHostportStopFence(raw string) (hostportStopFence, bool) { + var fence hostportStopFence + if err := json.Unmarshal([]byte(raw), &fence); err != nil { + return fence, false + } + return fence, true +} + func clearHostportStopFence(ss *appsv1.StatefulSet) error { raw := ss.Annotations[hostportStopFenceAnnotation] if raw == "" { diff --git a/operator/internal/controller/gameserver_hostport_stop_test.go b/operator/internal/controller/gameserver_hostport_stop_test.go index f3db432d2..4b08ba69e 100644 --- a/operator/internal/controller/gameserver_hostport_stop_test.go +++ b/operator/internal/controller/gameserver_hostport_stop_test.go @@ -292,3 +292,28 @@ func TestHostportDeletingNodeWaitsForCopyOwnerBeforeSaveStop(t *testing.T) { }) } } + +func TestHostportMalformedStopFenceCannotGrantGrace(t *testing.T) { + for _, raw := range []string{"malformed fence", "{}", "null"} { + t.Run(raw, func(t *testing.T) { + r, gs, tmpl, stopper := hostportStopFixture(t) + ss := hostportStopSet(t, r, gs) + ss.Annotations = map[string]string{hostportStopFenceAnnotation: raw} + ss.Spec.UpdateStrategy = appsv1.StatefulSetUpdateStrategy{Type: appsv1.OnDeleteStatefulSetStrategyType} + ss.Spec.Template.Spec.SchedulingGates = []corev1.PodSchedulingGate{{Name: hostportStopSchedulingGate}} + if err := r.Update(t.Context(), ss); err != nil { + t.Fatal(err) + } + ss.Status.ObservedGeneration = ss.Generation + if err := r.Status().Update(t.Context(), ss); err != nil { + t.Fatal(err) + } + if err := r.reconcileStatefulSet(t.Context(), gs, tmpl, nil, &materializedConfig{}, 1); err != nil { + t.Fatal(err) + } + if *hostportStopSet(t, r, gs).Spec.Replicas != 0 || stopper.calls != 0 { + t.Fatal("malformed or incomplete fence authorized grace or a stop RPC") + } + }) + } +} From 9acf960ab490ddf615a5264d2a9b562dd09f3250 Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 6 Nov 2026 18:05:12 -0500 Subject: [PATCH 19/20] fix(hostport): recover invalid stop fences without blocking power intent Signed-off-by: Baylem --- .../controller/gameserver_controller.go | 4 +- .../controller/gameserver_hostport_stop.go | 53 ++++++++++-- .../gameserver_hostport_stop_test.go | 81 +++++++++++++++++++ operator/specs.md | 6 +- 4 files changed, 132 insertions(+), 12 deletions(-) diff --git a/operator/internal/controller/gameserver_controller.go b/operator/internal/controller/gameserver_controller.go index a857f5c28..b140139fb 100644 --- a/operator/internal/controller/gameserver_controller.go +++ b/operator/internal/controller/gameserver_controller.go @@ -1536,9 +1536,7 @@ func (r *GameServerReconciler) reconcileStatefulSet( actualReplicas, replicas = 0, 0 } } - if err := clearHostportStopFence(ss); err != nil { - return err - } + clearHostportStopFence(ss) op, held, err := lifecycleOwner(&live) if err != nil { return err diff --git a/operator/internal/controller/gameserver_hostport_stop.go b/operator/internal/controller/gameserver_hostport_stop.go index 9501b9e6d..9dadd8b12 100644 --- a/operator/internal/controller/gameserver_hostport_stop.go +++ b/operator/internal/controller/gameserver_hostport_stop.go @@ -5,12 +5,16 @@ import ( "encoding/json" "fmt" "slices" + "strconv" + "strings" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/validation" "sigs.k8s.io/controller-runtime/pkg/client" gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" @@ -113,20 +117,53 @@ func decodeHostportStopFence(raw string) (hostportStopFence, bool) { if err := json.Unmarshal([]byte(raw), &fence); err != nil { return fence, false } - return fence, true + if fence.PodUID == "" || fence.NodeUID == "" { + return fence, false + } + switch fence.Strategy.Type { + case appsv1.OnDeleteStatefulSetStrategyType, appsv1.RecreateStatefulSetStrategyType: + return fence, fence.Strategy.RollingUpdate == nil + case appsv1.RollingUpdateStatefulSetStrategyType: + rolling := fence.Strategy.RollingUpdate + if rolling == nil { + return fence, true + } + if rolling.Partition != nil && *rolling.Partition < 0 { + return fence, false + } + if rolling.MaxUnavailable == nil { + return fence, true + } + // Check the finite saved-strategy fields before restoring them: an + // API-invalid payload must not reject an otherwise required scale-down. + value := rolling.MaxUnavailable + switch value.Type { + case intstr.Int: + return fence, value.IntVal > 0 + case intstr.String: + if len(validation.IsValidPercent(value.StrVal)) != 0 { + return fence, false + } + percent, err := strconv.Atoi(strings.TrimSuffix(value.StrVal, "%")) + return fence, err == nil && percent > 0 && percent <= 100 + default: + return fence, false + } + default: + return fence, false + } } -func clearHostportStopFence(ss *appsv1.StatefulSet) error { +func clearHostportStopFence(ss *appsv1.StatefulSet) { raw := ss.Annotations[hostportStopFenceAnnotation] if raw == "" { - return nil + return } - var fence hostportStopFence - if err := json.Unmarshal([]byte(raw), &fence); err != nil { - return fmt.Errorf("decode Hostport stop fence: %w", err) + if fence, valid := decodeHostportStopFence(raw); valid { + ss.Spec.UpdateStrategy = fence.Strategy } - ss.Spec.UpdateStrategy = fence.Strategy + // Unknown saved state must not replace the live strategy or obstruct + // required scale-down. Keep that strategy and retire only our own fence. ss.Spec.Template.Spec.SchedulingGates = slices.DeleteFunc(ss.Spec.Template.Spec.SchedulingGates, func(gate corev1.PodSchedulingGate) bool { return gate.Name == hostportStopSchedulingGate }) delete(ss.Annotations, hostportStopFenceAnnotation) - return nil } diff --git a/operator/internal/controller/gameserver_hostport_stop_test.go b/operator/internal/controller/gameserver_hostport_stop_test.go index 4b08ba69e..bd4d68aeb 100644 --- a/operator/internal/controller/gameserver_hostport_stop_test.go +++ b/operator/internal/controller/gameserver_hostport_stop_test.go @@ -1,6 +1,8 @@ package controller import ( + "encoding/json" + "fmt" "reflect" "slices" "testing" @@ -9,6 +11,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -317,3 +320,81 @@ func TestHostportMalformedStopFenceCannotGrantGrace(t *testing.T) { }) } } + +func TestHostportFenceCleanupRecoversInvalidMetadataWithoutGuessingStrategy(t *testing.T) { + partition, unavailable := int32(3), intstr.FromInt32(1) + saved := appsv1.StatefulSetUpdateStrategy{Type: appsv1.RollingUpdateStatefulSetStrategyType, RollingUpdate: &appsv1.RollingUpdateStatefulSetStrategy{Partition: &partition, MaxUnavailable: &unavailable}} + valid, err := json.Marshal(hostportStopFence{PodUID: "original-pod", NodeUID: "original-node", Strategy: saved}) + if err != nil { + t.Fatal(err) + } + recreate := appsv1.StatefulSetUpdateStrategy{Type: appsv1.RecreateStatefulSetStrategyType} + validRecreate, err := json.Marshal(hostportStopFence{PodUID: "original-pod", NodeUID: "original-node", Strategy: recreate}) + if err != nil { + t.Fatal(err) + } + percentage := saved.DeepCopy() + percent := intstr.FromString("25%") + percentage.RollingUpdate.MaxUnavailable = &percent + validPercentage, err := json.Marshal(hostportStopFence{PodUID: "original-pod", NodeUID: "original-node", Strategy: *percentage}) + if err != nil { + t.Fatal(err) + } + for _, test := range []struct { + name, raw string + restore *appsv1.StatefulSetUpdateStrategy + }{ + {"malformed", "malformed fence", nil}, + {"empty-object", "{}", nil}, + {"null", "null", nil}, + {"missing-identities", `{"strategy":{"type":"RollingUpdate"}}`, nil}, + {"empty-strategy", `{"podUID":"original-pod","nodeUID":"original-node","strategy":{}}`, nil}, + {"unknown-strategy", `{"podUID":"original-pod","nodeUID":"original-node","strategy":{"type":"unknown"}}`, nil}, + {"on-delete-with-rolling-fields", `{"podUID":"original-pod","nodeUID":"original-node","strategy":{"type":"OnDelete","rollingUpdate":{}}}`, nil}, + {"negative-partition", `{"podUID":"original-pod","nodeUID":"original-node","strategy":{"type":"RollingUpdate","rollingUpdate":{"partition":-1}}}`, nil}, + {"zero-max-unavailable", `{"podUID":"original-pod","nodeUID":"original-node","strategy":{"type":"RollingUpdate","rollingUpdate":{"maxUnavailable":0}}}`, nil}, + {"invalid-max-unavailable", `{"podUID":"original-pod","nodeUID":"original-node","strategy":{"type":"RollingUpdate","rollingUpdate":{"maxUnavailable":"invalid"}}}`, nil}, + {"excessive-max-unavailable-percent", `{"podUID":"original-pod","nodeUID":"original-node","strategy":{"type":"RollingUpdate","rollingUpdate":{"maxUnavailable":"101%"}}}`, nil}, + {"valid-custom-strategy", string(valid), &saved}, + {"valid-percentage-strategy", string(validPercentage), percentage}, + {"valid-recreate-strategy", string(validRecreate), &recreate}, + } { + for _, hostport := range []bool{false, true} { + for _, replicas := range []int32{0, 1} { + t.Run(fmt.Sprintf("%s/hostport=%t/replicas=%d", test.name, hostport, replicas), func(t *testing.T) { + r, gs, tmpl, _ := hostportStopFixture(t) + if !hostport { + gs.Spec.Networking.Expose = "NodePort" + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + } + r.APIReader = fake.NewClientBuilder().WithScheme(r.Scheme).WithObjects(gs, hostportIdentityNode("original-node", corev1.ConditionTrue)).Build() + ss := hostportStopSet(t, r, gs) + ss.Annotations = map[string]string{hostportStopFenceAnnotation: test.raw, "example.invalid/retained": "value"} + current := appsv1.StatefulSetUpdateStrategy{Type: appsv1.OnDeleteStatefulSetStrategyType} + ss.Spec.UpdateStrategy = current + otherGate := corev1.PodSchedulingGate{Name: "example.invalid/retained"} + ss.Spec.Template.Spec.SchedulingGates = []corev1.PodSchedulingGate{otherGate, {Name: hostportStopSchedulingGate}} + if err := r.Update(t.Context(), ss); err != nil { + t.Fatal(err) + } + if err := r.reconcileStatefulSet(t.Context(), gs, tmpl, nil, &materializedConfig{}, replicas); err != nil { + t.Fatal(err) + } + ss = hostportStopSet(t, r, gs) + want := current + if test.restore != nil { + want = *test.restore + } + if !reflect.DeepEqual(ss.Spec.UpdateStrategy, want) || ss.Spec.Replicas == nil || *ss.Spec.Replicas != replicas { + t.Fatalf("cleanup blocked power intent or guessed strategy: %+v", ss.Spec) + } + if !reflect.DeepEqual(ss.Spec.Template.Spec.SchedulingGates, []corev1.PodSchedulingGate{otherGate}) || ss.Annotations[hostportStopFenceAnnotation] != "" || ss.Annotations["example.invalid/retained"] != "value" { + t.Fatal("cleanup changed unrelated state or retained its fence") + } + }) + } + } + } +} diff --git a/operator/specs.md b/operator/specs.md index 48ddf789a..4abdf315d 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -69,7 +69,11 @@ be observed. The running Pod is not rolled. The save deadline/readiness check th scales to zero; a Backup or inherited copy recovery retains its cleanup precedence before any save/stop RPC. Pending, foreign or replacement Pods receive no save window. The temporary gate and previous update strategy are restored during valid endpoint -reconciliation or the documented explicit migration. The live Node +reconciliation or the documented explicit migration. Cleanup validates saved +identities and strategy fields, preserving valid custom RollingUpdate and Recreate +strategies. Malformed or incomplete records retain the current update strategy +while removing only the operator's stop gate/annotation; they cannot block a +requested scale-down. The live Node identity is checked again immediately before workload mutation, including backup replica preservation and restore startup; existing container and scheduling settings are otherwise retained. The sentinel Deployment also rechecks the live GameServer endpoint From a8aa9d88001ec2c8026780c6089d135c04509f86 Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 6 Nov 2026 19:53:11 -0500 Subject: [PATCH 20/20] fix(hostport): retire orphaned stop scheduling gates Signed-off-by: Baylem --- .../internal/controller/gameserver_hostport_stop.go | 3 --- .../controller/gameserver_hostport_stop_test.go | 10 ++++++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/operator/internal/controller/gameserver_hostport_stop.go b/operator/internal/controller/gameserver_hostport_stop.go index 9dadd8b12..8951ee838 100644 --- a/operator/internal/controller/gameserver_hostport_stop.go +++ b/operator/internal/controller/gameserver_hostport_stop.go @@ -156,9 +156,6 @@ func decodeHostportStopFence(raw string) (hostportStopFence, bool) { func clearHostportStopFence(ss *appsv1.StatefulSet) { raw := ss.Annotations[hostportStopFenceAnnotation] - if raw == "" { - return - } if fence, valid := decodeHostportStopFence(raw); valid { ss.Spec.UpdateStrategy = fence.Strategy } diff --git a/operator/internal/controller/gameserver_hostport_stop_test.go b/operator/internal/controller/gameserver_hostport_stop_test.go index bd4d68aeb..620261723 100644 --- a/operator/internal/controller/gameserver_hostport_stop_test.go +++ b/operator/internal/controller/gameserver_hostport_stop_test.go @@ -344,6 +344,8 @@ func TestHostportFenceCleanupRecoversInvalidMetadataWithoutGuessingStrategy(t *t name, raw string restore *appsv1.StatefulSetUpdateStrategy }{ + {"empty-annotation", "", nil}, + {"missing-annotation", "", nil}, {"malformed", "malformed fence", nil}, {"empty-object", "{}", nil}, {"null", "null", nil}, @@ -371,7 +373,10 @@ func TestHostportFenceCleanupRecoversInvalidMetadataWithoutGuessingStrategy(t *t } r.APIReader = fake.NewClientBuilder().WithScheme(r.Scheme).WithObjects(gs, hostportIdentityNode("original-node", corev1.ConditionTrue)).Build() ss := hostportStopSet(t, r, gs) - ss.Annotations = map[string]string{hostportStopFenceAnnotation: test.raw, "example.invalid/retained": "value"} + ss.Annotations = map[string]string{"example.invalid/retained": "value"} + if test.name != "missing-annotation" { + ss.Annotations[hostportStopFenceAnnotation] = test.raw + } current := appsv1.StatefulSetUpdateStrategy{Type: appsv1.OnDeleteStatefulSetStrategyType} ss.Spec.UpdateStrategy = current otherGate := corev1.PodSchedulingGate{Name: "example.invalid/retained"} @@ -390,7 +395,8 @@ func TestHostportFenceCleanupRecoversInvalidMetadataWithoutGuessingStrategy(t *t if !reflect.DeepEqual(ss.Spec.UpdateStrategy, want) || ss.Spec.Replicas == nil || *ss.Spec.Replicas != replicas { t.Fatalf("cleanup blocked power intent or guessed strategy: %+v", ss.Spec) } - if !reflect.DeepEqual(ss.Spec.Template.Spec.SchedulingGates, []corev1.PodSchedulingGate{otherGate}) || ss.Annotations[hostportStopFenceAnnotation] != "" || ss.Annotations["example.invalid/retained"] != "value" { + _, fenceRemains := ss.Annotations[hostportStopFenceAnnotation] + if !reflect.DeepEqual(ss.Spec.Template.Spec.SchedulingGates, []corev1.PodSchedulingGate{otherGate}) || fenceRemains || ss.Annotations["example.invalid/retained"] != "value" { t.Fatal("cleanup changed unrelated state or retained its fence") } })