test: exercise panel database and encryption-key recovery

.github/actions/install-trivy/action.yaml

@@ -0,0 +1,27 @@
+name: Install pinned Trivy
+description: Install the checksum-verified Linux amd64 scanner from its GitHub release.
+runs:
+  using: composite
+  steps:
+    - name: Download and verify Trivy
+      shell: bash
+      run: |
+        set -euo pipefail
+        if [[ "$(uname -s)" != Linux || "$(uname -m)" != x86_64 ]]; then
+          echo "This installer requires a Linux amd64 runner" >&2
+          exit 1
+        fi
+        version=0.74.0
+        sha256=2ae6fe3ee734b7fdf11335663e18c75ea12dccc76062f09f164a3b0f8be4371a
+        archive="trivy_${version}_Linux-64bit.tar.gz"
+        download_dir=$(mktemp -d "${RUNNER_TEMP:?}/trivy-download.XXXXXX")
+        trap 'rm -rf -- "$download_dir"' EXIT
+        # Bound DNS, connection and HTTP retries. Verify before extraction or execution.
+        curl --fail --show-error --location --retry 3 --retry-all-errors \
+          --retry-delay 2 --retry-max-time 180 --connect-timeout 15 --max-time 90 \
+          --output "$download_dir/$archive" \
+          "https://github.com/aquasecurity/trivy/releases/download/v${version}/${archive}"
+        printf '%s  %s\n' "$sha256" "$download_dir/$archive" | sha256sum --check --status
+        tar -xzf "$download_dir/$archive" -C "$download_dir" trivy
+        sudo install -m 0755 "$download_dir/trivy" /usr/local/bin/trivy
+        trivy --version

.github/workflows/ci.yaml

@@ -624,9 +624,7 @@ jobs:
           path: ~/.cache/trivy
           key: trivy-db-${{ steps.week.outputs.week }}
       - name: install trivy
-        uses: aquasecurity/setup-trivy@81e514348e19b6112ce2a7e3ecbafe19c1e1f567 # v0.3.1
-        with:
-          version: v0.74.0
+        uses: ./.github/actions/install-trivy
       - name: prove the gate rejects a vulnerable image
         run: ./hack/check-image-scan_test.sh
       - name: scan every e2e image
@@ -1823,8 +1821,9 @@ jobs:
         # t.Parallel(); `parallel` bounds in-flight tests per bucket
         # (operator does zero logins and is wait-dominated, so it runs
         # wide; the api buckets stay within the login limiter's budget).
-        # The arm64 job below uses the same steps and per-bucket settings.
-        bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, management-placement]
+        # The arm64 job below uses the same steps/settings except for the
+        # SQLite recovery drill, which runs once on amd64.
+        bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, management-placement, recovery]
         arch: [amd64]
         include:
           - bucket: operator
@@ -1864,6 +1863,10 @@ jobs:
             parallel: 1
             test_timeout: 35m # lifecycle plus two-GameServer TCP/UDP relay identity, outage and recovery checks
             job_timeout: 75 # Boots a kind cluster, installs the chart, runs the bucket's suite.
+          - bucket: recovery
+            parallel: 1
+            test_timeout: 20m # isolated panel backup/loss/restore; game workload stays running
+            job_timeout: 60
     env:
       CLUSTER: gameplane-e2e
       CTX: kind-gameplane-e2e
@@ -1896,12 +1899,22 @@ jobs:
         run: ./deploy/kind/e2e.sh up "$CLUSTER" e2e "$GAMEPLANE_E2E_PROFILE"
       - name: run go e2e (${{ matrix.bucket }})
         if: env.RUN_REGEX != ''
+        env:
+          GAMEPLANE_RECOVERY_REPORT_DIR: ${{ github.workspace }}/recovery-reports
         run: |
           cd test/e2e && \
             GAMEPLANE_E2E_REUSE_CLUSTER=1 \
             GAMEPLANE_E2E_CLUSTER="$CLUSTER" \
             go test -tags=e2e -timeout ${{ matrix.test_timeout }} -v \
               -parallel ${{ matrix.parallel }} -run "$RUN_REGEX" ./...
+      - name: upload measured panel recovery report
+        if: always() && matrix.bucket == 'recovery'
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: panel-recovery-${{ matrix.arch }}-${{ github.run_attempt }}
+          path: recovery-reports/*.json
+          if-no-files-found: warn
+          retention-days: 7
       - name: run go e2e (ratelimit tail)
         # LAST on purpose: this test deliberately drains the shared per-IP
         # and per-username login buckets; anything running after it queues
@@ -2706,7 +2719,7 @@ jobs:
               'go-e2e-unit': (n) => n === 'go (test/e2e)',
               'api-postgres': (n) => n === 'api (postgres)',
               'e2e-buckets': (n) => n === 'e2e bucket coverage',
-              'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|management-placement) \//.test(n) && n.includes('/ amd64 '),
+              'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|management-placement|recovery) \//.test(n) && n.includes('/ amd64 '),
               'e2e-go-arm64': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//.test(n) && n.includes('/ arm64 '),
               'e2e-multicluster': (n) => n.startsWith('e2e multicluster /') && n.includes('/ amd64 '),
               'e2e-multicluster-arm64': (n) => n.startsWith('e2e multicluster /') && n.includes('/ arm64 '),
@@ -2859,7 +2872,7 @@ jobs:
             const bucketSet = new Set();
             for (const j of apiJobs) {
               const n = j.name || '';
-              const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|management-placement) \//);
+              const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|management-placement|recovery) \//);
               if (m) bucketSet.add(m[1]);
               if (/^e2e multicluster \//.test(n)) bucketSet.add('multicluster');
               if (/^e2e upgrade \//.test(n)) bucketSet.add('upgrade');

.github/workflows/publish-edge.yaml

@@ -43,6 +43,7 @@ on:
       - "**/Dockerfile"
       - "**/Dockerfile.*"
       - ".github/workflows/publish-edge.yaml"
+      - ".github/actions/install-trivy/**"
       - "hack/ci_scope.py"
       - ".dockerignore"
       - "signing/cosign.pub"
@@ -149,9 +150,7 @@ jobs:
       # Spec 013 G2: never sign an image that has a fixed HIGH/CRITICAL
       # vulnerability. Same flags as the `vuln (images)` CI gate.
       - name: install trivy
-        uses: aquasecurity/setup-trivy@81e514348e19b6112ce2a7e3ecbafe19c1e1f567 # v0.3.1
-        with:
-          version: v0.74.0
+        uses: ./.github/actions/install-trivy
       - name: scan pushed image
         env:
           IMAGE: ${{ steps.image.outputs.name }}

.github/workflows/release.yaml

@@ -92,9 +92,7 @@ jobs:
       # Spec 013 G2: never sign an image that has a fixed HIGH/CRITICAL
       # vulnerability. Same flags as the `vuln (images)` CI gate.
       - name: install trivy
-        uses: aquasecurity/setup-trivy@81e514348e19b6112ce2a7e3ecbafe19c1e1f567 # v0.3.1
-        with:
-          version: v0.74.0
+        uses: ./.github/actions/install-trivy
       - name: scan pushed image
         env:
           IMAGE: ${{ steps.image.outputs.name }}

agent/internal/rcon/nuclearoption.go

@@ -112,14 +112,21 @@ func (c *NuclearOption) Close() error {
 // If the response has no body (body length = 0), an empty string is returned.
 // Non-2000 status codes produce an error that includes the status code and,
 // if present, the error detail from the response body.
+// One deadline bounds the complete exchange, including dialing and request writes.
 func (c *NuclearOption) Exec(cmd string) (string, error) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
+	d := c.execDeadline
+	if d <= 0 {
+		d = defaultNuclearOptionExecDeadline
+	}
+	exchangeDeadline := time.Now().Add(d)
+
 	// Ensure connection is open.
 	if c.conn == nil {
 		d := net.Dialer{Timeout: c.dialTimeout}
-		ctx, cancel := context.WithTimeout(context.Background(), c.dialTimeout)
+		ctx, cancel := context.WithDeadline(context.Background(), exchangeDeadline)
 		defer cancel()
 		conn, err := d.DialContext(ctx, "tcp", net.JoinHostPort(c.host, fmt.Sprint(c.port)))
 		if err != nil {
@@ -144,6 +151,14 @@ func (c *NuclearOption) Exec(cmd string) (string, error) {
 		return "", fmt.Errorf("nuclearoption rcon: marshal request: %w", err)
 	}
 
+	// Bound writes as well as reads. Arm this before sending anything: a
+	// server that stops reading must not retain the shared client forever.
+	if err := c.conn.SetDeadline(exchangeDeadline); err != nil {
+		_ = c.conn.Close()
+		c.conn = nil
+		return "", fmt.Errorf("nuclearoption rcon: set deadline: %w", err)
+	}
+
 	// Write request: 4-byte LE length + body.
 	reqLen := len(reqBody)
 	if reqLen < 0 || reqLen > math.MaxUint32 {
@@ -162,17 +177,6 @@ func (c *NuclearOption) Exec(cmd string) (string, error) {
 		return "", fmt.Errorf("nuclearoption rcon: write body: %w", err)
 	}
 
-	// Set deadline for the entire response read.
-	d := c.execDeadline
-	if d <= 0 {
-		d = defaultNuclearOptionExecDeadline
-	}
-	if err := c.conn.SetDeadline(time.Now().Add(d)); err != nil {
-		_ = c.conn.Close()
-		c.conn = nil
-		return "", fmt.Errorf("nuclearoption rcon: set deadline: %w", err)
-	}
-
 	// Read response: 4-byte status + 4-byte body length + body.
 	statusBuf := make([]byte, 4)
 	if _, err := io.ReadFull(c.conn, statusBuf); err != nil {

agent/internal/rcon/nuclearoption_deadline_test.go

@@ -0,0 +1,46 @@
+package rcon
+
+import (
+	"io"
+	"net"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestNuclearOptionDeadlineIncludesRequestWrites(t *testing.T) {
+	t.Parallel()
+	for _, readHeader := range []bool{false, true} {
+		stage := "write length"
+		if readHeader {
+			stage = "write body"
+		}
+		t.Run(stage, func(t *testing.T) {
+			t.Parallel()
+			local, remote := net.Pipe()
+			defer local.Close()
+			defer remote.Close()
+			client := NewNuclearOption("ignored", 1, nil)
+			client.conn = local
+			client.execDeadline = 100 * time.Millisecond
+			if readHeader {
+				go func() { _, _ = io.CopyN(io.Discard, remote, 4) }()
+			}
+			done := make(chan error, 1)
+			go func() { _, err := client.Exec("get-server-id"); done <- err }()
+			select {
+			case err := <-done:
+				if err == nil || !isTimeout(err) || !strings.Contains(err.Error(), stage) {
+					t.Fatalf("stalled %s must time out: %v", stage, err)
+				}
+			case <-time.After(2 * time.Second):
+				_ = remote.Close()
+				<-done
+				t.Fatalf("stalled %s exceeded the command budget", stage)
+			}
+			if client.conn != nil {
+				t.Fatal("timed-out partial request must discard the connection")
+			}
+		})
+	}
+}

agent/internal/rcon/nuclearoption_test.go

@@ -740,7 +740,7 @@ func TestNuclearOptionWriteBodyError(t *testing.T) {
 }
 
 // TestNuclearOptionSetDeadlineError verifies that a failure setting the
-// response read deadline (after the request was fully written) is reported
+// exchange deadline (before sending the request) is reported
 // and the connection is reset.
 func TestNuclearOptionSetDeadlineError(t *testing.T) {
 	local, remote := net.Pipe()

agent/specs.md

@@ -244,7 +244,7 @@ The agent and operator modules use `k8s.io/apimachinery`/`k8s.io/client-go` v0.3
   - `battleye`: UDP-based CRC32 packet framing.
   - `satisfactory`: HTTPS JSON function-call API with session token.
   - `palworld`: HTTP Basic REST admin API.
-  - `nuclearoption`: Dedicated TCP stream for Nuclear Option remote command protocol.
+  - `nuclearoption`: Dedicated TCP stream for Nuclear Option remote command protocol. One 30-second deadline bounds dialing, request writes, and response reads; responses are capped at 10 MiB. A timed-out partial exchange closes the connection before the next command.
   - `rest`: Generic HTTP/JSON REST console client with lazy auth (`PassFn`), 1 MiB bounded body read, configurable timeouts, auth failure cooldown, and loopback TLS protection. Provides adapters for FiveM (txAdmin: `POST /fxserver/commands`, bearer/X-TxAdmin-Token, `{"action":"console","parameter":"<cmd>"}`), Farming Simulator 25 (`POST /api/console`, Basic auth, `{"command":"<cmd>"}`), and generic REST (`POST /api/command`).
   - `cli`: Container stdin/PTY / local execution client (Option A in OPEN-DECISIONS.md). Drives commands via named FIFO pipe (`GAMEPLANE_CLI_PIPE`) or local process shell execution without requiring remote TCP networking or password secrets, while web console continues to attach via pod-attach PTY.