feat: add bounded game performance and relay monitoring

.github/workflows/ci.yaml

@@ -1194,6 +1194,8 @@ jobs:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1
       - run: helm lint charts/gameplane
+      - name: evaluate monitoring rules
+        run: bash charts/gameplane/tests/monitoring-rules.sh
 
   standalone-compose:
     name: standalone Compose smoke
@@ -1257,10 +1259,10 @@ jobs:
         run: python3 charts/tests/check_gateway.py
       - name: standalone panel isolation and persistence chart renders
         run: python3 charts/tests/check_standalone.py
-      - name: strict network profile and sentinel policy selectors
-        run: python3 charts/tests/check_network_isolation.py
       - name: auxiliary resources and management placement chart renders
         run: python3 charts/tests/check_workloads.py
+      - name: strict network profile and sentinel policy selectors
+        run: python3 charts/tests/check_network_isolation.py
       - name: operator Secret namespace permissions and cache configuration
         run: python3 charts/tests/check_operator_secrets.py
       - name: sanity-check rendered manifests
@@ -1347,10 +1349,16 @@ jobs:
             | grep -q 'app.kubernetes.io/name: gameplane-telemetry-receiver'
           awk '/kind: Service/,/^---$/' /tmp/telemetry-receiver-scrape.yaml \
             | grep -q 'name: dashboard'
-          # scrapeNamespaceSelector unset (the chart default): neither
-          # exception renders, matching the pre-fix behavior.
+          # Missing scrape ingress must fail with actionable guidance.
+          if helm template gameplane charts/gameplane --set serviceMonitors.enabled=true > /tmp/monitoring-invalid.txt 2>&1; then
+            echo "::error::monitoring without a scrape selector unexpectedly rendered"
+            exit 1
+          fi
+          grep -q 'serviceMonitors.scrapeNamespaceSelector' /tmp/monitoring-invalid.txt
+          # Explicit external policy ownership renders no chart exception.
           helm template gameplane charts/gameplane \
             --set serviceMonitors.enabled=true \
+            --set serviceMonitors.externallyManagedPolicy=true \
             --show-only templates/networkpolicies.yaml > /tmp/netpol-default.yaml
           if grep -q 'allow-prometheus-to-agent' /tmp/netpol-default.yaml; then
             echo "::error::allow-prometheus-to-agent rendered without serviceMonitors.scrapeNamespaceSelector set"
@@ -1527,7 +1535,8 @@ jobs:
           # keeps working without api.telemetry.enabled.
           helm template gameplane charts/gameplane \
             --set api.telemetry.receiver.enabled=true \
-            --set serviceMonitors.enabled=true > /tmp/rendered-reuse-values-receiver.yaml
+            --set serviceMonitors.enabled=true \
+            --set serviceMonitors.externallyManagedPolicy=true > /tmp/rendered-reuse-values-receiver.yaml
           python3 - <<'PY'
           import yaml
           docs = [d for d in yaml.safe_load_all(open('/tmp/rendered-reuse-values-receiver.yaml')) if d]
@@ -1824,9 +1833,13 @@ jobs:
         # (operator does zero logins and is wait-dominated, so it runs
         # wide; the api buckets stay within the login limiter's budget).
         # The arm64 job below uses the same steps and per-bucket settings.
-        bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, network-isolation, management-placement]
+        bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, management-placement, network-isolation, monitoring]
         arch: [amd64]
         include:
+          - bucket: monitoring
+            parallel: 1
+            test_timeout: 20m
+            job_timeout: 75
           - bucket: network-isolation
             parallel: 1
             test_timeout: 15m
@@ -1896,9 +1909,12 @@ jobs:
       - name: bring up cluster + helm install
         if: env.RUN_REGEX != '' || env.TAIL_REGEX != ''
         env:
-          GAMEPLANE_E2E_CNI: ${{ matrix.bucket == 'network-isolation' && 'cilium' || 'kindnet' }}
           GAMEPLANE_E2E_PROFILE: ${{ matrix.bucket == 'management-placement' && 'management' || 'default' }}
+          GAMEPLANE_E2E_CNI: ${{ (matrix.bucket == 'network-isolation' || matrix.bucket == 'monitoring') && 'cilium' || 'kindnet' }}
         run: ./deploy/kind/e2e.sh up "$CLUSTER" e2e "$GAMEPLANE_E2E_PROFILE"
+      - name: install real monitoring fixture
+        if: matrix.bucket == 'monitoring' && env.RUN_REGEX != ''
+        run: bash deploy/kind/monitoring.sh
       - name: run go e2e (${{ matrix.bucket }})
         if: env.RUN_REGEX != ''
         run: |
@@ -1922,7 +1938,7 @@ jobs:
         uses: ./.github/actions/dump-cluster-state
         with:
           context: ${{ env.CTX }}
-          namespaces: "gameplane-system,gameplane-games"
+          namespaces: "${{ matrix.bucket == 'monitoring' && 'gameplane-system,gameplane-games,monitoring' || 'gameplane-system,gameplane-games' }}"
           include-previous-logs: "true"
 
   e2e-go-arm64:
@@ -2711,7 +2727,7 @@ jobs:
               'go-e2e-unit': (n) => n === 'go (test/e2e)',
               'api-postgres': (n) => n === 'api (postgres)',
               'e2e-buckets': (n) => n === 'e2e bucket coverage',
-              'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|network-isolation|management-placement) \//.test(n) && n.includes('/ amd64 '),
+              'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|management-placement|network-isolation|monitoring) \//.test(n) && n.includes('/ amd64 '),
               'e2e-go-arm64': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//.test(n) && n.includes('/ arm64 '),
               'e2e-multicluster': (n) => n.startsWith('e2e multicluster /') && n.includes('/ amd64 '),
               'e2e-multicluster-arm64': (n) => n.startsWith('e2e multicluster /') && n.includes('/ arm64 '),
@@ -2864,7 +2880,7 @@ jobs:
             const bucketSet = new Set();
             for (const j of apiJobs) {
               const n = j.name || '';
-              const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|network-isolation|management-placement) \//);
+              const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|management-placement|network-isolation|monitoring) \//);
               if (m) bucketSet.add(m[1]);
               if (/^e2e multicluster \//.test(n)) bucketSet.add('multicluster');
               if (/^e2e upgrade \//.test(n)) bucketSet.add('upgrade');

agent/cmd/main.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/chi/v5/middleware"
+	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/GameplanePanel/gameplane/agent/internal/actions"
 	"github.com/GameplanePanel/gameplane/agent/internal/auth"
@@ -178,6 +179,7 @@ func main() {
 	// and isn't meant to be trusted with the console/files/RCON routes that
 	// cert would also unlock.
 
+	statusSource := status.New(rconClient, statusSpec)
 	mountProtected := func(protected chi.Router) {
 
 		// Console/Logs open long-lived WebSockets and Mods installs can run
@@ -198,7 +200,7 @@ func main() {
 			quiesce.Mount(bounded, rconClient, gameName, quiesceSpec)
 			lifecycle.Mount(bounded, rconClient, gameName, lifecycleSpec)
 			actions.Mount(bounded, rconClient, gameName, actionSpecs)
-			status.Mount(bounded, rconClient, statusSpec)
+			statusSource.Mount(bounded)
 		})
 	}
 	r.Group(func(protected chi.Router) {
@@ -230,6 +232,8 @@ func main() {
 	// Prometheus metrics for per-server resource usage and player counts.
 	// Registered with the default registry so promhttp.Handler() picks it up.
 	m := metrics.New()
+	prometheus.MustRegister(statusSource)
+	go statusSource.Run(ctx)
 
 	// Resource usage: in proc mode (the operator shares the pod's PID
 	// namespace and sets GAMEPLANE_USAGE_PROC) the agent reports the game

agent/internal/caps/caps.go

@@ -83,11 +83,19 @@ type ModInstall struct {
 // StatusMetric reads one live readout from an RCON command's output via a
 // named-group regex (group "value").
 type StatusMetric struct {
-	ID          string `json:"id"`
-	DisplayName string `json:"displayName,omitempty"`
-	Command     string `json:"command"`
-	Regex       string `json:"regex"`
-	Unit        string `json:"unit,omitempty"`
+	ID          string             `json:"id"`
+	DisplayName string             `json:"displayName,omitempty"`
+	Command     string             `json:"command"`
+	Regex       string             `json:"regex"`
+	Unit        string             `json:"unit,omitempty"`
+	Performance *PerformanceMetric `json:"performance,omitempty"`
+}
+
+// PerformanceMetric mirrors the optional, explicitly numeric status contract.
+type PerformanceMetric struct {
+	Signal            string  `json:"signal"`
+	Unit              string  `json:"unit"`
+	DegradedThreshold *string `json:"degradedThreshold,omitempty"`
 }
 
 // PlayerActions maps moderation actions to console command templates

agent/internal/rcon/nuclearoption.go

@@ -112,14 +112,21 @@ func (c *NuclearOption) Close() error {
 // If the response has no body (body length = 0), an empty string is returned.
 // Non-2000 status codes produce an error that includes the status code and,
 // if present, the error detail from the response body.
+// One deadline bounds the complete exchange, including dialing and request writes.
 func (c *NuclearOption) Exec(cmd string) (string, error) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
+	d := c.execDeadline
+	if d <= 0 {
+		d = defaultNuclearOptionExecDeadline
+	}
+	exchangeDeadline := time.Now().Add(d)
+
 	// Ensure connection is open.
 	if c.conn == nil {
 		d := net.Dialer{Timeout: c.dialTimeout}
-		ctx, cancel := context.WithTimeout(context.Background(), c.dialTimeout)
+		ctx, cancel := context.WithDeadline(context.Background(), exchangeDeadline)
 		defer cancel()
 		conn, err := d.DialContext(ctx, "tcp", net.JoinHostPort(c.host, fmt.Sprint(c.port)))
 		if err != nil {
@@ -144,6 +151,14 @@ func (c *NuclearOption) Exec(cmd string) (string, error) {
 		return "", fmt.Errorf("nuclearoption rcon: marshal request: %w", err)
 	}
 
+	// Bound writes as well as reads. Arm this before sending anything: a
+	// server that stops reading must not retain the shared client forever.
+	if err := c.conn.SetDeadline(exchangeDeadline); err != nil {
+		_ = c.conn.Close()
+		c.conn = nil
+		return "", fmt.Errorf("nuclearoption rcon: set deadline: %w", err)
+	}
+
 	// Write request: 4-byte LE length + body.
 	reqLen := len(reqBody)
 	if reqLen < 0 || reqLen > math.MaxUint32 {
@@ -162,17 +177,6 @@ func (c *NuclearOption) Exec(cmd string) (string, error) {
 		return "", fmt.Errorf("nuclearoption rcon: write body: %w", err)
 	}
 
-	// Set deadline for the entire response read.
-	d := c.execDeadline
-	if d <= 0 {
-		d = defaultNuclearOptionExecDeadline
-	}
-	if err := c.conn.SetDeadline(time.Now().Add(d)); err != nil {
-		_ = c.conn.Close()
-		c.conn = nil
-		return "", fmt.Errorf("nuclearoption rcon: set deadline: %w", err)
-	}
-
 	// Read response: 4-byte status + 4-byte body length + body.
 	statusBuf := make([]byte, 4)
 	if _, err := io.ReadFull(c.conn, statusBuf); err != nil {

agent/internal/rcon/nuclearoption_deadline_test.go

@@ -0,0 +1,46 @@
+package rcon
+
+import (
+	"io"
+	"net"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestNuclearOptionDeadlineIncludesRequestWrites(t *testing.T) {
+	t.Parallel()
+	for _, readHeader := range []bool{false, true} {
+		stage := "write length"
+		if readHeader {
+			stage = "write body"
+		}
+		t.Run(stage, func(t *testing.T) {
+			t.Parallel()
+			local, remote := net.Pipe()
+			defer local.Close()
+			defer remote.Close()
+			client := NewNuclearOption("ignored", 1, nil)
+			client.conn = local
+			client.execDeadline = 100 * time.Millisecond
+			if readHeader {
+				go func() { _, _ = io.CopyN(io.Discard, remote, 4) }()
+			}
+			done := make(chan error, 1)
+			go func() { _, err := client.Exec("get-server-id"); done <- err }()
+			select {
+			case err := <-done:
+				if err == nil || !isTimeout(err) || !strings.Contains(err.Error(), stage) {
+					t.Fatalf("stalled %s must time out: %v", stage, err)
+				}
+			case <-time.After(2 * time.Second):
+				_ = remote.Close()
+				<-done
+				t.Fatalf("stalled %s exceeded the command budget", stage)
+			}
+			if client.conn != nil {
+				t.Fatal("timed-out partial request must discard the connection")
+			}
+		})
+	}
+}

agent/internal/rcon/nuclearoption_test.go

@@ -740,7 +740,7 @@ func TestNuclearOptionWriteBodyError(t *testing.T) {
 }
 
 // TestNuclearOptionSetDeadlineError verifies that a failure setting the
-// response read deadline (after the request was fully written) is reported
+// exchange deadline (before sending the request) is reported
 // and the connection is reset.
 func TestNuclearOptionSetDeadlineError(t *testing.T) {
 	local, remote := net.Pipe()

agent/internal/rcon/rcon.go

@@ -56,6 +56,11 @@ const (
 	// byte length (F-104).
 	minPacketSize = 10
 	maxPacketSize = 4*4096 + 10 // 16394
+
+	// sourceMaxResponseBytes permits large administrative listings while
+	// bounding the aggregate allocation across otherwise valid fragments.
+	// Status collection applies its separate, smaller 64-KiB acceptance cap.
+	sourceMaxResponseBytes = 8 * 1024 * 1024
 )
 
 // PassFn resolves the RCON password on demand (allowing rotation without restart).
@@ -86,6 +91,8 @@ type Client struct {
 
 	// execDeadline bounds a single command exchange (see Exec).
 	execDeadline time.Duration
+	// authDeadline separately bounds authentication, including all preamble packets.
+	authDeadline time.Duration
 
 	// authFailureCooldown bounds how long ensureLocked refuses to re-dial
 	// after an explicit AUTH_RESPONSE rejection (id=-1). A struct field
@@ -114,6 +121,8 @@ type Client struct {
 // (heartbeat, players, quiesce, console).
 const defaultExecDeadline = 30 * time.Second
 
+const defaultSourceAuthDeadline = 10 * time.Second
+
 // defaultAuthFailureCooldown bounds how long ensureLocked backs off after
 // an explicit auth rejection before it will dial again. Mirrors
 // websocket.go's defaultWebSocketAuthFailureCooldown — same value, same
@@ -133,6 +142,7 @@ func New(host string, port int, pw PassFn) *Client {
 		addr:                net.JoinHostPort(host, fmt.Sprint(port)),
 		password:            pw,
 		execDeadline:        defaultExecDeadline,
+		authDeadline:        defaultSourceAuthDeadline,
 		authFailureCooldown: defaultAuthFailureCooldown,
 	}
 }
@@ -154,7 +164,8 @@ func (c *Client) Exec(cmd string) (string, error) {
 	if d <= 0 {
 		d = defaultExecDeadline
 	}
-	_ = conn.SetDeadline(time.Now().Add(d))
+	exchangeDeadline := time.Now().Add(d)
+	_ = conn.SetDeadline(exchangeDeadline)
 	defer func() { _ = conn.SetDeadline(time.Time{}) }()
 
 	reqID := c.allocID()
@@ -172,12 +183,22 @@ func (c *Client) Exec(cmd string) (string, error) {
 	// reqID) and bound the wait for any trailing fragments with a short grace
 	// window: a single-packet reply returns as soon as the grace
 	// elapses or the server closes; a fragmented Source reply keeps extending
-	// the window as each fragment arrives.
+	// the window as each fragment arrives, never beyond exchangeDeadline.
 	var out bytes.Buffer
 	gotResponse := false
 	for {
+		if !time.Now().Before(exchangeDeadline) {
+			c.dropLocked()
+			return "", fmt.Errorf("rcon: command exchange deadline exceeded: %w", os.ErrDeadlineExceeded)
+		}
 		id, _, body, err := c.readPacket()
 		if err != nil {
+			if !time.Now().Before(exchangeDeadline) {
+				// Reaching the hard deadline is not evidence of a complete
+				// response. Discard its partial text and any queued fragments.
+				c.dropLocked()
+				return "", fmt.Errorf("rcon: command exchange deadline exceeded: %w", os.ErrDeadlineExceeded)
+			}
 			if gotResponse {
 				// We already hold the reply. A timeout means the grace window
 				// elapsed on a still-healthy socket — keep it for reuse. Any
@@ -196,9 +217,17 @@ func (c *Client) Exec(cmd string) (string, error) {
 		// Source/Minecraft echo the request id in every reply fragment; ignore
 		// anything else.
 		if id == reqID {
+			if len(body) > sourceMaxResponseBytes-out.Len() {
+				c.dropLocked()
+				return "", fmt.Errorf("rcon: response exceeded limit of %d bytes", sourceMaxResponseBytes)
+			}
 			out.WriteString(body)
 			gotResponse = true
-			_ = conn.SetReadDeadline(time.Now().Add(responseGrace))
+			readDeadline := time.Now().Add(responseGrace)
+			if readDeadline.After(exchangeDeadline) {
+				readDeadline = exchangeDeadline
+			}
+			_ = conn.SetReadDeadline(readDeadline)
 		}
 	}
 }
@@ -247,7 +276,12 @@ func (c *Client) ensureLocked() error {
 		return err
 	}
 	c.conn = conn
-	_ = conn.SetDeadline(time.Now().Add(10 * time.Second))
+	authBudget := c.authDeadline
+	if authBudget <= 0 {
+		authBudget = defaultSourceAuthDeadline
+	}
+	authDeadline := time.Now().Add(authBudget)
+	_ = conn.SetDeadline(authDeadline)
 
 	authID := c.allocID()
 	if err := c.writePacket(authID, typeAuth, pw); err != nil {
@@ -256,6 +290,10 @@ func (c *Client) ensureLocked() error {
 	}
 	// Server MAY send an empty RESPONSE_VALUE before AUTH_RESPONSE.
 	for {
+		if !time.Now().Before(authDeadline) {
+			c.dropLocked()
+			return fmt.Errorf("rcon: authentication deadline exceeded: %w", os.ErrDeadlineExceeded)
+		}
 		id, kind, _, err := c.readPacket()
 		if err != nil {
 			c.dropLocked()