feat(operator): stage immutable template revisions per server

.github/workflows/ci.yaml

@@ -1259,6 +1259,8 @@ jobs:
         run: python3 charts/tests/check_gateway.py
       - name: standalone panel isolation and persistence chart renders
         run: python3 charts/tests/check_standalone.py
+      - name: workload security chart contract
+        run: python3 charts/tests/check_workload_security.py
       - name: auxiliary resources and management placement chart renders
         run: python3 charts/tests/check_workloads.py
       - name: operator Secret namespace permissions and cache configuration
@@ -1824,7 +1826,7 @@ jobs:
         # (operator does zero logins and is wait-dominated, so it runs
         # wide; the api buckets stay within the login limiter's budget).
         # The arm64 job below uses the same steps and per-bucket settings.
-        bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, management-placement]
+        bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, restricted-workloads, management-placement]
         arch: [amd64]
         include:
           - bucket: operator
@@ -1864,6 +1866,10 @@ jobs:
             parallel: 1
             test_timeout: 20m # one lifecycle test: helm upgrade, then mcp, syslog, frp tunnel and core checks in order
             job_timeout: 60 # Boots a kind cluster, installs the chart, runs the bucket's suite.
+          - bucket: restricted-workloads
+            parallel: 1
+            test_timeout: 25m # Includes real projected-token rotation and node capture cleanup.
+            job_timeout: 60
     env:
       CLUSTER: gameplane-e2e
       CTX: kind-gameplane-e2e
@@ -1937,7 +1943,7 @@ jobs:
         # (operator does zero logins and is wait-dominated, so it runs
         # wide; the api buckets stay within the login limiter's budget).
         # Mirrors e2e-go's steps and per-bucket settings.
-        bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components]
+        bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, restricted-workloads]
         arch: [arm64]
         include:
           - bucket: operator
@@ -1973,6 +1979,10 @@ jobs:
             parallel: 1
             test_timeout: 20m # one lifecycle test: helm upgrade, then mcp, syslog, frp tunnel and core checks in order
             job_timeout: 60 # Boots a kind cluster, installs the chart, runs the bucket's suite.
+          - bucket: restricted-workloads
+            parallel: 1
+            test_timeout: 25m # Includes real projected-token rotation and node capture cleanup.
+            job_timeout: 60
     env:
       CLUSTER: gameplane-e2e
       CTX: kind-gameplane-e2e
@@ -2697,8 +2707,8 @@ jobs:
               'go-e2e-unit': (n) => n === 'go (test/e2e)',
               'api-postgres': (n) => n === 'api (postgres)',
               'e2e-buckets': (n) => n === 'e2e bucket coverage',
-              'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|management-placement) \//.test(n) && n.includes('/ amd64 '),
-              'e2e-go-arm64': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//.test(n) && n.includes('/ arm64 '),
+              'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|restricted-workloads|management-placement) \//.test(n) && n.includes('/ amd64 '),
+              'e2e-go-arm64': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|restricted-workloads) \//.test(n) && n.includes('/ arm64 '),
               'e2e-multicluster': (n) => n.startsWith('e2e multicluster /') && n.includes('/ amd64 '),
               'e2e-multicluster-arm64': (n) => n.startsWith('e2e multicluster /') && n.includes('/ arm64 '),
               'e2e-upgrade': (n) => n.startsWith('e2e upgrade /') && n.includes('/ amd64 '),
@@ -2850,7 +2860,7 @@ jobs:
             const bucketSet = new Set();
             for (const j of apiJobs) {
               const n = j.name || '';
-              const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|management-placement) \//);
+              const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|restricted-workloads|management-placement) \//);
               if (m) bucketSet.add(m[1]);
               if (/^e2e multicluster \//.test(n)) bucketSet.add('multicluster');
               if (/^e2e upgrade \//.test(n)) bucketSet.add('upgrade');

agent/cmd/main.go

@@ -209,6 +209,13 @@ func main() {
 		protected.Use(authCheck.Middleware, target.Guard(serverUID))
 		mountProtected(protected)
 	})
+	// Backups bind save-off/save-on to the process they prepared. The service
+	// may already route to a replacement pod, which must reject the old UID.
+	r.Route("/v1/pods/{uid}", func(protected chi.Router) {
+		protected.Use(authCheck.Middleware, target.Guard(envOr("GAMEPLANE_POD_UID", "")), middleware.Timeout(30*time.Second))
+		protected.Get("/identity", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) })
+		quiesce.Mount(protected, rconClient, gameName, quiesceSpec)
+	})
 
 	srv := &http.Server{
 		Addr:              addr,

agent/internal/quiesce/quiesce_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/go-chi/chi/v5"
 
 	"github.com/GameplanePanel/gameplane/agent/internal/caps"
+	"github.com/GameplanePanel/gameplane/agent/internal/target"
 )
 
 type fakeRcon struct {
@@ -86,6 +87,33 @@ func newTestRouter(rc Rcon, spec *caps.Quiesce) *chi.Mux {
 	return r
 }
 
+func TestPodBoundRoutesRejectReplacementBeforeRCON(t *testing.T) {
+	for _, actual := range []string{"original", "replacement", ""} {
+		t.Run("pod="+actual, func(t *testing.T) {
+			rc := &fakeRcon{}
+			r := chi.NewRouter()
+			r.Route("/v1/pods/{uid}", func(bound chi.Router) {
+				bound.Use(target.Guard(actual))
+				Mount(bound, rc, "test", minecraftQuiesceSpec())
+			})
+			for _, path := range []string{"quiesce", "unquiesce"} {
+				response := httptest.NewRecorder()
+				r.ServeHTTP(response, httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/v1/pods/original/"+path, nil))
+				if actual == "original" {
+					if response.Code != http.StatusOK {
+						t.Fatalf("matching process rejected: %d", response.Code)
+					}
+				} else if response.Code != http.StatusNotFound {
+					t.Fatalf("wrong process admitted: %d", response.Code)
+				}
+			}
+			if (len(rc.calls()) > 0) != (actual == "original") {
+				t.Fatalf("commands reached wrong process: %v", rc.calls())
+			}
+		})
+	}
+}
+
 func decodeResponse(t *testing.T, body []byte) response {
 	t.Helper()
 	var got response

agent/internal/target/target.go

@@ -1,4 +1,4 @@
-// Package target binds versioned agent routes to an immutable GameServer UID.
+// Package target binds versioned agent routes to an immutable resource UID.
 package target
 
 import (
@@ -7,12 +7,13 @@ import (
 	"github.com/go-chi/chi/v5"
 )
 
-// Guard rejects a route for any server other than the one that started this
-// agent. Legacy routes stay available for local callers during rolling upgrades.
-func Guard(serverUID string) func(http.Handler) http.Handler {
+// Guard rejects a route for any resource other than the one that started this
+// agent (GameServer or Pod, according to the route). Legacy routes stay available
+// for local callers during rolling upgrades.
+func Guard(resourceUID string) func(http.Handler) http.Handler {
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-			if serverUID == "" || chi.URLParam(req, "uid") != serverUID {
+			if resourceUID == "" || chi.URLParam(req, "uid") != resourceUID {
 				http.NotFound(w, req)
 				return
 			}

agent/specs.md

@@ -5,6 +5,16 @@
 
 ## Purpose
 
+### Kubernetes token projection (#714)
+
+Game Pods disable automatic credentials. The agent alone mounts a rotating
+projected token plus kube-root CA and namespace at client-go's standard path;
+`rest.InClusterConfig` preserves `BearerTokenFile` refresh. No token is mounted
+into game/config-init. Shared PID remains intentional for process metrics and
+is residual in-Pod trust; mount isolation does not make the agent safe from
+all processes in its Pod. CI exercises live projection rotation and a heartbeat
+after the original short token expires.
+
 The agent is a per-pod HTTP/HTTPS sidecar that runs inside every game pod to expose the game server's operations to the control plane and dashboard. It translates dashboard requests into game-protocol actions (RCON, file I/O, container logs, player queries) and reports liveness, resource usage, and player metrics back to the operator via Kubernetes API patches, decoupling game-specific behavior from the cluster control plane.
 
 ## Responsibilities
@@ -177,6 +187,8 @@ Resource usage env vars (set by the operator):
 | `/players/banned` | GET | Currently banned players; response: `[ { name, reason?, source? } ]`; 502 if RCON unavailable |
 | `/quiesce` | POST | Pause auto-saves before snapshot; response: `{ quiesced, reason? }` (boolean, reason optional); unsupported games return `quiesced: false` with a reason |
 | `/unquiesce` | POST | Resume auto-saves after snapshot; response: `{ quiesced, reason? }` |
+| `/v1/pods/{uid}/identity` | GET | Operator preflight: 204 only when `{uid}` matches `GAMEPLANE_POD_UID`; never mutates game state |
+| `/v1/pods/{uid}/quiesce`, `/v1/pods/{uid}/unquiesce` | POST | Pod-bound backup RPCs with the same response contract as the legacy routes; reject missing/mismatched Pod UID with 404 before RCON |
 | `/lifecycle/stop` | POST | Run stop sequence before scale-to-zero; RCON connection drop is the expected outcome |
 | `/actions/run` | POST | Execute module-declared operator action; request: `{ id, params }`; response: `{ ok, raw? }` |
 | `/status` | GET | Live game metrics (module-declared); response: bare JSON array `[ { id, displayName?, value, unit? } ]` |
@@ -209,6 +221,7 @@ All endpoints on the `--addr` control mux, except `/healthz`, return `401 Unauth
 - **Extracted mod files stay world-readable**: `mods.moduleFileMode` is `0o644`, not a tighter `0o600`, because the mods volume is shared with the game container, which runs as whatever uid its image needs — a different uid than the agent's own. At `0o600` the game process couldn't read its own mods. This is the rationale behind the `.golangci.yml` gosec G302 exclusion scoped to `agent/internal/mods/mods.go`.
 - **Resource usage is in-pod**: The `usage` package reads from `/proc` or cgroups; no external metrics pipeline required. Cgroup mode is a fallback for older clusters; proc mode (default in production) requires the operator to set `ShareProcessNamespace: true`.
 - **Module capabilities drive behavior**: Every game-specific handler (players, quiesce, lifecycle, status, actions) reads its config from `--capabilities` (JSON unmarshaled into `caps.Spec`). New games require no agent code change.
+- **Backup process identity**: The operator injects `GAMEPLANE_POD_UID` from `metadata.uid` through the DownwardAPI. Pod-bound backup routes require the existing mTLS authentication and check this UID at the final network hop, including when the Service resolves to a replacement pod. Missing UID fails closed. Deploy the updated agent image and allow pod rollout when upgrading the operator; older agents return 404 and the operator does not fall back to unbound routes. Legacy endpoints remain available to existing callers during rollout.
 - **RCON connection errors are graceful**: A lost RCON connection does not crash the agent. `console`, `players`, `quiesce`, and `lifecycle` handlers catch connection errors and return appropriate HTTP status (e.g., `502 Bad Gateway`).
 - **Source RCON sends complete frames**: Each AUTH or command frame is submitted in one socket write, with its existing byte-size limit, request ID, type, body, and two NUL terminators preserved. This avoids separate header-only writes rejected by vanilla Minecraft's RCON reader. A short write is an error and drops the connection without replaying the command; response grace and healthy connection reuse remain unchanged.
 - **Log streams are tail-only**: The `logs` package does not support random-access reads. It streams from the current end (live mode) or from file start (backlog mode); clients must handle partial output and reconnection.

api/internal/gateway/capture.go

@@ -52,6 +52,12 @@ func (h *handler) serveCapture(w http.ResponseWriter, req *http.Request, target
 	}
 	defer transport.CloseIdleConnections()
 	host := target.Name + "-agent." + target.Namespace + ".svc.cluster.local:9091"
+	if name, namespace, node, err := kube.NodeCaptureService(nc); err != nil {
+		http.NotFound(w, req)
+		return
+	} else if node {
+		host = name + "-agent." + namespace + ".svc.cluster.local:9091"
+	}
 	proxy := &httputil.ReverseProxy{
 		Transport: transport,
 		Rewrite: func(pr *httputil.ProxyRequest) {

api/internal/handlers/capture.go

@@ -72,6 +72,8 @@ func MountCapture(r chi.Router, reg *kube.Registry, auditor *audit.Auditor, cfg
 	}
 	r.Post("/servers/{name}:capture-enable", h.captureEnable)
 	r.Post("/servers/{name}:capture-disable", h.captureDisable)
+	r.Post("/servers/{name}:capture-node-enable", h.captureNodeEnable)
+	r.Post("/servers/{name}:capture-node-disable", h.captureNodeDisable)
 	r.Post("/servers/{name}:capture-start", h.captureStart)
 	r.Post("/servers/{name}:capture-stop", h.captureStop)
 	r.Get("/servers/{name}:captures", h.captureList)
@@ -957,7 +959,9 @@ func (h *captureHandler) captureDelete(w http.ResponseWriter, req *http.Request)
 		}
 		deleteErr = deleteCaptureWithUID(req.Context(), k, nc)
 	} else {
-		h.deleteSidecarCaptureFile(req.Context(), gs.Name, gs.Namespace, captureID)
+		if nc.Annotations[kube.NodeCaptureNamespaceAnnotation] == "" {
+			h.deleteSidecarCaptureFile(req.Context(), gs.Name, gs.Namespace, captureID)
+		}
 		deleteErr = k.DeleteNetworkCapture(req.Context(), ns, captureID)
 	}
 
@@ -1099,7 +1103,11 @@ func (h *captureHandler) captureDownload(w http.ResponseWriter, req *http.Reques
 	if isRemoteCluster(req) {
 		status = h.remoteCaptureDownload(w, req, gs, nc)
 	} else {
-		status = h.proxyFileDownload(w, req, host, captureID)
+		if _, _, node, _ := kube.NodeCaptureService(nc); node {
+			status = h.proxyNodeCaptureDownload(w, req, gs, nc)
+		} else {
+			status = h.proxyFileDownload(w, req, host, captureID)
+		}
 	}
 	if status < http.StatusOK || (status >= http.StatusMultipleChoices && status != http.StatusNotModified) {
 		// A disconnect must not also erase the corrective audit outcome.

api/internal/handlers/capture_node.go

@@ -0,0 +1,111 @@
+package handlers
+
+import (
+	"encoding/json"
+	"errors"
+	"net/http"
+	"net/url"
+
+	"github.com/GameplanePanel/gameplane/api/internal/auth"
+	"github.com/GameplanePanel/gameplane/api/internal/httperr"
+	"github.com/GameplanePanel/gameplane/api/internal/kube"
+	"github.com/GameplanePanel/gameplane/api/internal/scope"
+	"github.com/go-chi/chi/v5"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+func (h *captureHandler) captureNodeEnable(w http.ResponseWriter, r *http.Request) {
+	h.captureNodeOptIn(w, r, true)
+}
+func (h *captureHandler) captureNodeDisable(w http.ResponseWriter, r *http.Request) {
+	h.captureNodeOptIn(w, r, false)
+}
+
+func (h *captureHandler) captureNodeOptIn(w http.ResponseWriter, req *http.Request, enabled bool) {
+	k, ok := resolveCluster(w, req, h.reg)
+	if !ok {
+		return
+	}
+	ns, ok := resolveNS(w, req)
+	if !ok {
+		return
+	}
+	name := chi.URLParam(req, "name")
+	u := auth.UserFromContext(req.Context())
+	if u == nil || (u.Role != "admin" && !u.Can("*", true, scope.RequestedCluster(req), "*")) {
+		if h.auditWriteOrFail(w, req, req.Method, req.URL.Path, name, "node_diagnostics_admin_required", http.StatusForbidden) {
+			httperr.WriteCode(w, req, http.StatusForbidden, errors.New("node diagnostic exception requires administrator privileges"))
+		}
+		return
+	}
+	gs, err := k.GetServer(req.Context(), ns, name)
+	if err != nil {
+		httperr.Write(w, req, err)
+		return
+	}
+	if gs.GetDeletionTimestamp() != nil {
+		httperr.WriteCode(w, req, http.StatusConflict, errors.New("server is terminating"))
+		return
+	}
+	reason := "node_diagnostics_opt_out"
+	if enabled {
+		reason = "node_diagnostics_opt_in_host_privileges"
+	}
+	// Record intent synchronously before granting host capabilities. A failed
+	// audit write prevents the exception; Kubernetes status remains separate.
+	if !h.auditWriteOrFail(w, req, req.Method, req.URL.Path, name+":"+string(gs.GetUID()), reason, http.StatusOK) {
+		return
+	}
+	patch, err := json.Marshal(map[string]any{"metadata": map[string]any{"uid": string(gs.GetUID()), "resourceVersion": gs.GetResourceVersion()}, "spec": map[string]any{"capture": map[string]any{"nodeDiagnostic": enabled}}})
+	if err != nil {
+		httperr.Write(w, req, err)
+		return
+	}
+	_, err = k.Dynamic.Resource(kube.GVRs["servers"]).Namespace(ns).Patch(req.Context(), name, types.MergePatchType, patch, metav1.PatchOptions{})
+	if err != nil {
+		if h.auditWriteOrFail(w, req, req.Method, req.URL.Path, name, "node_diagnostics_update_failed", http.StatusInternalServerError) {
+			httperr.Write(w, req, err)
+		}
+		return
+	}
+	if !enabled {
+		if err := h.stopActiveCaptures(req.Context(), k, ns, name); err != nil {
+			httperr.Write(w, req, err)
+			return
+		}
+	}
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]bool{"nodeDiagnostic": enabled})
+}
+
+func (h *captureHandler) proxyNodeCaptureDownload(w http.ResponseWriter, req *http.Request, gs *kube.GameServer, nc *kube.NetworkCapture) int {
+	bound, err := captureTarget(scope.RequestedCluster(req), gs, nc)
+	if err != nil {
+		http.NotFound(w, req)
+		return http.StatusNotFound
+	}
+	name, ns, _, err := kube.NodeCaptureService(nc)
+	if err != nil {
+		httperr.Write(w, req, err)
+		return http.StatusInternalServerError
+	}
+	host, ok := newValidatedHost(name, ns)
+	if !ok || h.tlsClient == nil {
+		http.Error(w, "capture artifacts unavailable", http.StatusServiceUnavailable)
+		return http.StatusServiceUnavailable
+	}
+	u := url.URL{Scheme: "https", Host: string(host), Path: "/v1/targets/" + bound.UID + "/captures/" + bound.Capture + "/uids/" + bound.CaptureUID + "/file"}
+	up, err := http.NewRequestWithContext(req.Context(), http.MethodGet, u.String(), nil)
+	if err != nil {
+		httperr.Write(w, req, err)
+		return http.StatusInternalServerError
+	}
+	copyProxyHeaders(up.Header, req.Header)
+	resp, err := h.tlsClient.Do(up)
+	if err != nil {
+		return writeUpstreamError(w, req, err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+	return streamCaptureResponse(w, req, resp, nc.Name)
+}