From 007733bce9076cb7097acf9c5cbf4924e352c90c Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 21 Oct 2026 12:31:31 -0400 Subject: [PATCH 01/13] test: exercise isolated panel database and key recovery Signed-off-by: Baylem --- .github/workflows/ci.yaml | 23 +- docs/panel-recovery.md | 167 +++++++ docs/standalone-panel.md | 5 + test/e2e/buckets.sh | 10 +- test/e2e/panel_recovery_e2e_test.go | 526 ++++++++++++++++++++++ test/e2e/specs.md | 14 + test/e2e/testdata/recovery/maintenance.py | 116 +++++ 7 files changed, 856 insertions(+), 5 deletions(-) create mode 100644 docs/panel-recovery.md create mode 100644 test/e2e/panel_recovery_e2e_test.go create mode 100644 test/e2e/testdata/recovery/maintenance.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a1c9aecdd..e84f04cc1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1663,8 +1663,9 @@ jobs: # t.Parallel(); `parallel` bounds in-flight tests per bucket # (operator does zero logins and is wait-dominated, so it runs # wide; the api buckets stay within the login limiter's budget). - # The arm64 job below uses the same steps and per-bucket settings. - bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components] + # The arm64 job below uses the same steps/settings except for the + # SQLite recovery drill, which runs once on amd64. + bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, recovery] arch: [amd64] include: - bucket: operator @@ -1700,6 +1701,10 @@ jobs: parallel: 1 test_timeout: 20m # one lifecycle test: helm upgrade, then mcp, syslog, frp tunnel and core checks in order job_timeout: 60 # Boots a kind cluster, installs the chart, runs the bucket's suite. + - bucket: recovery + parallel: 1 + test_timeout: 20m # isolated panel backup/loss/restore; game workload stays running + job_timeout: 60 env: CLUSTER: gameplane-e2e CTX: kind-gameplane-e2e @@ -1720,6 +1725,8 @@ jobs: - name: bring up cluster + helm install run: ./deploy/kind/e2e.sh up "$CLUSTER" e2e - name: run go e2e (${{ matrix.bucket }}) + env: + GAMEPLANE_RECOVERY_REPORT_DIR: ${{ github.workspace }}/recovery-reports run: | RUN_REGEX="$(./test/e2e/buckets.sh regex '${{ matrix.bucket }}')" cd test/e2e && \ @@ -1727,6 +1734,14 @@ jobs: GAMEPLANE_E2E_CLUSTER="$CLUSTER" \ go test -tags=e2e -timeout ${{ matrix.test_timeout }} -v \ -parallel ${{ matrix.parallel }} -run "$RUN_REGEX" ./... + - name: upload measured panel recovery report + if: always() && matrix.bucket == 'recovery' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: panel-recovery-${{ matrix.arch }}-${{ github.run_attempt }} + path: recovery-reports/*.json + if-no-files-found: warn + retention-days: 7 - name: run go e2e (ratelimit tail) # LAST on purpose: this test deliberately drains the shared per-IP # and per-username login buckets; anything running after it queues @@ -2510,7 +2525,7 @@ jobs: 'go-e2e-unit': (n) => n === 'go (test/e2e)', 'api-postgres': (n) => n === 'api (postgres)', 'e2e-buckets': (n) => n === 'e2e bucket coverage', - 'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//.test(n) && n.includes('/ amd64 '), + 'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|recovery) \//.test(n) && n.includes('/ amd64 '), 'e2e-go-arm64': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//.test(n) && n.includes('/ arm64 '), 'e2e-multicluster': (n) => n.startsWith('e2e multicluster /') && n.includes('/ amd64 '), 'e2e-multicluster-arm64': (n) => n.startsWith('e2e multicluster /') && n.includes('/ arm64 '), @@ -2663,7 +2678,7 @@ jobs: const bucketSet = new Set(); for (const j of apiJobs) { const n = j.name || ''; - const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//); + const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|recovery) \//); if (m) bucketSet.add(m[1]); if (/^e2e multicluster \//.test(n)) bucketSet.add('multicluster'); if (/^e2e upgrade \//.test(n)) bucketSet.add('upgrade'); diff --git a/docs/panel-recovery.md b/docs/panel-recovery.md new file mode 100644 index 000000000..719c00af2 --- /dev/null +++ b/docs/panel-recovery.md @@ -0,0 +1,167 @@ +# Recovering the panel while games keep running + +Panel SQL state, Kubernetes desired state and game files are separate recovery +domains. A game-volume backup cannot reconstruct panel users, password hashes, +roles/grants, sessions, share links, configuration, audit history or standalone +cluster registrations and encrypted credentials. Recover the failed domain; +restoring the panel does not require deleting or recreating live games. + +This runbook extends [standalone storage and key recovery](standalone-panel.md#storage-and-recovery). +Standalone SQL storage was introduced in PR [#630](https://github.com/GameplanePanel/Gameplane/pull/630). +PR [#707](https://github.com/GameplanePanel/Gameplane/pull/707) added PostgreSQL-backed +E2E coverage; it does not provide PostgreSQL backup, PITR or failover automation. + +## Backup inventory and targets + +Record the deployment mode, software/schema version, database location and +backup completion time. Keep the following inventory outside the failed panel: + +| Domain | Retain and verify | +| --- | --- | +| SQL, both modes | Consistent SQLite database/WAL snapshot or external PostgreSQL backups and WAL/PITR; users, role bindings, configuration, sessions/shares/API tokens and audit chain. Standalone also stores management objects and encrypted credentials here. | +| Kubernetes | Cluster/etcd recovery procedure, CRDs, GameServer/Module/ModuleSource/Backup/Restore resources, namespaces/RBAC, management/config/agent/registry Secrets, certificates and external provider credentials. Combined mode stores management resources and remote credentials here, rather than in standalone SQL. | +| Workload data | Primary, extra and mod PVCs, PV identities, storage classes/topology, retention policies, Restic repositories/passwords, CSI snapshot classes/content and backend access. Include game-specific save/restore consistency requirements. | +| Key custody | The exact standalone 32-byte management key, separately protected from database archives, plus keys needed by retained older backups after rotation. Preserve relevant certificate/private-key and secret-manager recovery procedures. | +| Authorization evidence | Intended users, roles, per-cluster/namespace grants, owner/collaborator associations, credentials revoked since the recovery point and audit exports. Record Kubernetes UIDs as well as names. | + +Set targets for each deployment before a drill. Fill in the maximum acceptable +age of restored management state (**RPO**), maximum time from loss until verified +service (**RTO**), responsible operator, backup schedule and restoration location. +Check measured results against those targets and adjust the backup schedule, +storage, tooling or staffing when they are missed. Chart defaults are not a +universal RPO/RTO promise. Measure game-data recovery separately. + +For SQLite, stop every API process and use a SQLite backup tool or copy the +database **and its WAL state** consistently. Never copy only a live `.db` file. +Run `PRAGMA integrity_check` against the restored copy. The CI drill uses SQLite's +backup API with the panel stopped and converts the archive to DELETE journal mode +so it is self-contained. Keep the archive read-only and restore to a separate +working copy. Use the ownership/key permission rules in the standalone guide. + +For PostgreSQL, assign backup/PITR/failover to the database platform: schedule and +test base backups plus WAL retention, protect credentials and select an explicit +recovery timestamp/LSN. Restore into an isolated instance, validate schema and +data, and prevent split-brain writes before switching the API's DSN. Stop all API +writers for offline credential invalidation and keep one API process after +recovery. The SQLite CI drill does not certify a PostgreSQL service's PITR/RTO. + +## Database outage + +Stop or restrict panel traffic and identify whether the database is unreachable, +out of space, locked or corrupt. Preserve logs and the failed storage before +changing it. Restore database connectivity/capacity first when the original +database is intact; do not initialize an empty database or bootstrap new users +as a connectivity repair. Verify readiness, login, scoped access and registered +cluster operations before reopening access. + +Game Pods and the Kubernetes operator can continue independently of the panel. +Do not assume this proves game health: check Kubernetes events, game process +state and saves through the operational access available during the outage. + +## Database loss and recovery order + +1. Restrict external panel access and stop **all** API writers. Fence the old + database instance so it cannot resume writing. Record incident/recovery start + time and live GameServer/PVC/Pod UIDs, container IDs and restart counts. Keep + the operator and healthy game Pods running when only panel SQL was lost. +2. Preserve the failed database and audit evidence. Select a verified backup or + PostgreSQL recovery point, its software/schema version, and the matching + standalone key version. Copy the backup into a new working location; keep + the original immutable. Restore database file ownership/permissions and + verify integrity before starting the API. +3. With every API stopped, reconcile authorization on the **working copy**. + An old backup can resurrect users, grants and credentials revoked later. + Compare the external authorization/revocation inventory, disable/remove stale + accounts and grants, and replace compromised credentials. Review SSO/provider + access too. Preserve original audit rows and record recovery actions in a + separate incident log rather than rewriting the audit chain. +4. Invalidate restored bearer access according to the recovery policy. The drill + deletes all restored `sessions` and `api_tokens`, and sets `share_links.revoked_at` + for every active link. Do this offline in one transaction, on the working copy: + + ```sql + BEGIN; + DELETE FROM sessions; + DELETE FROM api_tokens; + UPDATE share_links SET revoked_at = '' + WHERE revoked_at IS NULL; + COMMIT; + ``` + + Replace the timestamp placeholder before execution. Retain the archive as + evidence, protect access to it, and tell users that they must log in again and + recreate shares/tokens. Do not run cleanup against the evidence archive. +5. Mount the original standalone management key read-only from separate custody. + In combined mode, restore/validate required Kubernetes management objects and + Secrets through the cluster's recovery process instead. Start one API using + compatible software and the restored database; inspect migration/startup + results before exposing it. A wrong/missing key is a hard failure, not a reason + to replace the key or discard encrypted rows. +6. Reconcile the panel's inventory against each target cluster. Check the actual + registered cluster endpoint/identity and credential-backed API operations. + Compare GameServer, PVC and Pod UIDs with the recorded inventory; validate + owner/collaborator user IDs against intended accounts. Names alone do not + establish identity or ownership. Resolve deleted/recreated resources, dangling + references and reused numeric user IDs before granting access. Do not recreate + live workloads merely to make stale panel records appear consistent. +7. Verify restored administrators and restricted users, role definitions, scoped + allow/deny behavior, old-cookie/share denial, cluster credentials and audit + history/integrity. Confirm game Pod/container identities and persistent save + markers remain intact and games continue progressing. Reopen access only + after these checks; record completion time and any lost post-backup changes. +8. Record achieved recovery-point age and RTO, compare with the selected targets, + and retain the drill/incident report without passwords, keys, kubeconfigs, + session cookies or share tokens. Take a new protected backup after the intended + authorization and credential state has been restored. + +## Missing or wrong management key + +Keep the panel stopped. Recover the exact key version paired with the chosen +database backup from separate custody. Existing encrypted credentials cannot be +recovered by generating a new random key. Current startup rejects a missing key +with `management encryption key is missing`; an inactive/wrong key for a recorded +key identity reports `management encryption key is inactive`. Other corruption +may fail authentication of encrypted records. None is permission to serve partial +management state. + +If the matching key is irretrievably lost, retain the database/audit archive and +treat the encrypted credentials as lost. Recover them from their original issuers +or reissue/re-register them under an explicitly reviewed rebuild plan. Reconcile +users, grants and Kubernetes identities before exposing a rebuilt panel. A game +backup cannot recover this key or the encrypted management credentials. + +## Automated CI drill and measured results + +`TestPanel_OfflineRecovery` runs in the amd64 `recovery` E2E bucket. It uses a +unique namespace, its own standalone API from the built image, separate database +and archive PVCs, separate original/wrong key Secrets and a private login budget. +The API has no Kubernetes service-account token: reading the existing GameServer +through a registered cluster proves the restored encrypted kubeconfig works. + +The fixture seeds a custom role, a user with a scoped grant, a registration, +sessions, a share and audit events. It stops the API, takes a consistent archive, +mounts the archive read-only, creates a post-backup user, then deletes the working +database. It restores a copy, invalidates restored bearers while preserving audit +rows and the archive digest, and checks wrong-key startup exits without a successful +HTTP response during the startup probe. With the original key it checks login, +allow/deny permissions, registration-backed reads, old-cookie/share denial and +loss of the deliberately post-backup user. A busybox workload retains its Pod UID, +container ID/restart count and persistent marker while its counter advances. + +The `panel-recovery--` CI artifact contains JSON with: + +- `recovery_point_age_at_loss_seconds`: age of the completed fixture backup when + the working database was lost. This measures the fixture's recovery point; + post-backup changes are deliberately absent after restore. +- `rto_seconds`: time from simulated loss through restoration and acceptance + checks, **including** the negative wrong-key trial. Cluster/bootstrap setup and + backup creation precede this clock. +- Check results, archive/audit digests and row counts, and game identity evidence. + +The report records actual run values; no benchmark values are hard-coded here. +A failed run writes a partial report with `passed: false`; missing timing fields +mean that stage was not reached. The artifact contains no keys or raw database +rows. Use repeated passing runs and your own production-shaped drills to select +and demonstrate targets. This fixture covers SQLite panel recovery, not storage +cluster loss, PostgreSQL failover, a particular game's save format, or a complete +Kubernetes disaster recovery exercise. diff --git a/docs/standalone-panel.md b/docs/standalone-panel.md index c405b996b..d1deab8eb 100644 --- a/docs/standalone-panel.md +++ b/docs/standalone-panel.md @@ -66,6 +66,11 @@ registered clusters. Gateway credentials are configured through the API. ## Storage and recovery +For database outage/loss, key loss, authorization reconciliation and restoring +the panel while games continue running, follow the [panel recovery runbook](panel-recovery.md). +It builds on the storage/key procedures below and describes the CI recovery +drill's measured recovery-point age and recovery time. + The Compose volume `panel-data` contains `/data/gameplane.db`; a separate `panel-keys` volume contains `/keys/panel.key`. Standalone settings and remote credentials are stored in SQL; credential values are authenticated-encrypted diff --git a/test/e2e/buckets.sh b/test/e2e/buckets.sh index d60a26f14..d21411ea1 100755 --- a/test/e2e/buckets.sh +++ b/test/e2e/buckets.sh @@ -379,6 +379,13 @@ TestOptionalComponents_Lifecycle EOF } +# recovery owns a separate standalone API/database/key in a unique namespace. +# Zero shared panel logins; the fixture has its own login limiter and storage. +bucket_recovery() { cat <<'EOF' +TestPanel_OfflineRecovery +EOF +} + # capture-overhead is kept OUT of bot-fast on purpose: bot-fast's matrix legs # are blocking, while this bucket feeds the non-blocking e2e-capture-overhead # job (continue-on-error, never blocks a merge — spec 007 OD-1) and runs only @@ -395,7 +402,7 @@ EOF unbucketed() { :; } bucket_names() { - printf '%s\n' operator api-auth api-roles api-rbac api-agent api-mods ratelimit bot-fast bot-heavy multicluster upgrade telemetry capture-overhead optional-components + printf '%s\n' operator api-auth api-roles api-rbac api-agent api-mods ratelimit bot-fast bot-heavy multicluster upgrade telemetry capture-overhead optional-components recovery } list_bucket() { @@ -414,6 +421,7 @@ list_bucket() { telemetry) bucket_telemetry ;; capture-overhead) bucket_capture_overhead ;; optional-components) bucket_optional_components ;; + recovery) bucket_recovery ;; *) echo "unknown bucket: $1 (known: $(bucket_names | tr '\n' ' '))" >&2 exit 2 diff --git a/test/e2e/panel_recovery_e2e_test.go b/test/e2e/panel_recovery_e2e_test.go new file mode 100644 index 000000000..45a54bee7 --- /dev/null +++ b/test/e2e/panel_recovery_e2e_test.go @@ -0,0 +1,526 @@ +//go:build e2e + +package e2e + +import ( + "context" + "crypto/rand" + _ "embed" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" +) + +//go:embed testdata/recovery/maintenance.py +var recoveryMaintenance string + +type recoveryFixture struct { + t *testing.T + ns, image, node, allowedCIDR string +} + +// TestPanel_OfflineRecovery owns its API, database, archive and login budget. +// It never stops the installed API/operator or restores shared cluster state. +func TestPanel_OfflineRecovery(t *testing.T) { + t.Parallel() + e := envInstance + ctx := t.Context() + suffix := strconv.FormatInt(time.Now().UnixNano(), 36) + f := &recoveryFixture{t: t, ns: "e2e-recovery-" + suffix} + report := map[string]any{"fixture": "standalone-sqlite-kind", "namespace": f.ns, "started_utc": time.Now().UTC(), "stage": "bootstrap", "passed": false} + t.Cleanup(func() { + report["passed"], report["finished_utc"] = !t.Failed(), time.Now().UTC() + dir := os.Getenv("GAMEPLANE_RECOVERY_REPORT_DIR") + if dir == "" { + dir = t.TempDir() + } + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Errorf("create recovery report directory: %v", err) + return + } + data, err := json.MarshalIndent(report, "", " ") + if err != nil { + t.Errorf("encode recovery report: %v", err) + return + } + if err := os.WriteFile(filepath.Join(dir, f.ns+".json"), data, 0o600); err != nil { + t.Errorf("write recovery report: %v", err) + } + }) + if _, err := e.K8s.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: f.ns}}, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = e.K8s.CoreV1().Namespaces().Delete(context.Background(), f.ns, metav1.DeleteOptions{}) }) + deploy, err := e.K8s.AppsV1().Deployments("gameplane-system").Get(ctx, "gameplane-api", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + for _, c := range deploy.Spec.Template.Spec.Containers { + if c.Name == "api" { + f.image = c.Image + } + } + if f.image == "" { + t.Fatal("installed API image unavailable") + } + kubernetes, err := e.K8s.CoreV1().Services("default").Get(ctx, "kubernetes", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + f.allowedCIDR = kubernetes.Spec.ClusterIP + "/32" + if strings.Contains(kubernetes.Spec.ClusterIP, ":") { + f.allowedCIDR = kubernetes.Spec.ClusterIP + "/128" + } + for _, name := range []string{"data", "archive"} { + pvc := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: f.ns}, Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, Resources: corev1.VolumeResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("1Gi")}}}} + if _, err := e.K8s.CoreV1().PersistentVolumeClaims(f.ns).Create(ctx, pvc, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + } + password := recoveryRandom(t) + for name, data := range map[string]map[string][]byte{ + "original-key": {"panel.key": recoveryKey(t)}, "wrong-key": {"panel.key": recoveryKey(t)}, "bootstrap": {"password": []byte(password)}, + } { + if _, err := e.K8s.CoreV1().Secrets(f.ns).Create(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name}, Data: data}, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + } + service := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "panel", Namespace: f.ns}, Spec: corev1.ServiceSpec{Selector: map[string]string{"app": "recovery-panel"}, PublishNotReadyAddresses: true, Ports: []corev1.ServicePort{{Port: 8000, TargetPort: intstr.FromInt32(8000)}}}} + if _, err := e.K8s.CoreV1().Services(f.ns).Create(ctx, service, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + f.startAPI("initial", "original-key", true) + pod := f.waitPodReady("initial") + f.node = pod.Spec.NodeName + f.startMaintenance(false) + probeURL := "http://panel." + f.ns + ".svc:8000/healthz" + // Positive control: a blocked or misrouted Service must not make the later + // no-HTTP negative case appear to pass. + e.Eventually(t, 30*time.Second, func() (bool, string) { + result := f.maintenance("http-check", probeURL) + return result["status"] == float64(http.StatusOK), "waiting for recovery HTTP monitor positive control" + }) + report["http_monitor_positive_control"] = true + admin := f.login("initial", "admin", password) + defer admin.Close() + + // Register the existing kind cluster using stored, encrypted credentials. + // The panel has no service-account token, so successful dispatch depends on + // decrypting this registration after recovery. + clusterID := "fixture-cluster" + f.request(admin, http.MethodPost, "/clusters", map[string]any{"name": clusterID, "displayName": "Recovery fixture", "kubeconfig": recoveryKubeconfig(t)}, http.StatusCreated) + roleName := "recovery-reader" + f.request(admin, http.MethodPost, "/roles", map[string]any{"name": roleName, "description": "Recovery fixture read only", "permissions": []string{"servers:read"}}, http.StatusCreated) + userPassword := recoveryRandom(t) + created := f.request(admin, http.MethodPost, "/users", map[string]any{"username": "reader", "password": userPassword, "role": roleName}, http.StatusOK) + userID := extractIntField(string(created), "id") + if userID == "" { + t.Fatal("created reader has no ID") + } + f.request(admin, http.MethodPost, "/users/"+userID+"/bindings", map[string]any{"roleName": roleName, "cluster": clusterID, "namespace": "gameplane-games"}, http.StatusCreated) + reader := f.login("initial", "reader", userPassword) + defer reader.Close() + gameName, gameUID := recoveryGame(t, suffix) + serversPath := "/servers?cluster=" + clusterID + "&namespace=gameplane-games" + f.waitServer(reader, serversPath, gameUID) + shareBody := f.request(admin, http.MethodPost, "/servers/"+gameName+":shares?cluster="+clusterID+"&namespace=gameplane-games", map[string]any{"canStart": false, "neverExpires": true}, http.StatusOK) + var share struct { + Token string `json:"token"` + } + if err := json.Unmarshal(shareBody, &share); err != nil || share.Token == "" { + t.Fatal("share token missing") + } + public := &APIClient{BaseURL: admin.BaseURL, HTTP: &http.Client{Timeout: 10 * time.Second}} + f.request(public, http.MethodGet, "/shares/"+share.Token, nil, http.StatusOK) + beforeGame := recoveryGameIdentity(t, gameName) + beforeCounter := recoveryCounter(t, gameName) + report["game_before"] = beforeGame + report["counter_before"] = beforeCounter + + // Stop the sole API before SQLite's online-backup API takes the archive. + admin.Close() + reader.Close() + f.stopPod("initial") + report["stage"] = "backup" + backup := f.maintenance("backup") + report["backup"] = backup + backupTime, err := time.Parse(time.RFC3339Nano, backup["backup_utc"].(string)) + if err != nil { + t.Fatal("invalid archive recovery-point timestamp") + } + if backup["sessions"].(float64) < 2 || backup["active_shares"].(float64) < 1 || backup["audit_rows"].(float64) < 1 { + t.Fatal("backup lacks session/share/audit fixture evidence") + } + // From here on the archive is mounted read-only even in the maintenance Pod. + f.stopPod("maintenance") + f.startMaintenance(true) + f.startAPI("after-backup", "original-key", false) + f.waitPodReady("after-backup") + afterBackup := f.login("after-backup", "admin", password) + f.request(afterBackup, http.MethodPost, "/users", map[string]any{"username": "after-backup", "password": recoveryRandom(t), "role": "viewer"}, http.StatusOK) + afterBackup.Close() + f.stopPod("after-backup") + lossTime := time.Now() + report["stage"] = "loss-and-restore" + report["loss_utc"] = lossTime.UTC() + report["recovery_point_age_at_loss_seconds"] = lossTime.Sub(backupTime).Seconds() + loss := f.maintenance("loss") + if loss["working_database_absent"] != true || loss["archive_digest"] != backup["archive_digest"] { + t.Fatal("loss simulation did not preserve separate archive") + } + restored := f.maintenance("restore") + report["restored_copy"] = restored + if restored["archive_digest"] != backup["archive_digest"] || restored["audit_digest"] != backup["audit_digest"] || restored["audit_rows"] != backup["audit_rows"] { + t.Fatal("restore changed original archive or audit evidence") + } + if recoveryCounter(t, gameName) <= beforeCounter { + t.Fatal("game did not continue across panel downtime") + } + + // Start an HTTP monitor before the wrong-key process. A startup error alone + // is insufficient evidence if a listener briefly served partial state first. + report["stage"] = "wrong-key" + probeCtx, cancelProbe := context.WithCancel(ctx) + defer cancelProbe() + type probeResult struct { + output string + err error + } + probeDone := make(chan probeResult, 1) + go func() { + out, err := f.execMaintenance(probeCtx, "probe", probeURL) + probeDone <- probeResult{out, err} + }() + e.Eventually(t, 30*time.Second, func() (bool, string) { + result := f.maintenance("probe-ready") + return result["ready"] == true, "waiting for wrong-key HTTP monitor" + }) + f.startAPI("wrong-key", "wrong-key", false) + e.Eventually(t, 90*time.Second, func() (bool, string) { + p, err := e.K8s.CoreV1().Pods(f.ns).Get(ctx, "wrong-key", metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + for _, status := range p.Status.ContainerStatuses { + if status.Name == "api" && status.State.Terminated != nil { + return status.State.Terminated.ExitCode != 0, "wrong-key process exited successfully" + } + } + return false, "waiting for wrong-key startup rejection" + }) + logs, err := e.K8s.CoreV1().Pods(f.ns).GetLogs("wrong-key", &corev1.PodLogOptions{Container: "api"}).DoRaw(ctx) + if err != nil || !strings.Contains(string(logs), "management encryption key is inactive") { + t.Fatal("wrong-key startup did not report the expected inactive-key error (logs withheld)") + } + f.maintenance("probe-stop") + var probe map[string]any + select { + case result := <-probeDone: + if result.err != nil || json.Unmarshal([]byte(result.output), &probe) != nil { + t.Fatal("wrong-key HTTP monitor failed") + } + case <-ctx.Done(): + t.Fatal("wrong-key HTTP monitor did not stop") + } + if probe["responses"] != float64(0) || probe["stopped"] != true || probe["attempts"].(float64) < 1 { + t.Fatal("wrong-key process served HTTP or monitor did not finish") + } + report["wrong_key_probe"] = probe + f.stopPod("wrong-key") + report["stage"] = "restored-acceptance" + f.startAPI("restored", "original-key", false) + f.waitPodReady("restored") + recovered := f.login("restored", "admin", password) + defer recovered.Close() + // Reuse the original cookie jar against the recovered endpoint, without login. + stale := &APIClient{BaseURL: recovered.BaseURL, HTTP: reader.HTTP, CSRF: reader.CSRF} + f.request(stale, http.MethodGet, "/auth/me", nil, http.StatusUnauthorized) + public.BaseURL = recovered.BaseURL + f.request(public, http.MethodGet, "/shares/"+share.Token, nil, http.StatusNotFound) + recoveredReader := f.login("restored", "reader", userPassword) + defer recoveredReader.Close() + f.waitServer(recoveredReader, serversPath, gameUID) + f.request(recoveredReader, http.MethodPost, "/servers?cluster="+clusterID+"&namespace=gameplane-games", map[string]any{"metadata": map[string]any{"name": "forbidden"}}, http.StatusForbidden) + users := f.request(recovered, http.MethodGet, "/users", nil, http.StatusOK) + if strings.Contains(string(users), `"after-backup"`) || !strings.Contains(string(users), `"reader"`) { + t.Fatal("restored users do not match the recovery point") + } + roles := f.request(recovered, http.MethodGet, "/roles", nil, http.StatusOK) + if !strings.Contains(string(roles), `"`+roleName+`"`) { + t.Fatal("custom role missing after recovery") + } + bindings := f.request(recovered, http.MethodGet, "/users/"+userID+"/bindings", nil, http.StatusOK) + if !strings.Contains(string(bindings), `"`+clusterID+`"`) { + t.Fatal("scoped grant missing after recovery") + } + f.request(recovered, http.MethodGet, "/admin/audit", nil, http.StatusOK) + verification := f.request(recovered, http.MethodGet, "/admin/audit/verify", nil, http.StatusOK) + var integrity struct { + OK bool `json:"ok"` + Checked int64 `json:"checked"` + } + if json.Unmarshal(verification, &integrity) != nil || !integrity.OK || integrity.Checked < 1 { + t.Fatal("restored audit chain failed verification") + } + report["audit_chain_verified_rows"] = integrity.Checked + if f.maintenance("archive")["archive_digest"] != backup["archive_digest"] { + t.Fatal("original archive mutated during recovery") + } + afterGame := recoveryGameIdentity(t, gameName) + if afterGame != beforeGame { + t.Fatalf("game identity changed during panel recovery: before=%+v after=%+v", beforeGame, afterGame) + } + e.Eventually(t, 15*time.Second, func() (bool, string) { + return recoveryCounter(t, gameName) > beforeCounter, "game counter did not advance" + }) + report["game_after"], report["credential_backed_dispatch_verified"], report["restored_bearers_invalidated"] = afterGame, true, true + report["counter_after"] = recoveryCounter(t, gameName) + report["rto_seconds"] = time.Since(lossTime).Seconds() + report["rto_scope"] = "loss through restore, auth invalidation, wrong-key trial, and credential-backed acceptance checks" + report["stage"] = "complete" +} + +func recoveryKey(t *testing.T) []byte { + t.Helper() + key := make([]byte, 32) + if _, err := rand.Read(key); err != nil { + t.Fatal(err) + } + return key +} + +func recoveryRandom(t *testing.T) string { return hex.EncodeToString(recoveryKey(t)) } + +func (f *recoveryFixture) request(c *APIClient, method, path string, body any, want int) []byte { + f.t.Helper() + resp, data, err := c.Do(method, path, body) + if err != nil { + f.t.Fatalf("recovery HTTP request failed (details withheld): %s", method) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != want { + f.t.Fatalf("recovery HTTP %s: status=%d want=%d (path/body withheld)", method, resp.StatusCode, want) + } + return data +} + +func (f *recoveryFixture) login(pod, user, password string) *APIClient { + f.t.Helper() + port, stop := envInstance.PortForward(f.t, f.ns, "pod/"+pod, 8000) + c := &APIClient{BaseURL: fmt.Sprintf("http://127.0.0.1:%d", port), HTTP: &http.Client{Jar: newInsecureCookieJar(), Timeout: 20 * time.Second}, stop: stop} + f.t.Cleanup(stop) + data := f.request(c, http.MethodPost, "/auth/login", map[string]string{"username": user, "password": password}, http.StatusOK) + var login struct { + CSRF string `json:"csrf"` + } + if err := json.Unmarshal(data, &login); err != nil || login.CSRF == "" { + f.t.Fatal("login response lacked CSRF token") + } + c.CSRF = login.CSRF + return c +} + +func recoverySecurity() (*corev1.PodSecurityContext, *corev1.SecurityContext) { + uid, yes, no := int64(65532), true, false + return &corev1.PodSecurityContext{RunAsUser: &uid, RunAsGroup: &uid, FSGroup: &uid, RunAsNonRoot: &yes, SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}}, &corev1.SecurityContext{AllowPrivilegeEscalation: &no, ReadOnlyRootFilesystem: &yes, Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}} +} + +func (f *recoveryFixture) startAPI(name, key string, bootstrap bool) { + f.t.Helper() + psc, sc := recoverySecurity() + no, mode := false, int32(0o440) + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: f.ns, Labels: map[string]string{"app": "recovery-panel"}}, Spec: corev1.PodSpec{RestartPolicy: corev1.RestartPolicyNever, AutomountServiceAccountToken: &no, SecurityContext: psc, Containers: []corev1.Container{{Name: "api", Image: f.image, ImagePullPolicy: corev1.PullIfNotPresent, Command: []string{"/api"}, Args: []string{"--standalone", "--panel-key-file=/keys/panel.key", "--panel-key-provisioned", "--db-driver=sqlite", "--db-dsn=file:/data/gameplane.db?_pragma=journal_mode(WAL)", "--remote-allowed-cidrs=" + f.allowedCIDR}, SecurityContext: sc, VolumeMounts: []corev1.VolumeMount{{Name: "data", MountPath: "/data"}, {Name: "key", MountPath: "/keys", ReadOnly: true}}, ReadinessProbe: &corev1.Probe{ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{Path: "/healthz", Port: intstr.FromInt32(8000)}}, PeriodSeconds: 1}, Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m"), corev1.ResourceMemory: resource.MustParse("128Mi")}, Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("512Mi")}}}}, Volumes: []corev1.Volume{{Name: "data", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "data"}}}, {Name: "key", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: key, DefaultMode: &mode}}}}}} + if bootstrap { + pod.Spec.InitContainers = []corev1.Container{{Name: "bootstrap", Image: f.image, ImagePullPolicy: corev1.PullIfNotPresent, Command: []string{"/api", "bootstrap-admin", "--username", "admin"}, SecurityContext: sc, Env: []corev1.EnvVar{{Name: "GAMEPLANE_DB_DSN", Value: "file:/data/gameplane.db?_pragma=journal_mode(WAL)"}, {Name: "GAMEPLANE_ADMIN_PASSWORD", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "bootstrap"}, Key: "password"}}}}, VolumeMounts: []corev1.VolumeMount{{Name: "data", MountPath: "/data"}}}} + } + if _, err := envInstance.K8s.CoreV1().Pods(f.ns).Create(f.t.Context(), pod, metav1.CreateOptions{}); err != nil { + f.t.Fatal(err) + } +} + +func (f *recoveryFixture) waitPodReady(name string) *corev1.Pod { + f.t.Helper() + var pod *corev1.Pod + envInstance.Eventually(f.t, 3*time.Minute, func() (bool, string) { + var err error + pod, err = envInstance.K8s.CoreV1().Pods(f.ns).Get(f.t.Context(), name, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + for _, c := range pod.Status.Conditions { + if c.Type == corev1.PodReady && c.Status == corev1.ConditionTrue { + return true, "" + } + } + return false, "waiting for isolated fixture Pod " + name + " (phase=" + string(pod.Status.Phase) + ")" + }) + return pod +} + +func (f *recoveryFixture) stopPod(name string) { + f.t.Helper() + if err := envInstance.K8s.CoreV1().Pods(f.ns).Delete(f.t.Context(), name, metav1.DeleteOptions{}); err != nil { + f.t.Fatal(err) + } + envInstance.Eventually(f.t, 90*time.Second, func() (bool, string) { + _, err := envInstance.K8s.CoreV1().Pods(f.ns).Get(f.t.Context(), name, metav1.GetOptions{}) + return apierrors.IsNotFound(err), "waiting for API termination before offline database operation" + }) +} + +func (f *recoveryFixture) startMaintenance(archiveReadOnly bool) { + psc, sc := recoverySecurity() + no := false + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "maintenance", Namespace: f.ns}, Spec: corev1.PodSpec{RestartPolicy: corev1.RestartPolicyNever, AutomountServiceAccountToken: &no, SecurityContext: psc, Affinity: &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{NodeSelectorTerms: []corev1.NodeSelectorTerm{{MatchFields: []corev1.NodeSelectorRequirement{{Key: "metadata.name", Operator: corev1.NodeSelectorOpIn, Values: []string{f.node}}}}}}}}, Containers: []corev1.Container{{Name: "maintenance", Image: "python:3.12-alpine", Command: []string{"python3", "-c", "import time; time.sleep(1800)"}, SecurityContext: sc, VolumeMounts: []corev1.VolumeMount{{Name: "data", MountPath: "/data"}, {Name: "archive", MountPath: "/archive"}}}}, Volumes: []corev1.Volume{{Name: "data", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "data"}}}, {Name: "archive", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "archive"}}}}}} + pod.Spec.Containers[0].VolumeMounts[1].ReadOnly = archiveReadOnly + pod.Spec.Volumes[1].PersistentVolumeClaim.ReadOnly = archiveReadOnly + if _, err := envInstance.K8s.CoreV1().Pods(f.ns).Create(f.t.Context(), pod, metav1.CreateOptions{}); err != nil { + f.t.Fatal(err) + } + f.waitPodReady("maintenance") +} + +func (f *recoveryFixture) execMaintenance(ctx context.Context, args ...string) (string, error) { + command := []string{"exec", "-n", f.ns, "maintenance", "--", "python3", "-c", recoveryMaintenance} + return envInstance.Kubectl(ctx, append(command, args...)...) +} + +func (f *recoveryFixture) maintenance(op string, args ...string) map[string]any { + f.t.Helper() + out, err := f.execMaintenance(f.t.Context(), append([]string{op}, args...)...) + if err != nil { + f.t.Fatalf("offline recovery operation %s failed (output withheld)", op) + } + var result map[string]any + if err := json.Unmarshal([]byte(out), &result); err != nil { + f.t.Fatalf("decode offline recovery operation %s", op) + } + return result +} + +func recoveryKubeconfig(t *testing.T) string { + t.Helper() + cfg := rest.CopyConfig(envInstance.Cfg) + if err := rest.LoadTLSFiles(cfg); err != nil { + t.Fatal("could not load kind TLS fixture") + } + if len(cfg.CAData) == 0 || len(cfg.CertData) == 0 || len(cfg.KeyData) == 0 { + t.Fatal("recovery drill requires kind client certificate credentials") + } + kubeconfig := clientcmdapi.Config{Clusters: map[string]*clientcmdapi.Cluster{"kind": {Server: "https://kubernetes.default.svc", CertificateAuthorityData: cfg.CAData}}, AuthInfos: map[string]*clientcmdapi.AuthInfo{"kind": {ClientCertificateData: cfg.CertData, ClientKeyData: cfg.KeyData}}, Contexts: map[string]*clientcmdapi.Context{"kind": {Cluster: "kind", AuthInfo: "kind"}}, CurrentContext: "kind"} + data, err := clientcmd.Write(kubeconfig) + if err != nil { + t.Fatal("could not encode kind TLS fixture") + } + return string(data) +} + +func (f *recoveryFixture) waitServer(c *APIClient, path, uid string) { + f.t.Helper() + envInstance.Eventually(f.t, 90*time.Second, func() (bool, string) { + resp, data, err := c.Get(path) + if err != nil { + return false, "waiting for credential-backed cluster dispatch" + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return false, "waiting for credential-backed cluster dispatch" + } + var list struct { + Items []unstructured.Unstructured `json:"items"` + } + if json.Unmarshal(data, &list) != nil { + return false, "invalid server inventory" + } + for _, item := range list.Items { + if string(item.GetUID()) == uid { + return true, "" + } + } + return false, "expected live GameServer UID absent from registered cluster" + }) +} + +func recoveryGame(t *testing.T, suffix string) (string, string) { + t.Helper() + name := "recovery-game-" + suffix + applyBusyboxTemplate(t, name) + tmpl, err := envInstance.Dyn.Resource(gameTemplateGVR).Get(t.Context(), name, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if err := unstructured.SetNestedStringSlice(tmpl.Object, []string{"sh", "-c", "echo retained > /data/recovery-marker; n=0; while true; do n=$((n+1)); echo $n > /data/recovery-counter.next; mv /data/recovery-counter.next /data/recovery-counter; sleep 1; done"}, "spec", "command"); err != nil { + t.Fatal(err) + } + if _, err := envInstance.Dyn.Resource(gameTemplateGVR).Update(t.Context(), tmpl, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + applyBusyboxGameServer(t, "gameplane-games", name, name) + envInstance.Eventually(t, 3*time.Minute, func() (bool, string) { + pod, err := envInstance.K8s.CoreV1().Pods("gameplane-games").Get(t.Context(), name+"-0", metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + for _, c := range pod.Status.ContainerStatuses { + if c.Name == "game" && c.State.Running != nil { + return true, "" + } + } + return false, "waiting for persistent game fixture" + }) + gs, err := envInstance.Dyn.Resource(gameServerGVR).Namespace("gameplane-games").Get(t.Context(), name, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + return name, string(gs.GetUID()) +} + +type recoveryGameState struct { + PodUID, ContainerID string + Restarts int32 +} + +func recoveryGameIdentity(t *testing.T, name string) recoveryGameState { + t.Helper() + pod, err := envInstance.K8s.CoreV1().Pods("gameplane-games").Get(t.Context(), name+"-0", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + for _, c := range pod.Status.ContainerStatuses { + if c.Name == "game" && c.State.Running != nil && c.ContainerID != "" { + return recoveryGameState{string(pod.UID), c.ContainerID, c.RestartCount} + } + } + t.Fatal("game container is not running") + return recoveryGameState{} +} + +func recoveryCounter(t *testing.T, name string) int { + t.Helper() + out, err := envInstance.Kubectl(t.Context(), "exec", "-n", "gameplane-games", name+"-0", "-c", "game", "--", "sh", "-c", "test \"$(cat /data/recovery-marker)\" = retained && cat /data/recovery-counter") + if err != nil { + t.Fatal("read persistent game marker/counter") + } + n, err := strconv.Atoi(strings.TrimSpace(out)) + if err != nil { + t.Fatal("invalid game counter") + } + return n +} diff --git a/test/e2e/specs.md b/test/e2e/specs.md index abf7cda91..aaf952839 100644 --- a/test/e2e/specs.md +++ b/test/e2e/specs.md @@ -19,6 +19,20 @@ It was one of three `go.work` modules without CI lint coverage until this featur - Validate multi-cluster dispatch, upgrade-from-previous-release, wake-on-connect, module signing/SSRF rejection, and backup/restore round trips end to end. - Enforce, via `buckets.sh`, that every test in the suite is assigned to exactly one CI bucket so nothing silently drops out of coverage. +### Panel recovery drill + +The amd64 `recovery` bucket runs `TestPanel_OfflineRecovery` in a unique namespace +with its own standalone API, SQL database/archive PVCs, separately mounted keys +and private login budget. It exercises stopped SQLite backup, working database +loss, restore and offline bearer invalidation. It verifies correct-key users, +custom roles/scoped grants, encrypted-registration-backed GameServer UID reads, +wrong-key startup failure/no HTTP response, old session/share rejection, retained +audit evidence and an unchanged continuously running game Pod/container. The +read-only original archive is never changed by invalidation. Actual fixture +recovery-point age/RTO and acceptance evidence are uploaded as JSON, with no key, +credential, cookie, share token or raw SQL rows. It does not stop the installed +panel/operator or claim PostgreSQL PITR coverage. See `docs/panel-recovery.md`. + ### Remote gateway parity acceptance The multicluster bucket runs `TestMultiCluster_GatewayParity` against two real diff --git a/test/e2e/testdata/recovery/maintenance.py b/test/e2e/testdata/recovery/maintenance.py new file mode 100644 index 000000000..9c7d65711 --- /dev/null +++ b/test/e2e/testdata/recovery/maintenance.py @@ -0,0 +1,116 @@ +"""Offline SQLite maintenance for the isolated recovery fixture, never production. + +The API must be stopped for backup/loss/restore. No key is mounted here. Output +contains only counts, timestamps and digests, never rows, credentials or tokens. +""" +import datetime +from contextlib import closing +import hashlib +import json +import pathlib +import sqlite3 +import sys +import time +import urllib.error +import urllib.request + +DATA = pathlib.Path("/data/gameplane.db") +ARCHIVE = pathlib.Path("/archive/panel-backup.db") + + +def digest(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def readonly(path): + return sqlite3.connect("file:" + str(path) + "?mode=ro", uri=True) + + +def inventory(db): + assert db.execute("PRAGMA integrity_check").fetchall() == [("ok",)] + rows = db.execute("SELECT * FROM audit_events ORDER BY id").fetchall() + return { + "audit_rows": len(rows), + "audit_digest": hashlib.sha256(json.dumps(rows, separators=(",", ":")).encode()).hexdigest(), + "users": db.execute("SELECT COUNT(*) FROM users").fetchone()[0], + "sessions": db.execute("SELECT COUNT(*) FROM sessions").fetchone()[0], + "active_shares": db.execute("SELECT COUNT(*) FROM share_links WHERE revoked_at IS NULL").fetchone()[0], + } + + +def remove_working_database(): + for suffix in ("", "-wal", "-shm"): + pathlib.Path(str(DATA) + suffix).unlink(missing_ok=True) + + +op = sys.argv[1] +if op == "backup": + assert not ARCHIVE.exists(), "archive must not be overwritten" + with closing(readonly(DATA)) as source, closing(sqlite3.connect(ARCHIVE)) as dest: + source.backup(dest) + # Make the archive self-contained; do not retain a destination WAL. + dest.execute("PRAGMA journal_mode=DELETE") + result = inventory(dest) + # The original archive is never modified again, including by auth cleanup. + ARCHIVE.chmod(0o400) + result.update(archive_digest=digest(ARCHIVE), backup_utc=datetime.datetime.now(datetime.timezone.utc).isoformat()) +elif op == "loss": + assert ARCHIVE.is_file(), "loss requires a separate archive" + remove_working_database() + result = {"working_database_absent": not DATA.exists(), "archive_digest": digest(ARCHIVE)} +elif op == "restore": + assert not DATA.exists(), "restore only into the deleted working database" + before = digest(ARCHIVE) + with closing(readonly(ARCHIVE)) as source, closing(sqlite3.connect(DATA)) as dest: + source.backup(dest) + original = inventory(source) + assert inventory(dest) == original + # Invalidate restored bearer credentials on the working copy, preserving + # the original archive and audit chain for incident investigation. + dest.execute("DELETE FROM sessions") + dest.execute("DELETE FROM api_tokens") + dest.execute("UPDATE share_links SET revoked_at=? WHERE revoked_at IS NULL", (datetime.datetime.now(datetime.timezone.utc).isoformat(),)) + dest.commit() + result = inventory(dest) + assert result["audit_digest"] == original["audit_digest"] + assert result["sessions"] == 0 and result["active_shares"] == 0 + DATA.chmod(0o600) + assert digest(ARCHIVE) == before + result["archive_digest"] = before +elif op == "archive": + result = {"archive_digest": digest(ARCHIVE)} +elif op == "http-check": + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + try: + with opener.open(sys.argv[2], timeout=2) as response: + result = {"status": response.status} + except (urllib.error.URLError, TimeoutError, OSError): + result = {"status": 0} +elif op == "probe": + stop = pathlib.Path("/data/probe-stop") + stop.unlink(missing_ok=True) + pathlib.Path("/data/probe-ready").write_text("ready") + attempts, responses = 0, 0 + deadline = time.monotonic() + 120 + # Bypass proxy environment settings: this is an in-cluster Service probe. + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + while not stop.exists() and time.monotonic() < deadline: + attempts += 1 + try: + with opener.open(sys.argv[2], timeout=0.5) as response: + response.read(1) + responses += 1 + except urllib.error.HTTPError: + responses += 1 # Any HTTP response is a failure for wrong-key startup. + except (urllib.error.URLError, TimeoutError, OSError): + pass + time.sleep(0.05) + result = {"attempts": attempts, "responses": responses, "stopped": stop.exists()} +elif op == "probe-ready": + result = {"ready": pathlib.Path("/data/probe-ready").exists()} +elif op == "probe-stop": + pathlib.Path("/data/probe-stop").write_text("stop") + result = {"stopped": True} +else: + raise ValueError("unknown maintenance operation") +print(json.dumps(result, sort_keys=True)) From 51c63720147492bd88308ef62a44a93e7bd5051e Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 21 Oct 2026 12:32:23 -0400 Subject: [PATCH 02/13] test: bind recovery requests to the fixture context Signed-off-by: Baylem --- test/e2e/env.go | 7 ++++++- test/e2e/panel_recovery_e2e_test.go | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/test/e2e/env.go b/test/e2e/env.go index 35e3da008..abd376d19 100644 --- a/test/e2e/env.go +++ b/test/e2e/env.go @@ -571,6 +571,11 @@ func (c *APIClient) Close() { // marshalled as JSON and the Content-Type header is set. CSRF header is // attached for mutating methods. func (c *APIClient) Do(method, path string, body any) (*http.Response, []byte, error) { + return c.DoContext(context.Background(), method, path, body) +} + +// DoContext performs an authenticated request within the caller's lifetime. +func (c *APIClient) DoContext(ctx context.Context, method, path string, body any) (*http.Response, []byte, error) { var br io.Reader if body != nil { b, err := json.Marshal(body) @@ -579,7 +584,7 @@ func (c *APIClient) Do(method, path string, body any) (*http.Response, []byte, e } br = bytes.NewReader(b) } - req, err := http.NewRequestWithContext(context.Background(), method, c.BaseURL+path, br) + req, err := http.NewRequestWithContext(ctx, method, c.BaseURL+path, br) if err != nil { return nil, nil, fmt.Errorf("new request: %w", err) } diff --git a/test/e2e/panel_recovery_e2e_test.go b/test/e2e/panel_recovery_e2e_test.go index 45a54bee7..6fae31437 100644 --- a/test/e2e/panel_recovery_e2e_test.go +++ b/test/e2e/panel_recovery_e2e_test.go @@ -310,7 +310,7 @@ func recoveryRandom(t *testing.T) string { return hex.EncodeToString(recoveryKey func (f *recoveryFixture) request(c *APIClient, method, path string, body any, want int) []byte { f.t.Helper() - resp, data, err := c.Do(method, path, body) + resp, data, err := c.DoContext(f.t.Context(), method, path, body) if err != nil { f.t.Fatalf("recovery HTTP request failed (details withheld): %s", method) } @@ -435,7 +435,7 @@ func recoveryKubeconfig(t *testing.T) string { func (f *recoveryFixture) waitServer(c *APIClient, path, uid string) { f.t.Helper() envInstance.Eventually(f.t, 90*time.Second, func() (bool, string) { - resp, data, err := c.Get(path) + resp, data, err := c.DoContext(f.t.Context(), http.MethodGet, path, nil) if err != nil { return false, "waiting for credential-backed cluster dispatch" } From f12d902659b812e85124a1d6f6ff33c9b737b629 Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 21 Oct 2026 17:11:13 -0400 Subject: [PATCH 03/13] test: specialize recovery user ID parsing Signed-off-by: Baylem --- test/e2e/api_auth_e2e_test.go | 4 ++-- test/e2e/api_session_e2e_test.go | 2 +- test/e2e/env.go | 12 ++++++------ test/e2e/panel_recovery_e2e_test.go | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/test/e2e/api_auth_e2e_test.go b/test/e2e/api_auth_e2e_test.go index fafac66e1..510d277ec 100644 --- a/test/e2e/api_auth_e2e_test.go +++ b/test/e2e/api_auth_e2e_test.go @@ -187,7 +187,7 @@ func TestAPI_AuditEmitsOnMutation(t *testing.T) { } // Extract our id without pulling in encoding/json: the response // shape is small and we only need the leading "id":N field. - id := extractIntField(string(body), "id") + id := extractUserID(string(body)) if id == "" { t.Fatalf("could not parse id from /users/me: %s", string(body)) } @@ -224,7 +224,7 @@ func TestAPI_AuditEmitsOnMutation(t *testing.T) { // the *passed-down* context, which is out of scope for this test. } -// extractIntField has been promoted to env.go so non-test helper code +// extractUserID has been promoted to env.go so non-test helper code // (Env.CreateUser) can use it. Test files in this package still call // it through the package-level scope. diff --git a/test/e2e/api_session_e2e_test.go b/test/e2e/api_session_e2e_test.go index 5110beba0..603b8d69f 100644 --- a/test/e2e/api_session_e2e_test.go +++ b/test/e2e/api_session_e2e_test.go @@ -168,7 +168,7 @@ func TestAPI_AuditPaginationAndFilter(t *testing.T) { t.Fatalf("/users/me %d: %s", getMe.StatusCode, string(body)) } defer func() { _ = getMe.Body.Close() }() - id := extractIntField(string(body), "id") + id := extractUserID(string(body)) if id == "" { t.Fatalf("could not parse id from /users/me: %s", string(body)) } diff --git a/test/e2e/env.go b/test/e2e/env.go index abd376d19..4f5cb055d 100644 --- a/test/e2e/env.go +++ b/test/e2e/env.go @@ -696,21 +696,21 @@ func (e *Env) CreateUser(t *testing.T, admin *APIClient, role, prefix string) (u if resp.StatusCode != http.StatusOK { t.Fatalf("CreateUser %s/%s %d: %s", role, prefix, resp.StatusCode, string(body)) } - id = extractIntField(string(body), "id") + id = extractUserID(string(body)) if id == "" { t.Fatalf("CreateUser: could not parse id from response: %s", string(body)) } return username, userPhrase, id } -// extractIntField is a tiny scanner for `"":` inside a JSON -// blob. Lets the e2e tests parse the one or two integer fields they -// care about (user id, mostly) without pulling in encoding/json. +// extractUserID is a tiny scanner for `"id":` inside a JSON +// blob. It reads the user ID returned by the API without adding a JSON +// decoding dependency to callers that only need that field. // // Lives on env.go (not in a _test.go) so non-test helper code like // CreateUser can use it. -func extractIntField(body, key string) string { - needle := `"` + key + `":` +func extractUserID(body string) string { + const needle = `"id":` i := strings.Index(body, needle) if i < 0 { return "" diff --git a/test/e2e/panel_recovery_e2e_test.go b/test/e2e/panel_recovery_e2e_test.go index 6fae31437..d970c5717 100644 --- a/test/e2e/panel_recovery_e2e_test.go +++ b/test/e2e/panel_recovery_e2e_test.go @@ -130,7 +130,7 @@ func TestPanel_OfflineRecovery(t *testing.T) { f.request(admin, http.MethodPost, "/roles", map[string]any{"name": roleName, "description": "Recovery fixture read only", "permissions": []string{"servers:read"}}, http.StatusCreated) userPassword := recoveryRandom(t) created := f.request(admin, http.MethodPost, "/users", map[string]any{"username": "reader", "password": userPassword, "role": roleName}, http.StatusOK) - userID := extractIntField(string(created), "id") + userID := extractUserID(string(created)) if userID == "" { t.Fatal("created reader has no ID") } From fa7a28502d917dcbdea325fb85ec68a77fb39f13 Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 21 Oct 2026 17:15:52 -0400 Subject: [PATCH 04/13] fix(agent): bound NuclearOption request writes Signed-off-by: Baylem --- agent/internal/rcon/nuclearoption.go | 28 ++++++----- .../rcon/nuclearoption_deadline_test.go | 46 +++++++++++++++++++ agent/internal/rcon/nuclearoption_test.go | 2 +- agent/specs.md | 2 +- 4 files changed, 64 insertions(+), 14 deletions(-) create mode 100644 agent/internal/rcon/nuclearoption_deadline_test.go diff --git a/agent/internal/rcon/nuclearoption.go b/agent/internal/rcon/nuclearoption.go index 7b8f2b1f2..001135354 100644 --- a/agent/internal/rcon/nuclearoption.go +++ b/agent/internal/rcon/nuclearoption.go @@ -112,14 +112,21 @@ func (c *NuclearOption) Close() error { // If the response has no body (body length = 0), an empty string is returned. // Non-2000 status codes produce an error that includes the status code and, // if present, the error detail from the response body. +// One deadline bounds the complete exchange, including dialing and request writes. func (c *NuclearOption) Exec(cmd string) (string, error) { c.mu.Lock() defer c.mu.Unlock() + d := c.execDeadline + if d <= 0 { + d = defaultNuclearOptionExecDeadline + } + exchangeDeadline := time.Now().Add(d) + // Ensure connection is open. if c.conn == nil { d := net.Dialer{Timeout: c.dialTimeout} - ctx, cancel := context.WithTimeout(context.Background(), c.dialTimeout) + ctx, cancel := context.WithDeadline(context.Background(), exchangeDeadline) defer cancel() conn, err := d.DialContext(ctx, "tcp", net.JoinHostPort(c.host, fmt.Sprint(c.port))) if err != nil { @@ -144,6 +151,14 @@ func (c *NuclearOption) Exec(cmd string) (string, error) { return "", fmt.Errorf("nuclearoption rcon: marshal request: %w", err) } + // Bound writes as well as reads. Arm this before sending anything: a + // server that stops reading must not retain the shared client forever. + if err := c.conn.SetDeadline(exchangeDeadline); err != nil { + _ = c.conn.Close() + c.conn = nil + return "", fmt.Errorf("nuclearoption rcon: set deadline: %w", err) + } + // Write request: 4-byte LE length + body. reqLen := len(reqBody) if reqLen < 0 || reqLen > math.MaxUint32 { @@ -162,17 +177,6 @@ func (c *NuclearOption) Exec(cmd string) (string, error) { return "", fmt.Errorf("nuclearoption rcon: write body: %w", err) } - // Set deadline for the entire response read. - d := c.execDeadline - if d <= 0 { - d = defaultNuclearOptionExecDeadline - } - if err := c.conn.SetDeadline(time.Now().Add(d)); err != nil { - _ = c.conn.Close() - c.conn = nil - return "", fmt.Errorf("nuclearoption rcon: set deadline: %w", err) - } - // Read response: 4-byte status + 4-byte body length + body. statusBuf := make([]byte, 4) if _, err := io.ReadFull(c.conn, statusBuf); err != nil { diff --git a/agent/internal/rcon/nuclearoption_deadline_test.go b/agent/internal/rcon/nuclearoption_deadline_test.go new file mode 100644 index 000000000..9cae0e073 --- /dev/null +++ b/agent/internal/rcon/nuclearoption_deadline_test.go @@ -0,0 +1,46 @@ +package rcon + +import ( + "io" + "net" + "strings" + "testing" + "time" +) + +func TestNuclearOptionDeadlineIncludesRequestWrites(t *testing.T) { + t.Parallel() + for _, readHeader := range []bool{false, true} { + stage := "write length" + if readHeader { + stage = "write body" + } + t.Run(stage, func(t *testing.T) { + t.Parallel() + local, remote := net.Pipe() + defer local.Close() + defer remote.Close() + client := NewNuclearOption("ignored", 1, nil) + client.conn = local + client.execDeadline = 100 * time.Millisecond + if readHeader { + go func() { _, _ = io.CopyN(io.Discard, remote, 4) }() + } + done := make(chan error, 1) + go func() { _, err := client.Exec("get-server-id"); done <- err }() + select { + case err := <-done: + if err == nil || !isTimeout(err) || !strings.Contains(err.Error(), stage) { + t.Fatalf("stalled %s must time out: %v", stage, err) + } + case <-time.After(2 * time.Second): + _ = remote.Close() + <-done + t.Fatalf("stalled %s exceeded the command budget", stage) + } + if client.conn != nil { + t.Fatal("timed-out partial request must discard the connection") + } + }) + } +} diff --git a/agent/internal/rcon/nuclearoption_test.go b/agent/internal/rcon/nuclearoption_test.go index 248c6a7fb..cb46a4e21 100644 --- a/agent/internal/rcon/nuclearoption_test.go +++ b/agent/internal/rcon/nuclearoption_test.go @@ -740,7 +740,7 @@ func TestNuclearOptionWriteBodyError(t *testing.T) { } // TestNuclearOptionSetDeadlineError verifies that a failure setting the -// response read deadline (after the request was fully written) is reported +// exchange deadline (before sending the request) is reported // and the connection is reset. func TestNuclearOptionSetDeadlineError(t *testing.T) { local, remote := net.Pipe() diff --git a/agent/specs.md b/agent/specs.md index af1f57529..05d4bcbb9 100644 --- a/agent/specs.md +++ b/agent/specs.md @@ -244,7 +244,7 @@ The agent and operator modules use `k8s.io/apimachinery`/`k8s.io/client-go` v0.3 - `battleye`: UDP-based CRC32 packet framing. - `satisfactory`: HTTPS JSON function-call API with session token. - `palworld`: HTTP Basic REST admin API. - - `nuclearoption`: Dedicated TCP stream for Nuclear Option remote command protocol. + - `nuclearoption`: Dedicated TCP stream for Nuclear Option remote command protocol. One 30-second deadline bounds dialing, request writes, and response reads; responses are capped at 10 MiB. A timed-out partial exchange closes the connection before the next command. - `rest`: Generic HTTP/JSON REST console client with lazy auth (`PassFn`), 1 MiB bounded body read, configurable timeouts, auth failure cooldown, and loopback TLS protection. Provides adapters for FiveM (txAdmin: `POST /fxserver/commands`, bearer/X-TxAdmin-Token, `{"action":"console","parameter":""}`), Farming Simulator 25 (`POST /api/console`, Basic auth, `{"command":""}`), and generic REST (`POST /api/command`). - `cli`: Container stdin/PTY / local execution client (Option A in OPEN-DECISIONS.md). Drives commands via named FIFO pipe (`GAMEPLANE_CLI_PIPE`) or local process shell execution without requiring remote TCP networking or password secrets, while web console continues to attach via pod-attach PTY. From 56ba5910db4f77915f729d0f30f8c0e8f2135cf9 Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 21 Oct 2026 22:46:23 -0400 Subject: [PATCH 05/13] fix(e2e): wait for recovery cluster registration Signed-off-by: Baylem --- test/e2e/panel_recovery_e2e_test.go | 15 ++++++++------- test/e2e/specs.md | 4 +++- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/test/e2e/panel_recovery_e2e_test.go b/test/e2e/panel_recovery_e2e_test.go index d970c5717..fcd2965d1 100644 --- a/test/e2e/panel_recovery_e2e_test.go +++ b/test/e2e/panel_recovery_e2e_test.go @@ -22,6 +22,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -134,11 +135,14 @@ func TestPanel_OfflineRecovery(t *testing.T) { if userID == "" { t.Fatal("created reader has no ID") } + // Registration persists before the asynchronous registry watcher loads its + // client. Prove dispatch works before assigning a binding to that cluster. + gameName, gameUID := recoveryGame(t, suffix) + serversPath := "/servers?cluster=" + clusterID + "&namespace=gameplane-games" + f.waitServer(admin, serversPath, gameUID) f.request(admin, http.MethodPost, "/users/"+userID+"/bindings", map[string]any{"roleName": roleName, "cluster": clusterID, "namespace": "gameplane-games"}, http.StatusCreated) reader := f.login("initial", "reader", userPassword) defer reader.Close() - gameName, gameUID := recoveryGame(t, suffix) - serversPath := "/servers?cluster=" + clusterID + "&namespace=gameplane-games" f.waitServer(reader, serversPath, gameUID) shareBody := f.request(admin, http.MethodPost, "/servers/"+gameName+":shares?cluster="+clusterID+"&namespace=gameplane-games", map[string]any{"canStart": false, "neverExpires": true}, http.StatusOK) var share struct { @@ -462,14 +466,11 @@ func recoveryGame(t *testing.T, suffix string) (string, string) { t.Helper() name := "recovery-game-" + suffix applyBusyboxTemplate(t, name) - tmpl, err := envInstance.Dyn.Resource(gameTemplateGVR).Get(t.Context(), name, metav1.GetOptions{}) + patch, err := json.Marshal(map[string]any{"spec": map[string]any{"command": []string{"sh", "-c", "echo retained > /data/recovery-marker; n=0; while true; do n=$((n+1)); echo $n > /data/recovery-counter.next; mv /data/recovery-counter.next /data/recovery-counter; sleep 1; done"}}}) if err != nil { t.Fatal(err) } - if err := unstructured.SetNestedStringSlice(tmpl.Object, []string{"sh", "-c", "echo retained > /data/recovery-marker; n=0; while true; do n=$((n+1)); echo $n > /data/recovery-counter.next; mv /data/recovery-counter.next /data/recovery-counter; sleep 1; done"}, "spec", "command"); err != nil { - t.Fatal(err) - } - if _, err := envInstance.Dyn.Resource(gameTemplateGVR).Update(t.Context(), tmpl, metav1.UpdateOptions{}); err != nil { + if _, err := envInstance.Dyn.Resource(gameTemplateGVR).Patch(t.Context(), name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { t.Fatal(err) } applyBusyboxGameServer(t, "gameplane-games", name, name) diff --git a/test/e2e/specs.md b/test/e2e/specs.md index aaf952839..091552018 100644 --- a/test/e2e/specs.md +++ b/test/e2e/specs.md @@ -31,7 +31,9 @@ audit evidence and an unchanged continuously running game Pod/container. The read-only original archive is never changed by invalidation. Actual fixture recovery-point age/RTO and acceptance evidence are uploaded as JSON, with no key, credential, cookie, share token or raw SQL rows. It does not stop the installed -panel/operator or claim PostgreSQL PITR coverage. See `docs/panel-recovery.md`. +panel/operator or claim PostgreSQL PITR coverage. Initial scoped grants wait for +the asynchronous cluster registry to load the connection and permit dispatch. +See `docs/panel-recovery.md`. ### Remote gateway parity acceptance From 19b7932ea1be69d9d1bd627134c29430c4054146 Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 23 Oct 2026 00:25:54 -0400 Subject: [PATCH 06/13] test: authorize recovery fixture cluster access Signed-off-by: Baylem --- test/e2e/panel_recovery_e2e_test.go | 71 +++++++++++++++++++++++++---- test/e2e/specs.md | 6 ++- 2 files changed, 66 insertions(+), 11 deletions(-) diff --git a/test/e2e/panel_recovery_e2e_test.go b/test/e2e/panel_recovery_e2e_test.go index fcd2965d1..4f9156663 100644 --- a/test/e2e/panel_recovery_e2e_test.go +++ b/test/e2e/panel_recovery_e2e_test.go @@ -119,14 +119,30 @@ func TestPanel_OfflineRecovery(t *testing.T) { return result["status"] == float64(http.StatusOK), "waiting for recovery HTTP monitor positive control" }) report["http_monitor_positive_control"] = true - admin := f.login("initial", "admin", password) - defer admin.Close() + bootstrapAdmin := f.login("initial", "admin", password) + defer bootstrapAdmin.Close() + adminID := extractUserID(string(f.request(bootstrapAdmin, http.MethodGet, "/users/me", nil, http.StatusOK))) + if adminID == "" { + t.Fatal("bootstrap admin has no ID") + } // Register the existing kind cluster using stored, encrypted credentials. // The panel has no service-account token, so successful dispatch depends on // decrypting this registration after recovery. clusterID := "fixture-cluster" - f.request(admin, http.MethodPost, "/clusters", map[string]any{"name": clusterID, "displayName": "Recovery fixture", "kubeconfig": recoveryKubeconfig(t)}, http.StatusCreated) + f.request(bootstrapAdmin, http.MethodPost, "/clusters", map[string]any{"name": clusterID, "displayName": "Recovery fixture", "kubeconfig": recoveryKubeconfig(t)}, http.StatusCreated) + // Bootstrap grants control-plane administration through the local cluster, + // but remote server access requires an explicit target namespace binding. + // Discovery is safe before that grant; Healthy also proves the asynchronous + // registry has loaded the client needed to accept the binding. + f.waitClusterHealthy(bootstrapAdmin, clusterID) + f.request(bootstrapAdmin, http.MethodPost, "/users/"+adminID+"/bindings", map[string]any{"roleName": "admin", "cluster": clusterID, "namespace": "gameplane-games"}, http.StatusCreated) + // A binding change revokes the affected user's sessions. The new cookie, + // acquired after the grant, must remain live until the backup is taken. + f.request(bootstrapAdmin, http.MethodGet, "/users/me", nil, http.StatusUnauthorized) + bootstrapAdmin.Close() + admin := f.login("initial", "admin", password) + defer admin.Close() roleName := "recovery-reader" f.request(admin, http.MethodPost, "/roles", map[string]any{"name": roleName, "description": "Recovery fixture read only", "permissions": []string{"servers:read"}}, http.StatusCreated) userPassword := recoveryRandom(t) @@ -135,8 +151,8 @@ func TestPanel_OfflineRecovery(t *testing.T) { if userID == "" { t.Fatal("created reader has no ID") } - // Registration persists before the asynchronous registry watcher loads its - // client. Prove dispatch works before assigning a binding to that cluster. + // Prove credential-backed dispatch with the scoped administrator, then + // repeat it using only the reader's explicit target namespace grant. gameName, gameUID := recoveryGame(t, suffix) serversPath := "/servers?cluster=" + clusterID + "&namespace=gameplane-games" f.waitServer(admin, serversPath, gameUID) @@ -153,6 +169,9 @@ func TestPanel_OfflineRecovery(t *testing.T) { } public := &APIClient{BaseURL: admin.BaseURL, HTTP: &http.Client{Timeout: 10 * time.Second}} f.request(public, http.MethodGet, "/shares/"+share.Token, nil, http.StatusOK) + for _, original := range []*APIClient{admin, reader} { + f.request(original, http.MethodGet, "/users/me", nil, http.StatusOK) + } beforeGame := recoveryGameIdentity(t, gameName) beforeCounter := recoveryCounter(t, gameName) report["game_before"] = beforeGame @@ -253,9 +272,12 @@ func TestPanel_OfflineRecovery(t *testing.T) { f.waitPodReady("restored") recovered := f.login("restored", "admin", password) defer recovered.Close() - // Reuse the original cookie jar against the recovered endpoint, without login. - stale := &APIClient{BaseURL: recovered.BaseURL, HTTP: reader.HTTP, CSRF: reader.CSRF} - f.request(stale, http.MethodGet, "/auth/me", nil, http.StatusUnauthorized) + // Reuse both pre-backup cookie jars against the recovered endpoint, without + // login. Each was accepted immediately before the original API stopped. + for _, original := range []*APIClient{admin, reader} { + stale := &APIClient{BaseURL: recovered.BaseURL, HTTP: original.HTTP, CSRF: original.CSRF} + f.request(stale, http.MethodGet, "/users/me", nil, http.StatusUnauthorized) + } public.BaseURL = recovered.BaseURL f.request(public, http.MethodGet, "/shares/"+share.Token, nil, http.StatusNotFound) recoveredReader := f.login("restored", "reader", userPassword) @@ -436,16 +458,45 @@ func recoveryKubeconfig(t *testing.T) string { return string(data) } +func (f *recoveryFixture) waitClusterHealthy(c *APIClient, clusterID string) { + f.t.Helper() + envInstance.Eventually(f.t, 90*time.Second, func() (bool, string) { + resp, data, err := c.DoContext(f.t.Context(), http.MethodGet, "/clusters", nil) + if err != nil { + return false, fmt.Sprintf("cluster discovery request failed (error type=%T; details withheld)", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return false, fmt.Sprintf("cluster discovery: HTTP status=%d (body withheld)", resp.StatusCode) + } + var discovery struct { + Items []struct { + Name string `json:"name"` + Phase string `json:"phase"` + } `json:"items"` + } + if json.Unmarshal(data, &discovery) != nil { + return false, "invalid cluster discovery response (body withheld)" + } + for _, item := range discovery.Items { + if item.Name == clusterID && item.Phase == "Healthy" { + return true, "" + } + } + return false, "registered cluster connection is not Healthy yet" + }) +} + func (f *recoveryFixture) waitServer(c *APIClient, path, uid string) { f.t.Helper() envInstance.Eventually(f.t, 90*time.Second, func() (bool, string) { resp, data, err := c.DoContext(f.t.Context(), http.MethodGet, path, nil) if err != nil { - return false, "waiting for credential-backed cluster dispatch" + return false, fmt.Sprintf("credential-backed cluster dispatch failed (error type=%T; details withheld)", err) } defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { - return false, "waiting for credential-backed cluster dispatch" + return false, fmt.Sprintf("credential-backed cluster dispatch: HTTP status=%d (path/body withheld)", resp.StatusCode) } var list struct { Items []unstructured.Unstructured `json:"items"` diff --git a/test/e2e/specs.md b/test/e2e/specs.md index 091552018..a514b2303 100644 --- a/test/e2e/specs.md +++ b/test/e2e/specs.md @@ -32,7 +32,11 @@ read-only original archive is never changed by invalidation. Actual fixture recovery-point age/RTO and acceptance evidence are uploaded as JSON, with no key, credential, cookie, share token or raw SQL rows. It does not stop the installed panel/operator or claim PostgreSQL PITR coverage. Initial scoped grants wait for -the asynchronous cluster registry to load the connection and permit dispatch. +Healthy cluster discovery so the asynchronous registry has loaded the connection. +The locally bootstrapped administrator receives an explicit fixture-cluster/gameplane-games +grant and logs in again after that change invalidates its session. Both administrator +and reader cookies are accepted before backup and rejected through `/users/me` after +offline recovery; remote server UID and share checks retain their scoped authorization. See `docs/panel-recovery.md`. ### Remote gateway parity acceptance From a5c6d5352df4a6fd629f355d1dfc3bb61a79e4cc Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 23 Oct 2026 00:19:17 -0400 Subject: [PATCH 07/13] test: synchronize closed RCON response fixtures Signed-off-by: Baylem --- agent/internal/rcon/nuclearoption_test.go | 29 +++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/agent/internal/rcon/nuclearoption_test.go b/agent/internal/rcon/nuclearoption_test.go index cb46a4e21..ccde83c53 100644 --- a/agent/internal/rcon/nuclearoption_test.go +++ b/agent/internal/rcon/nuclearoption_test.go @@ -1275,18 +1275,47 @@ type pipeFrameCase struct { forbidErr string // substring the error must NOT contain (e.g. a detail suffix) } +type responseDeadlineConn struct { + net.Conn + deadlineSet chan<- struct{} +} + +func (c *responseDeadlineConn) SetDeadline(deadline time.Time) error { + if err := c.Conn.SetDeadline(deadline); err != nil { + return err + } + select { + case c.deadlineSet <- struct{}{}: + default: + } + return nil +} + // runPipeFrameCases sends each case's raw frame back to the client over net.Pipe // and checks the body or the error. func runPipeFrameCases(t *testing.T, tests []pipeFrameCase) { t.Helper() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + deadlineSet := make(chan struct{}, 1) client := newPipeClient(t, func(remote net.Conn) { if _, _, err := readNuclearOptionRequest(remote); err != nil { return } + if len(tt.frame) == 0 { + // Exec sets its response deadline after sending the request. + // Closing first can fail SetDeadline instead of the status read. + select { + case <-deadlineSet: + case <-t.Context().Done(): + return + } + } writeFrameSegments(remote, tt.frame, tt.oneByte) }) + if len(tt.frame) == 0 { + client.conn = &responseDeadlineConn{Conn: client.conn, deadlineSet: deadlineSet} + } body, err := client.Exec("test-command") if tt.wantErr != "" { From 2e8989ee6dd8c868d28cc24beeaa30c6deddfeea Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 23 Oct 2026 00:26:24 -0400 Subject: [PATCH 08/13] test: clarify closed response deadline synchronization Signed-off-by: Baylem --- agent/internal/rcon/nuclearoption_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/agent/internal/rcon/nuclearoption_test.go b/agent/internal/rcon/nuclearoption_test.go index ccde83c53..d03e76514 100644 --- a/agent/internal/rcon/nuclearoption_test.go +++ b/agent/internal/rcon/nuclearoption_test.go @@ -1303,8 +1303,8 @@ func runPipeFrameCases(t *testing.T, tests []pipeFrameCase) { return } if len(tt.frame) == 0 { - // Exec sets its response deadline after sending the request. - // Closing first can fail SetDeadline instead of the status read. + // Wait until the deadline is installed before closing so the + // intended error comes from the status read, not SetDeadline. select { case <-deadlineSet: case <-t.Context().Done(): From eddf9a2f674f1dc54520bd1604fbd08b38b29096 Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 23 Oct 2026 09:40:26 -0400 Subject: [PATCH 09/13] test: assign recovery fixture ownership before sharing Signed-off-by: Baylem --- test/e2e/panel_recovery_e2e_test.go | 14 +++++++++++--- test/e2e/specs.md | 3 +++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/test/e2e/panel_recovery_e2e_test.go b/test/e2e/panel_recovery_e2e_test.go index 4f9156663..ff8de4025 100644 --- a/test/e2e/panel_recovery_e2e_test.go +++ b/test/e2e/panel_recovery_e2e_test.go @@ -122,8 +122,9 @@ func TestPanel_OfflineRecovery(t *testing.T) { bootstrapAdmin := f.login("initial", "admin", password) defer bootstrapAdmin.Close() adminID := extractUserID(string(f.request(bootstrapAdmin, http.MethodGet, "/users/me", nil, http.StatusOK))) - if adminID == "" { - t.Fatal("bootstrap admin has no ID") + adminUserID, err := strconv.ParseInt(adminID, 10, 64) + if err != nil || adminUserID <= 0 { + t.Fatal("bootstrap admin has no valid ID") } // Register the existing kind cluster using stored, encrypted credentials. @@ -160,7 +161,14 @@ func TestPanel_OfflineRecovery(t *testing.T) { reader := f.login("initial", "reader", userPassword) defer reader.Close() f.waitServer(reader, serversPath, gameUID) - shareBody := f.request(admin, http.MethodPost, "/servers/"+gameName+":shares?cluster="+clusterID+"&namespace=gameplane-games", map[string]any{"canStart": false, "neverExpires": true}, http.StatusOK) + sharePath := "/servers/" + gameName + ":shares?cluster=" + clusterID + "&namespace=gameplane-games" + shareRequest := map[string]any{"canStart": false, "neverExpires": true} + // The game was created directly in Kubernetes, without a panel owner. + // Shares remain owner-only even for scoped administrators. Claim this + // fixture through the authorized transfer endpoint before minting a share. + f.request(admin, http.MethodPost, sharePath, shareRequest, http.StatusForbidden) + f.request(admin, http.MethodPost, "/servers/"+gameName+":transfer?cluster="+clusterID+"&namespace=gameplane-games", map[string]any{"userId": adminUserID}, http.StatusNoContent) + shareBody := f.request(admin, http.MethodPost, sharePath, shareRequest, http.StatusOK) var share struct { Token string `json:"token"` } diff --git a/test/e2e/specs.md b/test/e2e/specs.md index a514b2303..ee200bfa7 100644 --- a/test/e2e/specs.md +++ b/test/e2e/specs.md @@ -37,6 +37,9 @@ The locally bootstrapped administrator receives an explicit fixture-cluster/game grant and logs in again after that change invalidates its session. Both administrator and reader cookies are accepted before backup and rejected through `/users/me` after offline recovery; remote server UID and share checks retain their scoped authorization. +The Kubernetes-created game initially has no panel owner: the drill requires share +creation to fail for the scoped administrator until the authorized transfer endpoint +assigns that administrator ownership, then creates the share used for recovery checks. See `docs/panel-recovery.md`. ### Remote gateway parity acceptance From 7284b9942cda2488cc73158c0b806272a99cef38 Mon Sep 17 00:00:00 2001 From: Baylem Date: Sat, 24 Oct 2026 10:53:47 -0400 Subject: [PATCH 10/13] fix: clarify recovery keys and surface response read errors Signed-off-by: Baylem --- docs/panel-recovery.md | 14 +++-- test/e2e/api_client_response_e2e_test.go | 66 ++++++++++++++++++++++++ test/e2e/buckets.sh | 1 + test/e2e/env.go | 5 +- test/e2e/specs.md | 1 + test/e2e/suite-components.json | 2 +- 6 files changed, 84 insertions(+), 5 deletions(-) create mode 100644 test/e2e/api_client_response_e2e_test.go diff --git a/docs/panel-recovery.md b/docs/panel-recovery.md index 719c00af2..a2539eb9a 100644 --- a/docs/panel-recovery.md +++ b/docs/panel-recovery.md @@ -92,6 +92,9 @@ state and saves through the operational access available during the outage. evidence, protect access to it, and tell users that they must log in again and recreate shares/tokens. Do not run cleanup against the evidence archive. 5. Mount the original standalone management key read-only from separate custody. + For standalone recovery, start with `--panel-key-provisioned` and set + `--panel-key-file` to that mounted original key; this prevents startup from + generating a replacement if the key is missing. In combined mode, restore/validate required Kubernetes management objects and Secrets through the cluster's recovery process instead. Start one API using compatible software and the restored database; inspect migration/startup @@ -118,9 +121,14 @@ state and saves through the operational access available during the outage. Keep the panel stopped. Recover the exact key version paired with the chosen database backup from separate custody. Existing encrypted credentials cannot be -recovered by generating a new random key. Current startup rejects a missing key -with `management encryption key is missing`; an inactive/wrong key for a recorded -key identity reports `management encryption key is inactive`. Other corruption +recovered by generating a new random key. With `--panel-key-provisioned`, startup +rejects a missing file with `provisioned management key is missing` and never +creates a replacement, regardless of database contents. Without provisioned mode, +a database containing management secrets or key-state records requires its key +and a missing file reports `management encryption key is missing`; when neither +is present, startup can generate a new key. Recovery must use provisioned mode +to avoid that fallback. An inactive/wrong key for a recorded key identity reports +`management encryption key is inactive`. Other corruption may fail authentication of encrypted records. None is permission to serve partial management state. diff --git a/test/e2e/api_client_response_e2e_test.go b/test/e2e/api_client_response_e2e_test.go new file mode 100644 index 000000000..fb46e2aa2 --- /dev/null +++ b/test/e2e/api_client_response_e2e_test.go @@ -0,0 +1,66 @@ +//go:build e2e + +package e2e + +import ( + "errors" + "io" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" +) + +// TestAPIClient_DoContextResponseBody keeps transport failures distinct from +// fully received HTTP errors used by recovery's authorization assertions. +func TestAPIClient_DoContextResponseBody(t *testing.T) { + t.Parallel() + for _, test := range []struct { + name string + status int + body string + incomplete bool + }{ + {name: "complete", status: http.StatusOK, body: "complete response"}, + {name: "HTTP error", status: http.StatusServiceUnavailable, body: "unavailable"}, + {name: "truncated", status: http.StatusOK, body: "partial", incomplete: true}, + {name: "truncated HTTP error", status: http.StatusUnauthorized, body: "denied", incomplete: true}, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + length := len(test.body) + if test.incomplete { + length++ // Real HTTP transport observes EOF before Content-Length. + } + w.Header().Set("Content-Length", strconv.Itoa(length)) + w.WriteHeader(test.status) + if _, err := io.WriteString(w, test.body); err != nil { + t.Errorf("write fixture response: %v", err) + } + })) + t.Cleanup(server.Close) + client := &APIClient{BaseURL: server.URL, HTTP: server.Client()} + resp, body, err := client.DoContext(t.Context(), http.MethodGet, "/fixture", nil) + if resp != nil { + _ = resp.Body.Close() + } + if test.incomplete { + if !errors.Is(err, io.ErrUnexpectedEOF) || !strings.Contains(err.Error(), "GET /fixture") { + t.Fatalf("expected contextual wrapped unexpected EOF, got %v", err) + } + if resp != nil || body != nil { + t.Fatalf("incomplete response escaped: response=%v body=%q", resp, body) + } + return + } + if err != nil || resp == nil { + t.Fatalf("complete response failed: response=%v error=%v", resp, err) + } + if resp.StatusCode != test.status || string(body) != test.body { + t.Fatalf("response changed: status=%d body=%q", resp.StatusCode, body) + } + }) + } +} diff --git a/test/e2e/buckets.sh b/test/e2e/buckets.sh index d45d160e8..30c6b9e1c 100755 --- a/test/e2e/buckets.sh +++ b/test/e2e/buckets.sh @@ -397,6 +397,7 @@ EOF # Zero shared panel logins; the fixture has its own login limiter and storage. bucket_recovery() { cat <<'EOF' TestPanel_OfflineRecovery +TestAPIClient_DoContextResponseBody EOF } diff --git a/test/e2e/env.go b/test/e2e/env.go index 4f5cb055d..b306f69c8 100644 --- a/test/e2e/env.go +++ b/test/e2e/env.go @@ -599,7 +599,10 @@ func (c *APIClient) DoContext(ctx context.Context, method, path string, body any return nil, nil, fmt.Errorf("do %s %s: %w", method, path, err) } defer func() { _ = resp.Body.Close() }() - rb, _ := io.ReadAll(resp.Body) + rb, err := io.ReadAll(resp.Body) + if err != nil { + return nil, nil, fmt.Errorf("read response for %s %s: %w", method, path, err) + } return resp, rb, nil } diff --git a/test/e2e/specs.md b/test/e2e/specs.md index ee200bfa7..c67a70df2 100644 --- a/test/e2e/specs.md +++ b/test/e2e/specs.md @@ -147,6 +147,7 @@ This module has no runtime HTTP/RPC surface of its own — its "interface" is th - **`PodIsReady`, `CRDExists`** — typed/dynamic client lookups used by convergence checks. - **`PortForward(t, ns, target, remotePort)`** — retrying wrapper over `tryPortForward`; returns `(localPort, stopFunc)`. - **`APIClient(t, username, password)`** — logs in over a fresh port-forward (retrying through 429s with backoff, since the shared IP shares the login limiter across parallel tests), returns a session+CSRF-aware HTTP client whose `Do`/`Get`/`Post`/`Patch`/`Delete` attach `X-Gameplane-CSRF` on mutating methods. +- **Response reads:** `DoContext` closes response bodies after reading them. Complete responses preserve their status and bytes, including non-2xx statuses. A body read failure returns a wrapped method/path error and nil response/body, so a truncated response cannot satisfy a status or body assertion. `TestAPIClient_DoContextResponseBody` exercises both cases through real local HTTP transport in the `recovery` bucket without panel logins. - **`BootstrapAdmin(t, username, password)`** — runs the API's `bootstrap-admin` subcommand once per test process (`sync.Once`); a second call with a different credential pair fails loudly rather than silently using stale creds, since argon2id hashing inside the running pod is heavy enough that repeating it per test risks an OOM-kill. - **`OCIPush`/`OCIPushFromFixture`** — apply a ConfigMap+Job fixture and wait for it to succeed; guarded by `ociPushMu` at call sites that share the fixed job name. - **`CreateUser(t, admin, role, prefix)`** — creates a uniquely-named user via the admin `APIClient`; caller owns cleanup registration. diff --git a/test/e2e/suite-components.json b/test/e2e/suite-components.json index b6a036037..42090c537 100644 --- a/test/e2e/suite-components.json +++ b/test/e2e/suite-components.json @@ -16,7 +16,7 @@ "test/e2e/suite-components.json" ], "suites": [ - { "tests": "^TestPanel_OfflineRecovery$", "components": ["operator", "api", "agent"], + { "tests": "^(TestPanel_OfflineRecovery|TestAPIClient_DoContextResponseBody)$", "components": ["operator", "api", "agent"], "paths": ["test/e2e/testdata/recovery/"] }, { "tests": "^TestHelmInstall_", "components": ["*"] }, { "tests": "^TestCRD_Validation_", "components": ["operator"] }, From 41eea08a610780247f3d42692e657d033878253f Mon Sep 17 00:00:00 2001 From: Baylem Date: Sun, 25 Oct 2026 19:08:35 -0400 Subject: [PATCH 11/13] test: preserve recovery responses and generic ID parsing Signed-off-by: Baylem --- test/e2e/api_auth_e2e_test.go | 4 ++-- test/e2e/api_client_response_e2e_test.go | 4 ++-- test/e2e/api_session_e2e_test.go | 2 +- test/e2e/env.go | 14 +++++++------- test/e2e/panel_recovery_e2e_test.go | 4 ++-- test/e2e/specs.md | 2 +- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/test/e2e/api_auth_e2e_test.go b/test/e2e/api_auth_e2e_test.go index 510d277ec..fafac66e1 100644 --- a/test/e2e/api_auth_e2e_test.go +++ b/test/e2e/api_auth_e2e_test.go @@ -187,7 +187,7 @@ func TestAPI_AuditEmitsOnMutation(t *testing.T) { } // Extract our id without pulling in encoding/json: the response // shape is small and we only need the leading "id":N field. - id := extractUserID(string(body)) + id := extractIntField(string(body), "id") if id == "" { t.Fatalf("could not parse id from /users/me: %s", string(body)) } @@ -224,7 +224,7 @@ func TestAPI_AuditEmitsOnMutation(t *testing.T) { // the *passed-down* context, which is out of scope for this test. } -// extractUserID has been promoted to env.go so non-test helper code +// extractIntField has been promoted to env.go so non-test helper code // (Env.CreateUser) can use it. Test files in this package still call // it through the package-level scope. diff --git a/test/e2e/api_client_response_e2e_test.go b/test/e2e/api_client_response_e2e_test.go index fb46e2aa2..680d02a26 100644 --- a/test/e2e/api_client_response_e2e_test.go +++ b/test/e2e/api_client_response_e2e_test.go @@ -50,8 +50,8 @@ func TestAPIClient_DoContextResponseBody(t *testing.T) { if !errors.Is(err, io.ErrUnexpectedEOF) || !strings.Contains(err.Error(), "GET /fixture") { t.Fatalf("expected contextual wrapped unexpected EOF, got %v", err) } - if resp != nil || body != nil { - t.Fatalf("incomplete response escaped: response=%v body=%q", resp, body) + if resp == nil || resp.StatusCode != test.status || resp.Header.Get("Content-Length") != strconv.Itoa(len(test.body)+1) || string(body) != test.body { + t.Fatalf("incomplete response lost status, headers or partial bytes: response=%v body=%q", resp, body) } return } diff --git a/test/e2e/api_session_e2e_test.go b/test/e2e/api_session_e2e_test.go index 603b8d69f..5110beba0 100644 --- a/test/e2e/api_session_e2e_test.go +++ b/test/e2e/api_session_e2e_test.go @@ -168,7 +168,7 @@ func TestAPI_AuditPaginationAndFilter(t *testing.T) { t.Fatalf("/users/me %d: %s", getMe.StatusCode, string(body)) } defer func() { _ = getMe.Body.Close() }() - id := extractUserID(string(body)) + id := extractIntField(string(body), "id") if id == "" { t.Fatalf("could not parse id from /users/me: %s", string(body)) } diff --git a/test/e2e/env.go b/test/e2e/env.go index b306f69c8..6ce391482 100644 --- a/test/e2e/env.go +++ b/test/e2e/env.go @@ -601,7 +601,7 @@ func (c *APIClient) DoContext(ctx context.Context, method, path string, body any defer func() { _ = resp.Body.Close() }() rb, err := io.ReadAll(resp.Body) if err != nil { - return nil, nil, fmt.Errorf("read response for %s %s: %w", method, path, err) + return resp, rb, fmt.Errorf("read response for %s %s: %w", method, path, err) } return resp, rb, nil } @@ -699,21 +699,21 @@ func (e *Env) CreateUser(t *testing.T, admin *APIClient, role, prefix string) (u if resp.StatusCode != http.StatusOK { t.Fatalf("CreateUser %s/%s %d: %s", role, prefix, resp.StatusCode, string(body)) } - id = extractUserID(string(body)) + id = extractIntField(string(body), "id") if id == "" { t.Fatalf("CreateUser: could not parse id from response: %s", string(body)) } return username, userPhrase, id } -// extractUserID is a tiny scanner for `"id":` inside a JSON -// blob. It reads the user ID returned by the API without adding a JSON -// decoding dependency to callers that only need that field. +// extractIntField is a tiny scanner for `"":` inside a JSON +// blob. Lets the e2e tests parse the one or two integer fields they +// care about (user id, mostly) without pulling in encoding/json. // // Lives on env.go (not in a _test.go) so non-test helper code like // CreateUser can use it. -func extractUserID(body string) string { - const needle = `"id":` +func extractIntField(body, key string) string { + needle := `"` + key + `":` i := strings.Index(body, needle) if i < 0 { return "" diff --git a/test/e2e/panel_recovery_e2e_test.go b/test/e2e/panel_recovery_e2e_test.go index ff8de4025..f7739a1cd 100644 --- a/test/e2e/panel_recovery_e2e_test.go +++ b/test/e2e/panel_recovery_e2e_test.go @@ -121,7 +121,7 @@ func TestPanel_OfflineRecovery(t *testing.T) { report["http_monitor_positive_control"] = true bootstrapAdmin := f.login("initial", "admin", password) defer bootstrapAdmin.Close() - adminID := extractUserID(string(f.request(bootstrapAdmin, http.MethodGet, "/users/me", nil, http.StatusOK))) + adminID := extractIntField(string(f.request(bootstrapAdmin, http.MethodGet, "/users/me", nil, http.StatusOK)), "id") adminUserID, err := strconv.ParseInt(adminID, 10, 64) if err != nil || adminUserID <= 0 { t.Fatal("bootstrap admin has no valid ID") @@ -148,7 +148,7 @@ func TestPanel_OfflineRecovery(t *testing.T) { f.request(admin, http.MethodPost, "/roles", map[string]any{"name": roleName, "description": "Recovery fixture read only", "permissions": []string{"servers:read"}}, http.StatusCreated) userPassword := recoveryRandom(t) created := f.request(admin, http.MethodPost, "/users", map[string]any{"username": "reader", "password": userPassword, "role": roleName}, http.StatusOK) - userID := extractUserID(string(created)) + userID := extractIntField(string(created), "id") if userID == "" { t.Fatal("created reader has no ID") } diff --git a/test/e2e/specs.md b/test/e2e/specs.md index c67a70df2..1bfb559bb 100644 --- a/test/e2e/specs.md +++ b/test/e2e/specs.md @@ -147,7 +147,7 @@ This module has no runtime HTTP/RPC surface of its own — its "interface" is th - **`PodIsReady`, `CRDExists`** — typed/dynamic client lookups used by convergence checks. - **`PortForward(t, ns, target, remotePort)`** — retrying wrapper over `tryPortForward`; returns `(localPort, stopFunc)`. - **`APIClient(t, username, password)`** — logs in over a fresh port-forward (retrying through 429s with backoff, since the shared IP shares the login limiter across parallel tests), returns a session+CSRF-aware HTTP client whose `Do`/`Get`/`Post`/`Patch`/`Delete` attach `X-Gameplane-CSRF` on mutating methods. -- **Response reads:** `DoContext` closes response bodies after reading them. Complete responses preserve their status and bytes, including non-2xx statuses. A body read failure returns a wrapped method/path error and nil response/body, so a truncated response cannot satisfy a status or body assertion. `TestAPIClient_DoContextResponseBody` exercises both cases through real local HTTP transport in the `recovery` bucket without panel logins. +- **Response reads:** `DoContext` closes response bodies after reading them. Complete responses preserve their status and bytes, including non-2xx statuses. A body read failure returns the response (including status and headers), partial bytes, and a wrapped method/path error. Callers must check that error before treating the body as complete; callers inspecting only status still receive a valid response. `TestAPIClient_DoContextResponseBody` exercises both cases through real local HTTP transport in the `recovery` bucket without panel logins. - **`BootstrapAdmin(t, username, password)`** — runs the API's `bootstrap-admin` subcommand once per test process (`sync.Once`); a second call with a different credential pair fails loudly rather than silently using stale creds, since argon2id hashing inside the running pod is heavy enough that repeating it per test risks an OOM-kill. - **`OCIPush`/`OCIPushFromFixture`** — apply a ConfigMap+Job fixture and wait for it to succeed; guarded by `ociPushMu` at call sites that share the fixed job name. - **`CreateUser(t, admin, role, prefix)`** — creates a uniquely-named user via the admin `APIClient`; caller owns cleanup registration. From 3fd8ade0ea7dfcecf04a94aa2dba7817993eaf16 Mon Sep 17 00:00:00 2001 From: Baylem Date: Tue, 27 Oct 2026 11:55:40 -0400 Subject: [PATCH 12/13] test: exercise generic integer field parsing contract Signed-off-by: Baylem --- test/e2e/api_client_response_e2e_test.go | 20 ++++++++++++++++++++ test/e2e/buckets.sh | 1 + test/e2e/suite-components.json | 2 +- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/test/e2e/api_client_response_e2e_test.go b/test/e2e/api_client_response_e2e_test.go index 680d02a26..10c7b8c4c 100644 --- a/test/e2e/api_client_response_e2e_test.go +++ b/test/e2e/api_client_response_e2e_test.go @@ -12,6 +12,26 @@ import ( "testing" ) +func TestAPIClient_ExtractIntField(t *testing.T) { + t.Parallel() + for _, test := range []struct { + name, body, key, want string + }{ + {name: "user ID", body: `{"id":42,"count":7}`, key: "id", want: "42"}, + {name: "other integer field", body: `{"id":42,"count":7}`, key: "count", want: "7"}, + {name: "zero", body: `{"count":0}`, key: "count", want: "0"}, + {name: "exact key", body: `{"user_id":42}`, key: "id", want: ""}, + {name: "missing", body: `{"id":42}`, key: "count", want: ""}, + {name: "string value", body: `{"id":"42"}`, key: "id", want: ""}, + } { + t.Run(test.name, func(t *testing.T) { + if got := extractIntField(test.body, test.key); got != test.want { + t.Fatalf("extractIntField(%q, %q) = %q, want %q", test.body, test.key, got, test.want) + } + }) + } +} + // TestAPIClient_DoContextResponseBody keeps transport failures distinct from // fully received HTTP errors used by recovery's authorization assertions. func TestAPIClient_DoContextResponseBody(t *testing.T) { diff --git a/test/e2e/buckets.sh b/test/e2e/buckets.sh index aec5f01ae..4e4fefaa7 100755 --- a/test/e2e/buckets.sh +++ b/test/e2e/buckets.sh @@ -399,6 +399,7 @@ EOF bucket_recovery() { cat <<'EOF' TestPanel_OfflineRecovery TestAPIClient_DoContextResponseBody +TestAPIClient_ExtractIntField EOF } diff --git a/test/e2e/suite-components.json b/test/e2e/suite-components.json index b0692f2fd..4dd175d96 100644 --- a/test/e2e/suite-components.json +++ b/test/e2e/suite-components.json @@ -16,7 +16,7 @@ "test/e2e/suite-components.json" ], "suites": [ - { "tests": "^(TestPanel_OfflineRecovery|TestAPIClient_DoContextResponseBody)$", "components": ["operator", "api", "agent"], + { "tests": "^(TestPanel_OfflineRecovery|TestAPIClient_DoContextResponseBody|TestAPIClient_ExtractIntField)$", "components": ["operator", "api", "agent"], "paths": ["test/e2e/testdata/recovery/"] }, { "tests": "^TestManagementPlacementAndDisruption$", "components": ["operator", "api", "agent"] }, From cae07ec1002e78a29a135d2f4bf349f8a456e680 Mon Sep 17 00:00:00 2001 From: Baylem Date: Tue, 27 Oct 2026 12:16:32 -0400 Subject: [PATCH 13/13] test: verify restored API token cleanup in recovery drill Signed-off-by: Baylem --- test/e2e/panel_recovery_e2e_test.go | 38 +++++++++++++++++++---- test/e2e/specs.md | 5 +++ test/e2e/testdata/recovery/maintenance.py | 18 +++++++++-- 3 files changed, 53 insertions(+), 8 deletions(-) diff --git a/test/e2e/panel_recovery_e2e_test.go b/test/e2e/panel_recovery_e2e_test.go index f7739a1cd..96af3936b 100644 --- a/test/e2e/panel_recovery_e2e_test.go +++ b/test/e2e/panel_recovery_e2e_test.go @@ -190,14 +190,20 @@ func TestPanel_OfflineRecovery(t *testing.T) { reader.Close() f.stopPod("initial") report["stage"] = "backup" + // API-token issuance and Bearer authentication are not implemented yet. + // Seed the reserved table offline so removing its restore-time DELETE + // cannot pass merely because the real session/share checks still reject. + if f.maintenance("seed-api-token")["api_tokens"] != float64(1) { + t.Fatal("API-token cleanup fixture was not seeded") + } backup := f.maintenance("backup") report["backup"] = backup backupTime, err := time.Parse(time.RFC3339Nano, backup["backup_utc"].(string)) if err != nil { t.Fatal("invalid archive recovery-point timestamp") } - if backup["sessions"].(float64) < 2 || backup["active_shares"].(float64) < 1 || backup["audit_rows"].(float64) < 1 { - t.Fatal("backup lacks session/share/audit fixture evidence") + if backup["sessions"].(float64) < 2 || backup["api_tokens"] != float64(1) || backup["active_shares"].(float64) < 1 || backup["audit_rows"].(float64) < 1 { + t.Fatal("backup lacks session/API-token/share/audit fixture evidence") } // From here on the archive is mounted read-only even in the maintenance Pod. f.stopPod("maintenance") @@ -282,12 +288,22 @@ func TestPanel_OfflineRecovery(t *testing.T) { defer recovered.Close() // Reuse both pre-backup cookie jars against the recovered endpoint, without // login. Each was accepted immediately before the original API stopped. - for _, original := range []*APIClient{admin, reader} { + authChecks := map[string]bool{"api_token_rows_deleted": restored["api_tokens"] == float64(0)} + for name, original := range map[string]*APIClient{"admin_session_rejected": admin, "reader_session_rejected": reader} { stale := &APIClient{BaseURL: recovered.BaseURL, HTTP: original.HTTP, CSRF: original.CSRF} - f.request(stale, http.MethodGet, "/users/me", nil, http.StatusUnauthorized) + authChecks[name] = f.status(stale, "/users/me") == http.StatusUnauthorized } public.BaseURL = recovered.BaseURL - f.request(public, http.MethodGet, "/shares/"+share.Token, nil, http.StatusNotFound) + authChecks["share_rejected"] = f.status(public, "/shares/"+share.Token) == http.StatusNotFound + bearersInvalidated := true + for _, passed := range authChecks { + bearersInvalidated = bearersInvalidated && passed + } + report["restored_auth_checks"] = authChecks + report["restored_bearers_invalidated"] = bearersInvalidated + if !bearersInvalidated { + t.Fatalf("restored authentication cleanup failed: %v", authChecks) + } recoveredReader := f.login("restored", "reader", userPassword) defer recoveredReader.Close() f.waitServer(recoveredReader, serversPath, gameUID) @@ -324,7 +340,7 @@ func TestPanel_OfflineRecovery(t *testing.T) { e.Eventually(t, 15*time.Second, func() (bool, string) { return recoveryCounter(t, gameName) > beforeCounter, "game counter did not advance" }) - report["game_after"], report["credential_backed_dispatch_verified"], report["restored_bearers_invalidated"] = afterGame, true, true + report["game_after"], report["credential_backed_dispatch_verified"] = afterGame, true report["counter_after"] = recoveryCounter(t, gameName) report["rto_seconds"] = time.Since(lossTime).Seconds() report["rto_scope"] = "loss through restore, auth invalidation, wrong-key trial, and credential-backed acceptance checks" @@ -342,6 +358,16 @@ func recoveryKey(t *testing.T) []byte { func recoveryRandom(t *testing.T) string { return hex.EncodeToString(recoveryKey(t)) } +func (f *recoveryFixture) status(c *APIClient, path string) int { + f.t.Helper() + resp, _, err := c.DoContext(f.t.Context(), http.MethodGet, path, nil) + if err != nil { + f.t.Fatal("recovery HTTP status request failed (details withheld)") + } + defer func() { _ = resp.Body.Close() }() + return resp.StatusCode +} + func (f *recoveryFixture) request(c *APIClient, method, path string, body any, want int) []byte { f.t.Helper() resp, data, err := c.DoContext(f.t.Context(), method, path, body) diff --git a/test/e2e/specs.md b/test/e2e/specs.md index c5c0dbf92..8ad73f408 100644 --- a/test/e2e/specs.md +++ b/test/e2e/specs.md @@ -38,6 +38,11 @@ The locally bootstrapped administrator receives an explicit fixture-cluster/game grant and logs in again after that change invalidates its session. Both administrator and reader cookies are accepted before backup and rejected through `/users/me` after offline recovery; remote server UID and share checks retain their scoped authorization. +Because API-token issuance and Bearer authentication are not implemented, the drill +seeds one row in the reserved `api_tokens` table offline after stopping the API. +It requires that row in the archive and zero API-token rows after restoration. +The JSON bearer-invalidation result combines that database check with the actual +administrator/reader session and share-link HTTP rejection results. The Kubernetes-created game initially has no panel owner: the drill requires share creation to fail for the scoped administrator until the authorized transfer endpoint assigns that administrator ownership, then creates the share used for recovery checks. diff --git a/test/e2e/testdata/recovery/maintenance.py b/test/e2e/testdata/recovery/maintenance.py index 9c7d65711..41c020025 100644 --- a/test/e2e/testdata/recovery/maintenance.py +++ b/test/e2e/testdata/recovery/maintenance.py @@ -8,6 +8,7 @@ import hashlib import json import pathlib +import secrets import sqlite3 import sys import time @@ -34,6 +35,7 @@ def inventory(db): "audit_digest": hashlib.sha256(json.dumps(rows, separators=(",", ":")).encode()).hexdigest(), "users": db.execute("SELECT COUNT(*) FROM users").fetchone()[0], "sessions": db.execute("SELECT COUNT(*) FROM sessions").fetchone()[0], + "api_tokens": db.execute("SELECT COUNT(*) FROM api_tokens").fetchone()[0], "active_shares": db.execute("SELECT COUNT(*) FROM share_links WHERE revoked_at IS NULL").fetchone()[0], } @@ -44,7 +46,19 @@ def remove_working_database(): op = sys.argv[1] -if op == "backup": +if op == "seed-api-token": + # The API currently authenticates only session cookies; api_tokens is a + # reserved table with no issuance/Bearer-auth route. Seed its cleanup + # coverage offline, after the API stops, without exporting the token. + assert DATA.is_file() and not ARCHIVE.exists(), "seed before backup" + with closing(sqlite3.connect(DATA)) as db: + user = db.execute("SELECT id FROM users WHERE username=?", ("admin",)).fetchone() + assert user is not None, "fixture administrator missing" + db.execute("INSERT INTO api_tokens(token, user_id, name) VALUES (?, ?, ?)", + (secrets.token_urlsafe(32), user[0], "recovery-fixture")) + db.commit() + result = {"api_tokens": db.execute("SELECT COUNT(*) FROM api_tokens").fetchone()[0]} +elif op == "backup": assert not ARCHIVE.exists(), "archive must not be overwritten" with closing(readonly(DATA)) as source, closing(sqlite3.connect(ARCHIVE)) as dest: source.backup(dest) @@ -73,7 +87,7 @@ def remove_working_database(): dest.commit() result = inventory(dest) assert result["audit_digest"] == original["audit_digest"] - assert result["sessions"] == 0 and result["active_shares"] == 0 + assert result["sessions"] == 0 and result["api_tokens"] == 0 and result["active_shares"] == 0 DATA.chmod(0o600) assert digest(ARCHIVE) == before result["archive_digest"] = before