From 65d1beb41b7b2a00a9f26d69c9a6b1cd49b957cd Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 21 Oct 2026 12:09:34 -0400 Subject: [PATCH 01/14] fix(tunnel): isolate frp identities and report relay health Signed-off-by: Baylem --- .github/workflows/ci.yaml | 8 +- .../crds/gameplane.local_backups.yaml | 2 +- .../crds/gameplane.local_backupschedules.yaml | 2 +- .../crds/gameplane.local_clusters.yaml | 2 +- .../crds/gameplane.local_gameservers.yaml | 17 +- .../crds/gameplane.local_gametemplates.yaml | 2 +- .../crds/gameplane.local_modules.yaml | 2 +- .../crds/gameplane.local_modulesources.yaml | 2 +- .../crds/gameplane.local_networkcaptures.yaml | 2 +- .../crds/gameplane.local_restores.yaml | 2 +- .../gameplane.local_backups.yaml | 2 +- .../gameplane.local_backupschedules.yaml | 2 +- .../gameplane.local_clusters.yaml | 2 +- .../gameplane.local_gameservers.yaml | 17 +- .../gameplane.local_gametemplates.yaml | 2 +- .../gameplane.local_modules.yaml | 2 +- .../gameplane.local_modulesources.yaml | 2 +- .../gameplane.local_networkcaptures.yaml | 2 +- .../gameplane.local_restores.yaml | 2 +- docs/relay-health.md | 110 +++ docs/tailscale-forwarding-integration.md | 150 ++++ docs/tunnels.md | 41 +- go.work | 2 +- go.work.sum | 9 +- i18n/locales/en/app.json | 12 + operator/api/v1alpha1/gameserver_types.go | 15 +- .../crd/gameplane.local_gameservers.yaml | 15 +- operator/hack/condition-ids/main.go | 3 + .../gameserver_relay_health_test.go | 106 +++ .../internal/controller/gameserver_status.go | 99 ++- .../controller/gameserver_tailscale_test.go | 48 ++ .../internal/controller/gameserver_tunnel.go | 53 +- .../gameserver_tunnel_envtest_test.go | 26 + operator/specs.md | 3 + .../OPEN-DECISIONS.md | 6 + test/e2e/optional_components_e2e_test.go | 12 +- test/e2e/relay_health_e2e_test.go | 280 ++++++++ test/e2e/specs.md | 2 + tunnel/Dockerfile.frp | 7 +- tunnel/Dockerfile.playit | 3 +- tunnel/Dockerfile.tailscale | 10 +- tunnel/go.mod | 50 +- tunnel/go.sum | 236 +++++++ tunnel/main.go | 542 ++------------- tunnel/main_test.go | 650 +----------------- tunnel/playit_reporter.go | 4 +- tunnel/relay_health.go | 272 ++++++++ tunnel/relay_health_test.go | 223 ++++++ tunnel/specs.md | 119 ++-- tunnel/tailscale.go | 218 ++++++ tunnel/tailscale_forward.go | 253 +++++++ tunnel/tailscale_forward_test.go | 140 ++++ tunnel/tailscale_test.go | 249 +++++++ 53 files changed, 2734 insertions(+), 1308 deletions(-) create mode 100644 docs/relay-health.md create mode 100644 docs/tailscale-forwarding-integration.md create mode 100644 operator/internal/controller/gameserver_relay_health_test.go create mode 100644 operator/internal/controller/gameserver_tailscale_test.go create mode 100644 test/e2e/relay_health_e2e_test.go create mode 100644 tunnel/go.sum create mode 100644 tunnel/relay_health.go create mode 100644 tunnel/relay_health_test.go create mode 100644 tunnel/tailscale.go create mode 100644 tunnel/tailscale_forward.go create mode 100644 tunnel/tailscale_forward_test.go create mode 100644 tunnel/tailscale_test.go diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a902708f0..b1bec0211 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1704,8 +1704,8 @@ jobs: job_timeout: 75 # Boots a kind cluster, installs the chart, runs the bucket's suite. - bucket: optional-components parallel: 1 - test_timeout: 20m # one lifecycle test: helm upgrade, then mcp, syslog, frp tunnel and core checks in order - job_timeout: 60 # Boots a kind cluster, installs the chart, runs the bucket's suite. + test_timeout: 35m # lifecycle plus two-GameServer TCP/UDP relay identity, outage and recovery checks + job_timeout: 75 # Boots a kind cluster, installs the chart, runs the bucket's suite. env: CLUSTER: gameplane-e2e CTX: kind-gameplane-e2e @@ -1805,8 +1805,8 @@ jobs: job_timeout: 75 # Boots a kind cluster, installs the chart, runs the bucket's suite. - bucket: optional-components parallel: 1 - test_timeout: 20m # one lifecycle test: helm upgrade, then mcp, syslog, frp tunnel and core checks in order - job_timeout: 60 # Boots a kind cluster, installs the chart, runs the bucket's suite. + test_timeout: 35m # lifecycle plus two-GameServer TCP/UDP relay identity, outage and recovery checks + job_timeout: 75 # Boots a kind cluster, installs the chart, runs the bucket's suite. env: CLUSTER: gameplane-e2e CTX: kind-gameplane-e2e diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml index d902ea995..8b0b52abe 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: backups.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml index d85406090..513c2dbca 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: backupschedules.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml index 4d6aadcbb..b19459f33 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: clusters.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml index 95178e56e..fba969dab 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: gameservers.gameplane.local spec: @@ -1546,15 +1546,12 @@ spec: tags: description: |- Tags are Tailscale ACL tags requested for the device when it - registers, via `tailscale up --advertise-tags` (tailscaled's - declarative config has no field for tags, and changing the tags of a - registered node needs a fresh login). Each entry is "tag:" or a bare - name, which gets the "tag:" prefix. The tailnet ACL must grant - tagOwners for these tags to the auth key's owner; if the request is - refused, the tunnel pod logs the error and the tunnel runs untagged. - If any entry is invalid, the whole list is logged and ignored and the - device registers untagged. See - tunnel/specs.md. + registers through embedded tsnet. Each entry is "tag:" or a + bare name, which gets the "tag:" prefix. The tailnet policy must grant + tagOwners to the auth key's owner. Invalid or refused tags keep the + tunnel not ready; it never silently registers without requested tags. + Changing tags replaces the Pod and requires a reusable auth key. + See tunnel/specs.md for identity persistence and registration details. items: type: string maxItems: 8 diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml index e5e3e1e3b..9d2de5bda 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: gametemplates.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml index 5ade28cee..b0e1c69b5 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: modules.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml index 6c5dcbf36..953e12936 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: modulesources.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml index cc113e7e8..ef8e19316 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: networkcaptures.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml index b77cd5c25..14d0478f0 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: restores.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_backups.yaml b/charts/gameplane/crd-manifests/gameplane.local_backups.yaml index d902ea995..8b0b52abe 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_backups.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_backups.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: backups.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml b/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml index d85406090..513c2dbca 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: backupschedules.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml b/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml index 4d6aadcbb..b19459f33 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: clusters.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml b/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml index 95178e56e..fba969dab 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: gameservers.gameplane.local spec: @@ -1546,15 +1546,12 @@ spec: tags: description: |- Tags are Tailscale ACL tags requested for the device when it - registers, via `tailscale up --advertise-tags` (tailscaled's - declarative config has no field for tags, and changing the tags of a - registered node needs a fresh login). Each entry is "tag:" or a bare - name, which gets the "tag:" prefix. The tailnet ACL must grant - tagOwners for these tags to the auth key's owner; if the request is - refused, the tunnel pod logs the error and the tunnel runs untagged. - If any entry is invalid, the whole list is logged and ignored and the - device registers untagged. See - tunnel/specs.md. + registers through embedded tsnet. Each entry is "tag:" or a + bare name, which gets the "tag:" prefix. The tailnet policy must grant + tagOwners to the auth key's owner. Invalid or refused tags keep the + tunnel not ready; it never silently registers without requested tags. + Changing tags replaces the Pod and requires a reusable auth key. + See tunnel/specs.md for identity persistence and registration details. items: type: string maxItems: 8 diff --git a/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml b/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml index e5e3e1e3b..9d2de5bda 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: gametemplates.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_modules.yaml b/charts/gameplane/crd-manifests/gameplane.local_modules.yaml index 5ade28cee..b0e1c69b5 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_modules.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_modules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: modules.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml b/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml index 6c5dcbf36..953e12936 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: modulesources.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml b/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml index cc113e7e8..ef8e19316 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: networkcaptures.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_restores.yaml b/charts/gameplane/crd-manifests/gameplane.local_restores.yaml index b77cd5c25..14d0478f0 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_restores.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_restores.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: restores.gameplane.local spec: diff --git a/docs/relay-health.md b/docs/relay-health.md new file mode 100644 index 000000000..6be7a1658 --- /dev/null +++ b/docs/relay-health.md @@ -0,0 +1,110 @@ +# Relay readiness and registration identity + +The frp/Playit wrapper's process health and its provider connection are separate. +On Pod TCP port 8091, `/healthz` reports that the wrapper's HTTP server is alive, +`/readyz` returns 200 only when the provider connection and configured forwards +are confirmed, and `/status` returns a small JSON state without credentials or +provider response text. The operator installs a startup probe on `/healthz` and +a readiness probe on `/readyz`. An external relay outage does not trigger a +liveness restart of the wrapper. + +Provider checks run every two seconds with a two-second query deadline. Local +observations expire after ten seconds, and child exit immediately clears them. +The operator also requires the current Deployment generation, updated replicas +and the expected readiness probe; an old Pod without the probe cannot satisfy +the new readiness contract. Configured addresses remain visible while the +connection is unconfirmed, but `TunnelReady` is not true. + +## frp + +The generated frpc TOML sets `user` to the immutable GameServer UID. The wire +registration is `.`; the GameTemplate's port names and +`remotePorts[].name` do not change. Pod and container restarts reuse the UID. +Deleting and recreating a GameServer creates a new identity, including when its +namespace and name are reused. Distinct clusters' UIDs also separate their +registrations on a shared frps. + +frpc's authenticated status API listens only on `127.0.0.1:7400`. It uses the +mounted token as its local admin password; it is never exposed by a Service or +forwarded from the public relay. The wrapper does not use an environment HTTP +proxy or follow redirects for this request. Every configured TCP/UDP proxy must +appear with the matching name and protocol, a `running` status and no error. +Authentication failure, an occupied registration, absent proxy or unavailable +local API cannot produce readiness. This confirms the relay registrations, not +the health of a sleeping game backend. + +Application heartbeat settings are five seconds between requests and fifteen +seconds before a silent control connection is closed; TCP multiplex keepalive +also uses five seconds. Allow the provider detection interval, local check and +Kubernetes probe/reconcile intervals for an outage to reach the dashboard. + +### Upgrade and replacement + +Upgrade the operator and tunnel image together. The new image requires +`GAMESERVER_UID`, so a mixed old-operator/new-image installation fails closed. +Existing unprefixed proxy names change to UID-prefixed names. Stop the old +client, wait for frps to unregister its old proxies and release their public +ports, or remove those registrations using the frps administrator's normal +procedure. Readiness stays false while a port is still occupied. Do not remove +an unrelated GameServer's registration to make an upgrade succeed. + +The operator uses one replica and the Recreate strategy for relay Deployments. +It never deliberately rolls out two clients with the same provider identity. +Replacement has a short forwarding interruption while the old connection +closes and the new one registers. When a deleted old Pod's node is unreachable, +wait for frps to expire its old session before expecting the replacement ready. +Do not manually scale a relay Deployment above one replica. + +## Playit connection evidence + +The pinned Playit v1.0.10 IPC v2 contract exposes lifecycle and address +assignment, but no authenticated relay heartbeat or connection timestamp. +Its daemon caches `Running` after a successful API refresh and leaves that +cache unchanged when later refreshes fail. Receiving another `get_state` +response therefore cannot prove that a relay is connected or refresh a health +lease. Process age and the presence of an assigned address do not prove it +either. + +Gameplane keeps reporting assigned addresses, but Playit's `/readyz` returns +503 and `TunnelReady` reports `RelayConnectionUnknown`; before assignment it +reports `AwaitingAddress`. `/healthz` remains successful, the child keeps running +and forwarding when it can, and Kubernetes does not restart it solely for this +unknown connection state. The JSON status distinguishes known assignment from +unknown connection. This is a deliberate limitation of the pinned provider, +not a claim that every Playit tunnel is disconnected. + +Positive Playit readiness requires a future provider interface that exposes +fresh, authenticated relay connection evidence, or a separately designed +end-to-end probe. The relevant upstream definitions are +[the IPC model](https://github.com/playit-cloud/playit-agent/blob/v1.0.10/packages/playit-ipc/src/model.rs) +and [the daemon's cached lifecycle publisher](https://github.com/playit-cloud/playit-agent/blob/v1.0.10/packages/playitd/src/daemon.rs). + +## Retry and resource policy + +Unexpected child exits, including clean exits, wait 1, 2, 4, 8 seconds and so on, +up to 300 seconds, before another attempt. Cancellation interrupts the wait. +There is no jitter, as requested by the maintainer. Retry escalation resets only +after one uninterrupted minute of confirmed provider readiness. A failed or +stale observation breaks that interval. A long-lived disconnected process does +not reset it. Playit's unsupported connection evidence cannot reset it either. +The embedded Tailscale loop uses the same interval, fed by confirmed online +tailnet status and active forwarding listeners rather than process lifetime. + +Resources use the existing `operator.tunnelResources` Helm value and +`--tunnel-resources` operator flag from the workload-resource policy. Omitted or +null configuration defaults to requests of 25m CPU and 64Mi memory, with no +limits. A non-null empty object explicitly clears defaults; supplied requests +and limits replace them. These are scheduling reservations, not a fleet +capacity guarantee. Readiness and replacement changes preserve this contract. + +## Regression coverage + +The CI optional-components lifecycle test runs two GameServers from the same +TCP/UDP echo template against one real frps, with different remote ports. An +independent client Pod checks replies identify the intended backend through +both protocols. It also covers registration rejection, replacement isolation, +invalid credentials, a shared relay outage and recovery, and credential-free +logs. Unit tests cover UID validation/stability, namespace/recreation identity, +status decoding, stale expiry, bounded retries/cancellation, healthy-interval +reset, old-deployment readiness, Playit's cached-running limitation, and +resource override removal. These regression tests are executed in CI. diff --git a/docs/tailscale-forwarding-integration.md b/docs/tailscale-forwarding-integration.md new file mode 100644 index 000000000..2874177b5 --- /dev/null +++ b/docs/tailscale-forwarding-integration.md @@ -0,0 +1,150 @@ +# Tailscale forwarding integration + +This procedure needs a disposable Kubernetes installation and a disposable +tailnet with two client devices. It creates real Tailscale devices and sends real +TCP and UDP traffic. It is an integration environment for #722, not evidence +that a live tailnet run has passed. Ordinary CI runs the local socket and +controller regressions without provider credentials. Follow repository policy: +execute this in an authorized integration environment, not a local test run. + +Use the operator and tunnel image built from the same revision. Record their +digests, the revision, client versions, tailnet device IDs, conditions and traffic +results. Do not record auth keys, login URLs or private node state. + +## Provision the disposable environment + +Enable MagicDNS. Create a reusable auth key permitted to request +`tag:gameplane-test`. Configure a tailnet policy with no broad pre-existing allow +rules: permit the approved client to reach that tag on TCP 25565 and UDP 19132. +The second client must have no grant to the tag. For example, replacing the +example email with the approved test user's identity: + +```json +{ + "tagOwners": {"tag:gameplane-test": ["autogroup:admin"]}, + "grants": [{ + "src": ["approved-test-user@example.com"], + "dst": ["tag:gameplane-test"], + "ip": ["tcp:25565", "udp:19132"] + }] +} +``` + +Install this fixture in the disposable Gameplane cluster. Its third TCP port is +deliberately unadvertised. The backend prefixes replies so replacing it can be +distinguished from continuing to reach an old process. + +```yaml +apiVersion: gameplane.local/v1alpha1 +kind: GameTemplate +metadata: + name: tailnet-echo +spec: + displayName: Tailnet echo integration + game: tailnet-echo + version: "1" + image: python:3.14-alpine + consoleMode: none + command: [python, -u, -c] + args: + - | + import socketserver, threading + class TCP(socketserver.BaseRequestHandler): + def handle(self): + while True: + data = self.request.recv(65536) + if not data: return + self.request.sendall(b'backend-v1:' + data) + class UDP(socketserver.BaseRequestHandler): + def handle(self): + data, sock = self.request + sock.sendto(b'backend-v1:' + data, self.client_address) + for port in (25565, 25575): + server = socketserver.ThreadingTCPServer(('0.0.0.0', port), TCP) + threading.Thread(target=server.serve_forever, daemon=True).start() + socketserver.ThreadingUDPServer(('0.0.0.0', 19132), UDP).serve_forever() + ports: + - {name: game, containerPort: 25565, protocol: TCP, advertise: true} + - {name: query, containerPort: 19132, protocol: UDP, advertise: true} + - {name: private, containerPort: 25575, protocol: TCP, advertise: false} +--- +apiVersion: gameplane.local/v1alpha1 +kind: GameServer +metadata: + name: tailnet-echo + namespace: gameplane-games +spec: + templateRef: {name: tailnet-echo} + networking: + expose: ClusterIP + tunnel: + enabled: true + provider: tailscale + credentialsSecretRef: {name: tailnet-echo-tunnel-auth} + tailscale: + hostname: gameplane-forward-test + tags: ["tag:gameplane-test"] +``` + +Provide the reusable auth key through the dashboard's tunnel credential form or +the credential API; these create the required GameServer ownership reference. +Wait for the tunnel Deployment's ready replica and `TunnelReady=True`. Confirm +the admin console shows the requested tag and exact hostname, with no advertised +subnet routes. Record the node ID and tailnet IPs. Remove conflicting stale test +devices if the hostname has acquired a suffix; that state must remain not ready. + +## Exercise the datapath + +On the approved tailnet client, run this with the fixture's MagicDNS name: + +```python +import socket +host = 'gameplane-forward-test' +for kind, port in [(socket.SOCK_STREAM, 25565), (socket.SOCK_DGRAM, 19132)]: + with socket.socket(socket.AF_INET, kind) as client: + client.settimeout(5) + client.connect((socket.gethostbyname(host), port)) + client.sendall(b'game-payload') + assert client.recv(65536) == b'backend-v1:game-payload' +``` + +Repeat with the node's IPv6 tailnet address and `AF_INET6` when the environment +supports IPv6. Run two UDP clients concurrently with distinct payloads; each +must receive only its own reply. TCP 25575 and any other unadvertised port must +fail even from the approved client. Repeat the advertised-port probes from the +denied client; both must time out/fail while the tunnel remains ready. Removing +the approved grant must block new flows too; restore it before continuing. + +Create an unrelated echo Service with a distinguishable prefix in the cluster. +It must not be reachable through this tailnet device on any port, and no route +to its ClusterIP or the cluster's Service/Pod CIDRs may appear on either client. +The local regression also rejects unrelated backing DNS names before listeners +are opened; no incoming packet contains a selectable destination. + +Change the fixture backend prefix to `backend-v2:` and wait for its game Pod +rollout. New TCP connections must receive v2. Stop UDP traffic for over 30 seconds +to expire the old session, then send again and require v2. In the disposable +cluster, delete only the fixture's game Service and wait for the operator to +recreate it; repeat both probes. Existing connections may fail during replacement; +new sessions must resolve the Service again and recover without replacing the +tunnel Pod. + +## Identity, authentication and shutdown outcomes + +| Change | Required outcome | +|---|---| +| Restart the tunnel **container** while retaining the Pod and EmptyDir (use the integration runner's container runtime controls) | Same node ID, hostname and tags reconnect; TCP/UDP recover. | +| Delete the tunnel **Pod**, remove its stale device from the admin console, allow replacement | New identity registers using the reusable key; exact hostname/tags recover and both protocols work. | +| Replace Pod with an expired/revoked key in its credential Secret | Registration never makes the Pod ready and no Tailscale endpoint is advertised. Store a valid reusable key, replace the Pod and require recovery. | +| Request a tag not permitted by tagOwners | Not ready; there must be no untagged fallback device serving traffic. Restore permitted tags and repair credentials/device state, then verify recovery. | +| Rename an existing device to collide with the configured hostname | A suffix assigned to the replacement device cannot make Gameplane advertise the wrong hostname as ready. Resolve the collision and re-register. | +| Disconnect the node from the tailnet/control plane | Readiness turns false after the status/probe intervals; restore connectivity and require recovery. Do not use client ACL denial as a simulation of a disconnected server. | +| Delete/terminate tunnel Pod during active TCP and UDP traffic | Listeners and sessions close, worker shutdown does not hang, and clients reconnect only after the replacement is ready. | + +Remove the GameServer and fixture template, revoke the disposable auth key, and +delete all test devices from the tailnet. Retain only the sanitized result record. + +The implementation uses the official [tsnet APIs](https://tailscale.com/docs/reference/tsnet-server-api). +Its Go 1.26.10 minimum and Go 1.27.3 image builders include the `crypto/x509` +security fix described in the [Go release history](https://go.dev/doc/devel/release) +and [Tailscale security bulletin TS-2026-013](https://tailscale.com/security-bulletins#ts-2026-013). diff --git a/docs/tunnels.md b/docs/tunnels.md index b8ce3fd2a..504d98d96 100644 --- a/docs/tunnels.md +++ b/docs/tunnels.md @@ -197,21 +197,27 @@ specify in `spec.networking.tunnel.tailscale.hostname` (or defaults to the GameServer name if left empty). Port 25565 is available to devices on your tailnet. -> **`tags` requires an ACL grant.** `spec.networking.tunnel.tailscale.tags` -> is requested when the device registers with your tailnet: the tunnel pod -> runs `tailscale up --advertise-tags=` once, over the local daemon's -> socket. Tailscale's headless daemon config has no field for tags, and -> changing the tags of a registered device needs a fresh login (see -> `tunnel/specs.md`). The tailnet's ACL must grant `tagOwners` for the -> requested tags to the auth key's owner. If it doesn't, Tailscale refuses -> the request and the tunnel pod logs the error. If the device is not -> running afterwards, the pod tries once more without tags. That untagged -> attempt can fail too (for example with a single-use auth key), in which -> case the error is logged and the device stays logged out. Configure `tagOwners` in your tailnet's ACL before setting -> `tags` here. Each tag is `tag:` or a bare ``, where the name -> starts with a letter and holds only letters, digits and `-`. If any tag is -> invalid, the whole list is logged and ignored and the device registers -> untagged. +The embedded Tailscale node forwards every advertised TCP and UDP port to this +GameServer's backing Service. It does not provide a subnet route or expose +unadvertised ports. `TunnelReady` requires an online device and active forwarding +listeners. Tailnet grants still govern each player's access; readiness is not a +test of an individual grant or of the sleeping game's application health. + +Requested tags require a matching `tagOwners` policy. A bare tag name receives +the `tag:` prefix. Invalid or denied tags keep the tunnel not ready; it never +silently falls back to an untagged identity. + +Container restarts retain identity in a Pod volume. Replacing the Pod, changing +its tags, or upgrading from the older tailscaled implementation re-registers the +device and requires a valid reusable auth key. Remove stale devices from the +tailnet admin console so the configured MagicDNS hostname remains available. +If Tailscale assigns a suffixed name because of a conflict, Gameplane keeps the +advertised endpoint not ready until the conflict is resolved. Enable MagicDNS. + +Upgrade the operator and tunnel images together: the forwarding contract now +carries each port's TCP/UDP protocol. See the +[disposable tailnet integration procedure](tailscale-forwarding-integration.md) +for traffic, access-policy and restart checks. The `credentialsSecretRef.name` is the name that the dashboard or API write after you provide the credential. @@ -503,3 +509,8 @@ changes. the same security practices as you would for a public-facing LoadBalancer (e.g., disable admin consoles, run security updates, restrict in-game permissions). + +See [relay readiness and registration identity](relay-health.md) for the frp +UID migration, replacement policy, health probes, retry reset and the pinned +Playit version's connection-status limitation. An assigned address alone does +not make a relay ready. diff --git a/go.work b/go.work index fa6b1bb44..7ccf88062 100644 --- a/go.work +++ b/go.work @@ -1,4 +1,4 @@ -go 1.26.0 +go 1.26.10 use ( ./agent diff --git a/go.work.sum b/go.work.sum index 2da1910fe..1a92b2fe8 100644 --- a/go.work.sum +++ b/go.work.sum @@ -148,6 +148,7 @@ github.com/cavaliercoder/badio v0.0.0-20160213150051-ce5280129e9e/go.mod h1:V284 github.com/cavaliercoder/go-rpm v0.0.0-20200122174316-8cb9fd9c31a8/go.mod h1:AZIh1CCnMrcVm6afFf96PBvE2MRpWFco91z8ObJtgDY= github.com/cavaliergopher/cpio v1.0.1/go.mod h1:pBdaqQjnvXxdS/6CvNDwIANIFSP0xRKI16PX4xejRQc= github.com/ccojocar/zxcvbn-go v1.0.4/go.mod h1:3GxGX+rHmueTUMvm5ium7irpyjmm7ikxYFOSJB21Das= +github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= github.com/chainguard-dev/clog v1.7.0/go.mod h1:4+WFhRMsGH79etYXY3plYdp+tCz/KCkU8fAr0HoaPvs= github.com/chainguard-dev/clog v1.8.0/go.mod h1:5MQOZi+Iu7fV7GcJG8ag8rCB5elEOpqRMKEASgnGVdo= @@ -218,6 +219,8 @@ github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJP github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= github.com/ettle/strcase v0.2.0/go.mod h1:DajmHElDSaX76ITe3/VHVyMin4LWSJN5Z909Wp+ED1A= +github.com/evanphx/json-patch v5.9.11+incompatible h1:ixHHqfcGvxhWkniF1tWxBHA0yb4Z+d1UQi45df52xW8= +github.com/evanphx/json-patch v5.9.11+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= github.com/firefart/nonamedreturns v1.0.6/go.mod h1:R8NisJnSIpvPWheCq0mNRXJok6D8h7fagJTF8EMEwCo= github.com/flynn/go-docopt v0.0.0-20140912013429-f6dd2ebbb31e/go.mod h1:HyVoz1Mz5Co8TFO8EupIdlcpwShBmY98dkT2xeHkvEI= @@ -306,7 +309,6 @@ github.com/golangci/rowserrcheck v0.0.0-20260419091836-c5f79b8a11ba/go.mod h1:sC github.com/golangci/swaggoswag v0.0.0-20250504205917-77f2aca3143e/go.mod h1:Vrn4B5oR9qRwM+f54koyeH3yzphlecwERs0el27Fr/s= github.com/golangci/unconvert v0.0.0-20250410112200-a129a6e6413e/go.mod h1:h+wZwLjUTJnm/P2rwlbJdRPZXOzaT36/FwnPnY2inzc= github.com/google/addlicense v1.1.1/go.mod h1:Sm/DHu7Jk+T5miFHHehdIjbi4M5+dJDRS3Cq0rncIxA= -github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/cel-go v0.26.0/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= @@ -405,6 +407,8 @@ github.com/maratori/testableexamples v1.0.1/go.mod h1:XE2F/nQs7B9N08JgyRmdGjYVGq github.com/maratori/testpackage v1.1.2/go.mod h1:8F24GdVDFW5Ew43Et02jamrVMNXLUNaOynhDssITGfc= github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/matoous/godox v1.1.0/go.mod h1:jgE/3fUXiTurkdHOLT5WEkThTSuE7yxHv5iWPa80afs= +github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= @@ -721,6 +725,7 @@ golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= @@ -756,7 +761,6 @@ golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= -golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7 h1:EBZoQjiKKPaLbPrbpssUfuHtwM6KV/vb4U85g/cigFY= @@ -809,7 +813,6 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/go-jose/go-jose.v2 v2.6.3/go.mod h1:zzZDPkNNw/c9IE7Z9jr11mBZQhKQTMzoEEIoEdZlFBI= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= -honnef.co/go/tools v0.7.0/go.mod h1:pm29oPxeP3P82ISxZDgIYeOaf9ta6Pi0EWvCFoLG2vc= k8s.io/gengo/v2 v2.0.0-20250604051438-85fd79dbfd9f/go.mod h1:EJykeLsmFC60UQbYJezXkEsG2FLrt0GPNkU5iK5GWxU= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= diff --git a/i18n/locales/en/app.json b/i18n/locales/en/app.json index 059470fd2..a39c85d1a 100644 --- a/i18n/locales/en/app.json +++ b/i18n/locales/en/app.json @@ -567,6 +567,10 @@ "defaultMessage": "These settings apply to the backing Service, not to tunnel traffic: {settings}.", "description": "Informational condition when hostname, source ranges or node ports are set on a server with a tunnel. settings is a comma-separated list of field names." }, + "condition.tunnelready.awaitingaddress": { + "defaultMessage": "Waiting for the provider to assign a tunnel endpoint; the relay connection is unconfirmed.", + "description": "TunnelReady condition before Playit assigns an endpoint." + }, "condition.tunnelready.deploymentnotready": { "defaultMessage": "Waiting for the tunnel deployment to become ready.", "description": "TunnelReady condition while the tunnel pod is not ready." @@ -587,6 +591,10 @@ "defaultMessage": "{provider, select, playit {The tunnel deployment is ready. Waiting for the playit endpoint assignment.} tailscale {The tunnel deployment is ready and authenticated.} other {The tunnel deployment is ready and all ports are mapped.}}", "description": "TunnelReady condition when the tunnel works." }, + "condition.tunnelready.relayconnectionunknown": { + "defaultMessage": "The relay connection cannot be confirmed. Check the provider's connection status and the tunnel deployment's readiness probe.", + "description": "TunnelReady condition when provider connection evidence is missing or unsupported." + }, "condition.tunnelready.tunnelcredentialrefused": { "defaultMessage": "The tunnel credentials secret ''{secret}'' is not owned by this GameServer. {remedy, select, defaultName {Add an ownerReference to this GameServer to the Secret, or delete the Secret and save the credential again from the dashboard.} other {Save the credential from the dashboard, or add an ownerReference to this GameServer to the Secret.}}", "description": "TunnelReady condition when the operator refuses to mount a credentials secret. secret is the secret name, remedy is defaultName or other." @@ -595,6 +603,10 @@ "defaultMessage": "Unknown tunnel provider ''{provider}''.", "description": "TunnelReady condition when the provider name is not supported." }, + "condition.tunnelready.unsupportedprotocol": { + "defaultMessage": "Tailscale forwards only advertised TCP and UDP ports. Remove unsupported protocols from the advertised ports.", + "description": "TunnelReady condition when a Tailscale template advertises a protocol other than TCP or UDP." + }, "condition.unquiesced.failed": { "defaultMessage": "The server could not be resumed after the backup. Its world may still have auto-save turned off.", "description": "Unquiesced condition when the unfreeze failed." diff --git a/operator/api/v1alpha1/gameserver_types.go b/operator/api/v1alpha1/gameserver_types.go index 654cba6ed..df2a65660 100644 --- a/operator/api/v1alpha1/gameserver_types.go +++ b/operator/api/v1alpha1/gameserver_types.go @@ -460,15 +460,12 @@ type TailscaleTunnelSpec struct { Hostname string `json:"hostname,omitempty"` // Tags are Tailscale ACL tags requested for the device when it - // registers, via `tailscale up --advertise-tags` (tailscaled's - // declarative config has no field for tags, and changing the tags of a - // registered node needs a fresh login). Each entry is "tag:" or a bare - // name, which gets the "tag:" prefix. The tailnet ACL must grant - // tagOwners for these tags to the auth key's owner; if the request is - // refused, the tunnel pod logs the error and the tunnel runs untagged. - // If any entry is invalid, the whole list is logged and ignored and the - // device registers untagged. See - // tunnel/specs.md. + // registers through embedded tsnet. Each entry is "tag:" or a + // bare name, which gets the "tag:" prefix. The tailnet policy must grant + // tagOwners to the auth key's owner. Invalid or refused tags keep the + // tunnel not ready; it never silently registers without requested tags. + // Changing tags replaces the Pod and requires a reusable auth key. + // See tunnel/specs.md for identity persistence and registration details. // +kubebuilder:validation:MaxItems=8 // +optional Tags []string `json:"tags,omitempty"` diff --git a/operator/config/crd/gameplane.local_gameservers.yaml b/operator/config/crd/gameplane.local_gameservers.yaml index 4be670ebc..b323bcd1d 100644 --- a/operator/config/crd/gameplane.local_gameservers.yaml +++ b/operator/config/crd/gameplane.local_gameservers.yaml @@ -1545,15 +1545,12 @@ spec: tags: description: |- Tags are Tailscale ACL tags requested for the device when it - registers, via `tailscale up --advertise-tags` (tailscaled's - declarative config has no field for tags, and changing the tags of a - registered node needs a fresh login). Each entry is "tag:" or a bare - name, which gets the "tag:" prefix. The tailnet ACL must grant - tagOwners for these tags to the auth key's owner; if the request is - refused, the tunnel pod logs the error and the tunnel runs untagged. - If any entry is invalid, the whole list is logged and ignored and the - device registers untagged. See - tunnel/specs.md. + registers through embedded tsnet. Each entry is "tag:" or a + bare name, which gets the "tag:" prefix. The tailnet policy must grant + tagOwners to the auth key's owner. Invalid or refused tags keep the + tunnel not ready; it never silently registers without requested tags. + Changing tags replaces the Pod and requires a reusable auth key. + See tunnel/specs.md for identity persistence and registration details. items: type: string maxItems: 8 diff --git a/operator/hack/condition-ids/main.go b/operator/hack/condition-ids/main.go index e644cf041..cf7502c17 100644 --- a/operator/hack/condition-ids/main.go +++ b/operator/hack/condition-ids/main.go @@ -121,8 +121,11 @@ var rows = []row{ {types: []string{"AddressAssignment"}, reason: "Assigned", english: "{assignedFrom, select, pool {Address {address} assigned from pool ''{pool}''.} other {Address {address} assigned.}}", description: "AddressAssignment condition once an address is assigned. assignedFrom is pool or default."}, // GameServer: tunnel and other + {types: []string{"TunnelReady"}, reason: "UnsupportedProtocol", english: "Tailscale forwards only advertised TCP and UDP ports. Remove unsupported protocols from the advertised ports.", description: "TunnelReady condition when a Tailscale template advertises a protocol other than TCP or UDP."}, {types: []string{"TunnelReady"}, reason: "InvalidConfig", english: "{provider, select, none {The tunnel is enabled but the server has no tunnel configuration.} other {The ''{provider}'' tunnel provider is selected but its configuration is missing.}}", description: "TunnelReady condition for an incomplete tunnel setup. provider is the selected provider, or none."}, {types: []string{"TunnelReady"}, reason: "DeploymentNotReady", english: "Waiting for the tunnel deployment to become ready.", description: "TunnelReady condition while the tunnel pod is not ready."}, + {types: []string{"TunnelReady"}, reason: "RelayConnectionUnknown", english: "The relay connection cannot be confirmed. Check the provider's connection status and the tunnel deployment's readiness probe.", description: "TunnelReady condition when provider connection evidence is missing or unsupported."}, + {types: []string{"TunnelReady"}, reason: "AwaitingAddress", english: "Waiting for the provider to assign a tunnel endpoint; the relay connection is unconfirmed.", description: "TunnelReady condition before Playit assigns an endpoint."}, {types: []string{"TunnelReady"}, reason: "PortNotMapped", english: "No frp remote port is configured for: {ports}.", description: "TunnelReady condition for frp when advertised ports have no remote port. ports is a comma-separated list of port names."}, {types: []string{"TunnelReady"}, reason: "NoCredentials", english: "No credentials secret was provided, so the ''{provider}'' tunnel cannot authenticate.", description: "TunnelReady condition when a tunnel provider needs credentials that are missing."}, {types: []string{"TunnelReady"}, reason: "Ready", english: "{provider, select, playit {The tunnel deployment is ready. Waiting for the playit endpoint assignment.} tailscale {The tunnel deployment is ready and authenticated.} other {The tunnel deployment is ready and all ports are mapped.}}", description: "TunnelReady condition when the tunnel works."}, diff --git a/operator/internal/controller/gameserver_relay_health_test.go b/operator/internal/controller/gameserver_relay_health_test.go new file mode 100644 index 000000000..2f3597791 --- /dev/null +++ b/operator/internal/controller/gameserver_relay_health_test.go @@ -0,0 +1,106 @@ +package controller + +import ( + "reflect" + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +func TestRelayDeploymentIdentityProbesAndResourceRemoval(t *testing.T) { + t.Parallel() + s := wipeScheme(t) + for _, identity := range []struct{ namespace, uid string }{{"one", "uid-one"}, {"two", "uid-two"}, {"one", "replacement-uid"}} { + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "same", Namespace: identity.namespace, UID: types.UID(identity.uid)}, Spec: gameplanev1alpha1.GameServerSpec{Networking: gameplanev1alpha1.GameServerNetworking{Tunnel: &gameplanev1alpha1.GameServerTunnel{Enabled: true, Provider: "frp", Frp: &gameplanev1alpha1.FrpTunnelSpec{ServerAddr: "relay.example"}}}}} + c := fake.NewClientBuilder().WithScheme(s).WithObjects(gs).Build() + r := &GameServerReconciler{Client: c, Scheme: s} + for _, resources := range []*corev1.ResourceRequirements{nil, {Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("90m")}, Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("128Mi")}}, {}, nil} { + r.TunnelResources = resources + if err := r.reconcileTunnel(t.Context(), gs, singlePortTemplate(true, "minecraft"), true); err != nil { + t.Fatal(err) + } + var dep appsv1.Deployment + if err := c.Get(t.Context(), client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-tunnel"}, &dep); err != nil { + t.Fatal(err) + } + if dep.Spec.Strategy.Type != appsv1.RecreateDeploymentStrategyType || dep.Spec.Strategy.RollingUpdate != nil { + t.Fatal("relay permits overlapping identities") + } + container := dep.Spec.Template.Spec.Containers[0] + if container.StartupProbe == nil || container.StartupProbe.HTTPGet == nil || container.StartupProbe.HTTPGet.Path != "/healthz" || !hasRelayReadinessProbe(&dep) || container.LivenessProbe != nil { + t.Fatal("provider outage must affect readiness without restarting the supervisor") + } + if !reflect.DeepEqual(container.Resources, auxiliaryResources(resources, "25m", "64Mi")) { + t.Fatal("resource defaults or override removal changed") + } + found := false + for _, env := range container.Env { + if env.Name == "GAMESERVER_UID" { + found = env.Value == identity.uid + } + } + if !found { + t.Fatal("UID identity missing or unstable") + } + } + } +} + +func TestRelayConditionsRequireCurrentProviderProbe(t *testing.T) { + t.Parallel() + s := wipeScheme(t) + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "relay", Namespace: "games", UID: "uid"}, Spec: gameplanev1alpha1.GameServerSpec{Networking: gameplanev1alpha1.GameServerNetworking{Tunnel: &gameplanev1alpha1.GameServerTunnel{Provider: "frp", Frp: &gameplanev1alpha1.FrpTunnelSpec{ServerAddr: "relay"}}}}} + c := fake.NewClientBuilder().WithScheme(s).WithObjects(gs).Build() + r := &GameServerReconciler{Client: c, Scheme: s} + if err := r.reconcileTunnel(t.Context(), gs, singlePortTemplate(true, "minecraft"), true); err != nil { + t.Fatal(err) + } + var dep appsv1.Deployment + if err := c.Get(t.Context(), client.ObjectKey{Namespace: gs.Namespace, Name: "relay-tunnel"}, &dep); err != nil { + t.Fatal(err) + } + dep.Generation = 2 + dep.Status = appsv1.DeploymentStatus{ReadyReplicas: 1, UpdatedReplicas: 1, ObservedGeneration: 2} + assert := func(want metav1.ConditionStatus) { + t.Helper() + cond := meta.FindStatusCondition(computeTunnelConditions(gs, tunnelPlan{wantTunnel: true}, &dep), "TunnelReady") + if cond == nil || cond.Status != want { + t.Fatalf("condition=%+v want=%s", cond, want) + } + } + assert(metav1.ConditionTrue) + dep.Status.ObservedGeneration = 1 + assert(metav1.ConditionUnknown) + dep.Status.ObservedGeneration = 2 + dep.Status.ReadyReplicas = 0 + assert(metav1.ConditionFalse) + dep.Status.ReadyReplicas = 1 + dep.Spec.Template.Spec.Containers[0].ReadinessProbe = nil + assert(metav1.ConditionUnknown) +} + +func TestPlayitAssignmentDoesNotImplyRelayReady(t *testing.T) { + t.Parallel() + gs := &gameplanev1alpha1.GameServer{Spec: gameplanev1alpha1.GameServerSpec{Networking: gameplanev1alpha1.GameServerNetworking{Tunnel: &gameplanev1alpha1.GameServerTunnel{Provider: "playit", Playit: &gameplanev1alpha1.PlayitTunnelSpec{}, CredentialsSecretRef: &gameplanev1alpha1.SecretNameRef{Name: "relay-credentials"}}}}} + dep := tunnelReadyDeployment() + for _, endpoints := range [][]gameplanev1alpha1.GameServerEndpoint{nil, {{Name: "game", Host: "example.playit.gg", Port: 30000}}} { + gs.Status.TunnelEndpoints = endpoints + cond := meta.FindStatusCondition(computeTunnelConditions(gs, tunnelPlan{wantTunnel: true}, dep), "TunnelReady") + wantStatus, wantReason := metav1.ConditionUnknown, "RelayConnectionUnknown" + if len(endpoints) == 0 { + wantStatus, wantReason = metav1.ConditionFalse, "AwaitingAddress" + } + if cond == nil || cond.Status != wantStatus || cond.Reason != wantReason { + t.Fatalf("playit readiness=%+v, want %s/%s", cond, wantStatus, wantReason) + } + } +} diff --git a/operator/internal/controller/gameserver_status.go b/operator/internal/controller/gameserver_status.go index 54393de0b..19ecbe8d4 100644 --- a/operator/internal/controller/gameserver_status.go +++ b/operator/internal/controller/gameserver_status.go @@ -260,8 +260,19 @@ func (r *GameServerReconciler) reconcileStatus( // Connection card reads the address players actually use. if svcExists { gs.Status.Endpoints = endpointsFromService(&svc, addrPlan) + } else { + // A deleted/replacing Service must not preserve a stale tailnet endpoint + // when the forwarding deployment has since lost readiness. + var retained []gameplanev1alpha1.GameServerEndpoint + for _, endpoint := range gs.Status.Endpoints { + if endpoint.TunnelProvider != "tailscale" { + retained = append(retained, endpoint) + } + } + gs.Status.Endpoints = retained } - if tunnelPlan.wantTunnel && len(tunnelPlan.endpoints) > 0 { + if tunnelPlan.wantTunnel && len(tunnelPlan.endpoints) > 0 && + (gs.Spec.Networking.Tunnel.Provider != "tailscale" || meta.IsStatusConditionTrue(gs.Status.Conditions, "TunnelReady")) { gs.Status.Endpoints = append(tunnelPlan.endpoints, gs.Status.Endpoints...) } if phase == gameplanev1alpha1.GameServerPhaseRunning { @@ -1105,6 +1116,21 @@ func computeTunnelConditions( } // Tunnel Deployment not yet created or not ready. + if tunnel.Provider == "tailscale" && len(plan.unsupportedPorts) > 0 { + tunnelReady.Status = metav1.ConditionFalse + tunnelReady.Reason = "UnsupportedProtocol" + tunnelReady.Message = "Tailscale supports only TCP and UDP; unsupported advertised ports: " + strings.Join(plan.unsupportedPorts, ", ") + return upsertCondition(conds, tunnelReady) + } + if tunnel.Provider == "playit" && (tunnel.Playit == nil || tunnel.CredentialsSecretRef == nil || tunnel.CredentialsSecretRef.Name == "") { + tunnelReady.Status = metav1.ConditionFalse + tunnelReady.Reason = "NoCredentials" + tunnelReady.Message = "playit configuration and credentials are required before relay connection can be confirmed" + if tunnel.Playit == nil { + tunnelReady.Reason = "InvalidConfig" + } + return upsertCondition(conds, tunnelReady) + } if tunnelDep == nil { tunnelReady.Status = metav1.ConditionFalse tunnelReady.Reason = "DeploymentNotReady" @@ -1112,11 +1138,30 @@ func computeTunnelConditions( conds = upsertCondition(conds, tunnelReady) return conds } + if tunnel.Provider == "frp" || tunnel.Provider == "playit" { + if !hasRelayReadinessProbe(tunnelDep) || tunnelDep.Status.ObservedGeneration < tunnelDep.Generation || tunnelDep.Status.UpdatedReplicas == 0 { + tunnelReady.Status = metav1.ConditionUnknown + tunnelReady.Reason = "RelayConnectionUnknown" + tunnelReady.Message = "waiting for the current relay deployment and its connection readiness probe" + return upsertCondition(conds, tunnelReady) + } + if tunnel.Provider == "playit" { + tunnelReady.Status = metav1.ConditionUnknown + tunnelReady.Reason = "RelayConnectionUnknown" + tunnelReady.Message = "playit v1.0.10 IPC reports cached assignment but cannot confirm relay connection; check provider connectivity separately" + if len(gs.Status.TunnelEndpoints) == 0 { + tunnelReady.Status = metav1.ConditionFalse + tunnelReady.Reason = "AwaitingAddress" + tunnelReady.Message = "waiting for playit endpoint assignment; relay connection is unconfirmed" + } + return upsertCondition(conds, tunnelReady) + } + } if tunnelDep.Status.ReadyReplicas == 0 { tunnelReady.Status = metav1.ConditionFalse tunnelReady.Reason = "DeploymentNotReady" - tunnelReady.Message = "tunnel deployment has no ready replicas" + tunnelReady.Message = "tunnel deployment has no ready replicas; relay connection or forward registration is unconfirmed" conds = upsertCondition(conds, tunnelReady) return conds } @@ -1147,7 +1192,7 @@ func computeTunnelConditions( // All endpoints known: deployment ready and mappings complete. tunnelReady.Status = metav1.ConditionTrue tunnelReady.Reason = "Ready" - tunnelReady.Message = "tunnel deployment ready and all ports mapped" + tunnelReady.Message = "relay connection and all configured frp forward registrations confirmed" case "tailscale": if tunnel.Tailscale == nil { @@ -1167,37 +1212,23 @@ func computeTunnelConditions( return conds } - // Deployment is ready and credentials are configured. - tunnelReady.Status = metav1.ConditionTrue - tunnelReady.Reason = "Ready" - tunnelReady.Message = "tunnel deployment ready and authenticated" - - case "playit": - if tunnel.Playit == nil { - tunnelReady.Status = metav1.ConditionFalse - tunnelReady.Reason = "InvalidConfig" - tunnelReady.Message = "playit provider selected but no playit config" - conds = upsertCondition(conds, tunnelReady) - return conds + // A legacy image merely registering a device cannot satisfy readiness. + forwardingProbe := false + for _, container := range tunnelDep.Spec.Template.Spec.Containers { + if container.Name == "tunnel" && container.ReadinessProbe != nil && container.ReadinessProbe.HTTPGet != nil { + probe := container.ReadinessProbe.HTTPGet + forwardingProbe = probe.Path == "/readyz" && probe.Port.IntValue() == 8091 + } } - - // Check if credentials are provided (playit requires authentication). - if tunnel.CredentialsSecretRef == nil { + if !forwardingProbe || tunnelDep.Status.ObservedGeneration < tunnelDep.Generation || tunnelDep.Status.UpdatedReplicas == 0 { tunnelReady.Status = metav1.ConditionFalse - tunnelReady.Reason = "NoCredentials" - tunnelReady.Message = "no credentials secret provided; playit requires authentication" - conds = upsertCondition(conds, tunnelReady) - return conds + tunnelReady.Reason = "DeploymentNotReady" + tunnelReady.Message = "waiting for the Tailscale forwarding deployment and its readiness probe" + return upsertCondition(conds, tunnelReady) } - - // Deployment is ready; playit endpoint will arrive asynchronously. - // Report AwaitingAddress while the tunnel pod hasn't yet reported an address. - // This is optimistic: the condition flips based on whether the tunnel pod - // has written to status.tunnelEndpoint. The agent or operator reconcile must - // populate it separately. tunnelReady.Status = metav1.ConditionTrue tunnelReady.Reason = "Ready" - tunnelReady.Message = "tunnel deployment ready; waiting for playit endpoint assignment" + tunnelReady.Message = "tailnet online and all advertised TCP/UDP forwarding listeners active; client access remains subject to tailnet policy" default: tunnelReady.Status = metav1.ConditionFalse @@ -1209,6 +1240,16 @@ func computeTunnelConditions( return conds } +func hasRelayReadinessProbe(dep *appsv1.Deployment) bool { + for _, container := range dep.Spec.Template.Spec.Containers { + if container.Name == "tunnel" && container.ReadinessProbe != nil && container.ReadinessProbe.HTTPGet != nil { + probe := container.ReadinessProbe.HTTPGet + return probe.Path == "/readyz" && probe.Port.IntValue() == 8091 + } + } + return false +} + // removeCondition removes a condition by type from the list, or returns the list unchanged. func removeCondition(conds []metav1.Condition, condType string) []metav1.Condition { out := make([]metav1.Condition, 0, len(conds)) diff --git a/operator/internal/controller/gameserver_tailscale_test.go b/operator/internal/controller/gameserver_tailscale_test.go new file mode 100644 index 000000000..1653fccf2 --- /dev/null +++ b/operator/internal/controller/gameserver_tailscale_test.go @@ -0,0 +1,48 @@ +package controller + +import ( + "context" + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +func TestTailscaleAdvertisedProtocolsAndReadiness(t *testing.T) { + tmpl := &gameplanev1alpha1.GameTemplate{Spec: gameplanev1alpha1.GameTemplateSpec{Ports: []gameplanev1alpha1.GamePort{ + {Name: "game", ContainerPort: 25565, Advertise: true}, + {Name: "query", ContainerPort: 19132, Protocol: corev1.ProtocolUDP, Advertise: true}, + {Name: "rcon", ContainerPort: 25575, Protocol: corev1.ProtocolTCP}, + }}} + if got := buildTailscalePortsConfig(tmpl); got != "game:25565:tcp,query:19132:udp" { + t.Fatalf("advertised mapping: %s", got) + } + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "game", Namespace: "games"}, Spec: gameplanev1alpha1.GameServerSpec{Networking: gameplanev1alpha1.GameServerNetworking{Tunnel: &gameplanev1alpha1.GameServerTunnel{Enabled: true, Provider: "tailscale", Tailscale: &gameplanev1alpha1.TailscaleTunnelSpec{}, CredentialsSecretRef: &gameplanev1alpha1.SecretNameRef{Name: "auth"}}}}} + r := &GameServerReconciler{} + plan := r.planTunnel(context.Background(), gs, tmpl) + dep := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Generation: 2}, Status: appsv1.DeploymentStatus{ReadyReplicas: 1, UpdatedReplicas: 1, ObservedGeneration: 2}} + assert := func(want metav1.ConditionStatus) { + t.Helper() + c := meta.FindStatusCondition(computeTunnelConditions(gs, plan, dep), "TunnelReady") + if c == nil || c.Status != want { + t.Fatalf("TunnelReady=%+v, want %s", c, want) + } + } + assert(metav1.ConditionFalse) // A registered legacy deployment has no forwarding probe. + dep.Spec.Template.Spec.Containers = []corev1.Container{{Name: "tunnel", ReadinessProbe: &corev1.Probe{ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{Path: "/readyz", Port: intstr.FromInt(8091)}}}}} + assert(metav1.ConditionTrue) + dep.Status.ObservedGeneration = 1 + assert(metav1.ConditionFalse) + dep.Status.ObservedGeneration = 2 + dep.Status.ReadyReplicas = 0 + assert(metav1.ConditionFalse) + dep.Status.ReadyReplicas = 1 + tmpl.Spec.Ports[0].Protocol = corev1.ProtocolSCTP + plan = r.planTunnel(context.Background(), gs, tmpl) + assert(metav1.ConditionFalse) +} diff --git a/operator/internal/controller/gameserver_tunnel.go b/operator/internal/controller/gameserver_tunnel.go index e5712d96b..03785c236 100644 --- a/operator/internal/controller/gameserver_tunnel.go +++ b/operator/internal/controller/gameserver_tunnel.go @@ -52,7 +52,8 @@ type tunnelPlan struct { endpoints []gameplanev1alpha1.GameServerEndpoint // noMapping is the set of advertised port names that have no frp remote-port mapping. // Used only by frp to report a helpful condition reason. - noMapping []string + noMapping []string + unsupportedPorts []string } // planTunnel is the single source of truth for the tunnel pod's existence @@ -74,6 +75,7 @@ func (r *GameServerReconciler) planTunnel( tunnel := gs.Spec.Networking.Tunnel endpoints := make([]gameplanev1alpha1.GameServerEndpoint, 0) var noMapping []string + var unsupportedPorts []string switch tunnel.Provider { case "frp": @@ -128,6 +130,10 @@ func (r *GameServerReconciler) planTunnel( if !p.Advertise { continue } + if p.Protocol != "" && p.Protocol != corev1.ProtocolTCP && p.Protocol != corev1.ProtocolUDP { + unsupportedPorts = append(unsupportedPorts, p.Name) + continue + } ep := gameplanev1alpha1.GameServerEndpoint{ Name: p.Name, @@ -152,9 +158,10 @@ func (r *GameServerReconciler) planTunnel( } return tunnelPlan{ - wantTunnel: true, - endpoints: endpoints, - noMapping: noMapping, + wantTunnel: true, + endpoints: endpoints, + noMapping: noMapping, + unsupportedPorts: unsupportedPorts, } } @@ -199,6 +206,11 @@ func (r *GameServerReconciler) reconcileTunnel( refusedSecret = "" replicas := int32(1) dep.Spec.Replicas = &replicas + // A provider identity owns one relay session. Recreate unregisters the + // old child before its replacement can claim the same names/ports. + if tunnel.Provider == "frp" || tunnel.Provider == "playit" || tunnel.Provider == "tailscale" { + dep.Spec.Strategy = appsv1.DeploymentStrategy{Type: appsv1.RecreateDeploymentStrategyType} + } dep.Spec.Selector = &metav1.LabelSelector{ MatchLabels: map[string]string{ tunnelLabel: tunnelValue, @@ -272,6 +284,7 @@ func (r *GameServerReconciler) reconcileTunnel( envVars = []corev1.EnvVar{ {Name: "GAMESERVER_NAME", Value: gs.Name}, {Name: "GAMESERVER_NAMESPACE", Value: gs.Namespace}, + {Name: "GAMESERVER_UID", Value: string(gs.UID)}, {Name: "TUNNEL_TYPE", Value: "frp"}, {Name: "FRP_SERVER_ADDR", Value: tunnel.Frp.ServerAddr}, {Name: "FRP_SERVER_PORT", Value: fmt.Sprintf("%d", serverPort)}, @@ -388,14 +401,21 @@ func (r *GameServerReconciler) reconcileTunnel( } } + var readiness *corev1.Probe + if tunnel.Provider == "tailscale" { + volumes = append(volumes, corev1.Volume{Name: "tailscale-state", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}) + volumeMounts = append(volumeMounts, corev1.VolumeMount{Name: "tailscale-state", MountPath: "/var/lib/gameplane-tailscale"}) + readiness = &corev1.Probe{ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{Path: "/readyz", Port: intstr.FromInt(8091)}}, PeriodSeconds: 5, TimeoutSeconds: 2, FailureThreshold: 1} + } dep.Spec.Template.Spec.Volumes = volumes dep.Spec.Template.Spec.Containers = []corev1.Container{{ - Name: "tunnel", - Image: image, - Resources: auxiliaryResources(r.TunnelResources, "25m", "64Mi"), - Env: envVars, - Ports: ports, - VolumeMounts: volumeMounts, + Name: "tunnel", + Image: image, + Resources: auxiliaryResources(r.TunnelResources, "25m", "64Mi"), + Env: envVars, + Ports: ports, + VolumeMounts: volumeMounts, + ReadinessProbe: readiness, SecurityContext: &corev1.SecurityContext{ RunAsNonRoot: &nonRoot, RunAsUser: &uid, @@ -403,6 +423,11 @@ func (r *GameServerReconciler) reconcileTunnel( Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, }, }} + if tunnel.Provider == "frp" || tunnel.Provider == "playit" { + container := &dep.Spec.Template.Spec.Containers[0] + container.StartupProbe = &corev1.Probe{ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{Path: "/healthz", Port: intstr.FromInt32(8091)}}, PeriodSeconds: 2, TimeoutSeconds: 1, FailureThreshold: 30} + container.ReadinessProbe = &corev1.Probe{ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{Path: "/readyz", Port: intstr.FromInt32(8091)}}, PeriodSeconds: 2, TimeoutSeconds: 1, FailureThreshold: 1} + } return controllerutil.SetControllerReference(gs, dep, r.Scheme) }) @@ -695,14 +720,18 @@ func buildFrpRemotePortsConfig(frp *gameplanev1alpha1.FrpTunnelSpec, tmpl *gamep } // buildTailscalePortsConfig constructs the BACKING_SERVICE_PORTS env var for Tailscale. -// Format: "port_name:port,..." e.g. "java:25565,bedrock:19133" +// Format: "port_name:port:protocol,..." e.g. "java:25565:tcp,bedrock:19133:udp". func buildTailscalePortsConfig(tmpl *gameplanev1alpha1.GameTemplate) string { var entries []string for _, p := range tmpl.Spec.Ports { if !p.Advertise { continue } - entries = append(entries, fmt.Sprintf("%s:%d", p.Name, p.ContainerPort)) + protocol := strings.ToLower(string(p.Protocol)) + if protocol == "" { + protocol = "tcp" + } + entries = append(entries, fmt.Sprintf("%s:%d:%s", p.Name, p.ContainerPort, protocol)) } if len(entries) == 0 { return "" diff --git a/operator/internal/controller/gameserver_tunnel_envtest_test.go b/operator/internal/controller/gameserver_tunnel_envtest_test.go index d86582fb9..2447b2ee1 100644 --- a/operator/internal/controller/gameserver_tunnel_envtest_test.go +++ b/operator/internal/controller/gameserver_tunnel_envtest_test.go @@ -109,6 +109,32 @@ func TestTunnelReconciliation_CredentialKeyProjection(t *testing.T) { } // Verify the Secret volume has Items projection with only "authKey" + if dep.Spec.Strategy.Type != appsv1.RecreateDeploymentStrategyType || dep.Spec.Strategy.RollingUpdate != nil { + t.Fatalf("Tailscale must recreate its single identity: %+v", dep.Spec.Strategy) + } + container := dep.Spec.Template.Spec.Containers[0] + if container.ReadinessProbe == nil || container.ReadinessProbe.HTTPGet == nil || container.ReadinessProbe.HTTPGet.Path != "/readyz" || container.ReadinessProbe.HTTPGet.Port.IntValue() != 8091 { + t.Fatalf("Tailscale lacks forwarding readiness probe: %+v", container.ReadinessProbe) + } + stateVolume, stateMount, protocolEnv := false, false, false + for _, volume := range dep.Spec.Template.Spec.Volumes { + if volume.Name == "tailscale-state" && volume.EmptyDir != nil { + stateVolume = true + } + } + for _, mount := range container.VolumeMounts { + if mount.Name == "tailscale-state" && mount.MountPath == "/var/lib/gameplane-tailscale" { + stateMount = true + } + } + for _, env := range container.Env { + if env.Name == "BACKING_SERVICE_PORTS" && env.Value == "game:25565:tcp" { + protocolEnv = true + } + } + if !stateVolume || !stateMount || !protocolEnv { + t.Fatal("Tailscale lost persistent pod state or advertised protocol contract") + } if len(dep.Spec.Template.Spec.Volumes) == 0 { t.Fatal("no volumes in tunnel pod template") } diff --git a/operator/specs.md b/operator/specs.md index 2db4f71cf..b640c8c82 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -10,6 +10,8 @@ The operator is a Kubernetes controller-runtime-based process that reconciles Ga ## Responsibilities +- **Relay health:** frp identities use GameServer UID without changing advertised port names. frp/Playit Deployments use one replica, Recreate, a wrapper startup probe and a provider readiness probe on TCP 8091. `TunnelReady` requires the current observed generation and updated/ready replicas with the new probe; legacy unprobed Pods cannot report success. Playit v1.0.10's cached IPC assignment cannot confirm a relay connection, so it reports unknown/not ready rather than optimistic success. Preserve `TunnelResources` defaults and override removal. See `docs/relay-health.md` for health bounds, migration and the provider limitation. + - **GameServer lifecycle:** reconcile GameServer CRs into StatefulSets, Services, PVCs, and pod-injected agent sidecars; manage pod affinity, RCON/console integration, file I/O, and graceful stop sequences. - **Module installation:** pull OCI module bundles from remote or local sources, verify signatures (cosign), and materialize GameTemplate CRs; refresh source indexes periodically. Translation files (`locales/.json`) are carried by every module source (OCI layers, directory, git, HTTP archive, upload, `gp-module package` tarball) and validated against the ids derived from the module (`i18n/modlocale`); valid entries are written to the `