diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ab9ded873..8898c6cf4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1259,6 +1259,8 @@ jobs: run: python3 charts/tests/check_gateway.py - name: standalone panel isolation and persistence chart renders run: python3 charts/tests/check_standalone.py + - name: strict network profile and sentinel policy selectors + run: python3 charts/tests/check_network_isolation.py - name: auxiliary resources and management placement chart renders run: python3 charts/tests/check_workloads.py - name: operator Secret namespace permissions and cache configuration @@ -1824,9 +1826,13 @@ jobs: # (operator does zero logins and is wait-dominated, so it runs # wide; the api buckets stay within the login limiter's budget). # The arm64 job below uses the same steps and per-bucket settings. - bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, management-placement] + bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, network-isolation, management-placement] arch: [amd64] include: + - bucket: network-isolation + parallel: 1 + test_timeout: 15m + job_timeout: 60 - bucket: operator parallel: 8 test_timeout: 20m @@ -1892,6 +1898,7 @@ jobs: - name: bring up cluster + helm install if: env.RUN_REGEX != '' || env.TAIL_REGEX != '' env: + GAMEPLANE_E2E_CNI: ${{ matrix.bucket == 'network-isolation' && 'cilium' || 'kindnet' }} GAMEPLANE_E2E_PROFILE: ${{ matrix.bucket == 'management-placement' && 'management' || 'default' }} run: ./deploy/kind/e2e.sh up "$CLUSTER" e2e "$GAMEPLANE_E2E_PROFILE" - name: run go e2e (${{ matrix.bucket }}) @@ -2697,7 +2704,7 @@ jobs: 'go-e2e-unit': (n) => n === 'go (test/e2e)', 'api-postgres': (n) => n === 'api (postgres)', 'e2e-buckets': (n) => n === 'e2e bucket coverage', - 'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|management-placement) \//.test(n) && n.includes('/ amd64 '), + 'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|network-isolation|management-placement) \//.test(n) && n.includes('/ amd64 '), 'e2e-go-arm64': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//.test(n) && n.includes('/ arm64 '), 'e2e-multicluster': (n) => n.startsWith('e2e multicluster /') && n.includes('/ amd64 '), 'e2e-multicluster-arm64': (n) => n.startsWith('e2e multicluster /') && n.includes('/ arm64 '), @@ -2850,7 +2857,7 @@ jobs: const bucketSet = new Set(); for (const j of apiJobs) { const n = j.name || ''; - const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|management-placement) \//); + const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|network-isolation|management-placement) \//); if (m) bucketSet.add(m[1]); if (/^e2e multicluster \//.test(n)) bucketSet.add('multicluster'); if (/^e2e upgrade \//.test(n)) bucketSet.add('upgrade'); diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml index e2aa9dce4..0fc5ae27d 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: afbe99d46e972c35a07f42486a1f8f6ec895c183017a9d2eef67b7ea07824f6b + gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 controller-gen.kubebuilder.io/version: v0.22.0 name: backups.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml index f74e59433..2bdad0023 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: afbe99d46e972c35a07f42486a1f8f6ec895c183017a9d2eef67b7ea07824f6b + gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 controller-gen.kubebuilder.io/version: v0.22.0 name: backupschedules.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml index 5fb318214..b2a11b475 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: afbe99d46e972c35a07f42486a1f8f6ec895c183017a9d2eef67b7ea07824f6b + gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 controller-gen.kubebuilder.io/version: v0.22.0 name: clusters.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml index be016422a..f768c5616 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: afbe99d46e972c35a07f42486a1f8f6ec895c183017a9d2eef67b7ea07824f6b + gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 controller-gen.kubebuilder.io/version: v0.22.0 name: gameservers.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml index b15227625..7408d7b80 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: afbe99d46e972c35a07f42486a1f8f6ec895c183017a9d2eef67b7ea07824f6b + gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 controller-gen.kubebuilder.io/version: v0.22.0 name: gametemplates.gameplane.local spec: @@ -2114,6 +2114,12 @@ spec: minimum: 0 type: integer type: object + sentinelIsolation: + description: |- + SentinelIsolation restricts a wake sentinel to its own server's advertised + ports. Templates opt in after module connectivity validation; absent or + false retains the legacy namespace-wide sentinel allowances. + type: boolean storage: description: Storage describes the default persistent storage layout. properties: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml index 91ec252d1..2fe842fde 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: afbe99d46e972c35a07f42486a1f8f6ec895c183017a9d2eef67b7ea07824f6b + gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 controller-gen.kubebuilder.io/version: v0.22.0 name: modules.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml index 8722e927a..39450ee94 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: afbe99d46e972c35a07f42486a1f8f6ec895c183017a9d2eef67b7ea07824f6b + gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 controller-gen.kubebuilder.io/version: v0.22.0 name: modulesources.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml index 17d38fc46..c88fd000e 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: afbe99d46e972c35a07f42486a1f8f6ec895c183017a9d2eef67b7ea07824f6b + gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 controller-gen.kubebuilder.io/version: v0.22.0 name: networkcaptures.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml index 37f7a476c..a57f794c0 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: afbe99d46e972c35a07f42486a1f8f6ec895c183017a9d2eef67b7ea07824f6b + gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 controller-gen.kubebuilder.io/version: v0.22.0 name: restores.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_backups.yaml b/charts/gameplane/crd-manifests/gameplane.local_backups.yaml index e2aa9dce4..0fc5ae27d 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_backups.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_backups.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: afbe99d46e972c35a07f42486a1f8f6ec895c183017a9d2eef67b7ea07824f6b + gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 controller-gen.kubebuilder.io/version: v0.22.0 name: backups.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml b/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml index f74e59433..2bdad0023 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: afbe99d46e972c35a07f42486a1f8f6ec895c183017a9d2eef67b7ea07824f6b + gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 controller-gen.kubebuilder.io/version: v0.22.0 name: backupschedules.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml b/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml index 5fb318214..b2a11b475 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: afbe99d46e972c35a07f42486a1f8f6ec895c183017a9d2eef67b7ea07824f6b + gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 controller-gen.kubebuilder.io/version: v0.22.0 name: clusters.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml b/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml index be016422a..f768c5616 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: afbe99d46e972c35a07f42486a1f8f6ec895c183017a9d2eef67b7ea07824f6b + gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 controller-gen.kubebuilder.io/version: v0.22.0 name: gameservers.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml b/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml index b15227625..7408d7b80 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: afbe99d46e972c35a07f42486a1f8f6ec895c183017a9d2eef67b7ea07824f6b + gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 controller-gen.kubebuilder.io/version: v0.22.0 name: gametemplates.gameplane.local spec: @@ -2114,6 +2114,12 @@ spec: minimum: 0 type: integer type: object + sentinelIsolation: + description: |- + SentinelIsolation restricts a wake sentinel to its own server's advertised + ports. Templates opt in after module connectivity validation; absent or + false retains the legacy namespace-wide sentinel allowances. + type: boolean storage: description: Storage describes the default persistent storage layout. properties: diff --git a/charts/gameplane/crd-manifests/gameplane.local_modules.yaml b/charts/gameplane/crd-manifests/gameplane.local_modules.yaml index 91ec252d1..2fe842fde 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_modules.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_modules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: afbe99d46e972c35a07f42486a1f8f6ec895c183017a9d2eef67b7ea07824f6b + gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 controller-gen.kubebuilder.io/version: v0.22.0 name: modules.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml b/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml index 8722e927a..39450ee94 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: afbe99d46e972c35a07f42486a1f8f6ec895c183017a9d2eef67b7ea07824f6b + gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 controller-gen.kubebuilder.io/version: v0.22.0 name: modulesources.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml b/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml index 17d38fc46..c88fd000e 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: afbe99d46e972c35a07f42486a1f8f6ec895c183017a9d2eef67b7ea07824f6b + gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 controller-gen.kubebuilder.io/version: v0.22.0 name: networkcaptures.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_restores.yaml b/charts/gameplane/crd-manifests/gameplane.local_restores.yaml index 37f7a476c..a57f794c0 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_restores.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_restores.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: afbe99d46e972c35a07f42486a1f8f6ec895c183017a9d2eef67b7ea07824f6b + gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 controller-gen.kubebuilder.io/version: v0.22.0 name: restores.gameplane.local spec: diff --git a/charts/gameplane/templates/networkpolicies.yaml b/charts/gameplane/templates/networkpolicies.yaml index 81f963f4d..612fc5071 100644 --- a/charts/gameplane/templates/networkpolicies.yaml +++ b/charts/gameplane/templates/networkpolicies.yaml @@ -331,8 +331,9 @@ spec: - { protocol: TCP, port: 443 } - { protocol: TCP, port: 6443 } --- -# Allow the sentinel to egress to game pods on advertised ports for the proxy path. -# This is the INGRESS half only, on the game pod's own selector. +# Legacy sentinel proxy allowance, ingress half. This grants ALL ports between +# non-isolated sentinels and games. NetworkPolicies are additive; per-server +# player ingress does not narrow it. Opted-in templates use operator policies. # NetworkPolicy requires both sides to agree: the destination (this policy) # admits the sentinel, but default-deny-egress's podSelector: {} still # governs the sentinel as a SOURCE and only opens DNS. allow-sentinel-to-game-egress @@ -349,14 +350,17 @@ spec: podSelector: matchLabels: app.kubernetes.io/name: gameplane-game + matchExpressions: + - { key: gameplane.local/sentinel-isolation, operator: NotIn, values: ["true"] } policyTypes: [Ingress] ingress: - from: - podSelector: matchLabels: app.kubernetes.io/name: gameplane-waker - # All ports; the per-server ingress policy covers which ports are - # actually advertised for each server. + matchExpressions: + - { key: gameplane.local/sentinel-isolation, operator: NotIn, values: ["true"] } + # Intentionally unrestricted for templates that have not opted in. --- # The Egress half of allow-sentinel-to-game (see its comment above): admits # the sentinel, as a traffic SOURCE, to reach game pods. Selects the @@ -374,15 +378,17 @@ spec: podSelector: matchLabels: app.kubernetes.io/name: gameplane-waker + matchExpressions: + - { key: gameplane.local/sentinel-isolation, operator: NotIn, values: ["true"] } policyTypes: [Egress] egress: - to: - podSelector: matchLabels: app.kubernetes.io/name: gameplane-game - # All ports; allow-sentinel-to-game above (the matching ingress half) - # and the per-server ingress policy cover which ports are actually - # advertised for each server. + matchExpressions: + - { key: gameplane.local/sentinel-isolation, operator: NotIn, values: ["true"] } + # Intentionally unrestricted for templates that have not opted in. {{- end }} {{- end }} diff --git a/charts/gameplane/templates/validate.yaml b/charts/gameplane/templates/validate.yaml index 79b5e17b6..bf4a4580f 100644 --- a/charts/gameplane/templates/validate.yaml +++ b/charts/gameplane/templates/validate.yaml @@ -1,3 +1,31 @@ +{{- $network := .Values.networkPolicies | default dict }} +{{- if $network.strict }} +{{- if not $network.enabled }} +{{- fail "networkPolicies.strict requires networkPolicies.enabled=true" }} +{{- end }} +{{- range $key := list "kubeletCIDRs" "apiServerCIDRs" "probePorts" }} +{{- $entries := index $network $key }} +{{- if or (not (kindIs "slice" $entries)) (eq (len $entries) 0) }} +{{- fail (printf "networkPolicies.strict requires a non-empty networkPolicies.%s list" $key) }} +{{- end }} +{{- end }} +{{- $octet := "(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])" }} +{{- $ipv4 := printf "^%s(\\.%s){3}/([0-9]|[12][0-9]|3[0-2])$" $octet $octet }} +{{- $h := "[0-9A-Fa-f]{1,4}" }} +{{- $ipv6 := printf "^((%s:){7}%s|(%s:){1,7}:|(%s:){1,6}:%s|(%s:){1,5}(:%s){1,2}|(%s:){1,4}(:%s){1,3}|(%s:){1,3}(:%s){1,4}|(%s:){1,2}(:%s){1,5}|%s:((:%s){1,6})|:((:%s){1,7}|:))/([0-9]|[1-9][0-9]|1[01][0-9]|12[0-8])$" $h $h $h $h $h $h $h $h $h $h $h $h $h $h $h $h }} +{{- range $key := list "kubeletCIDRs" "apiServerCIDRs" }} +{{- range (index $network $key) }} +{{- if or (not (kindIs "string" .)) (not (or (regexMatch $ipv4 (toString .)) (regexMatch $ipv6 (toString .)))) }} +{{- fail (printf "networkPolicies.%s entries must be explicit IPv4/IPv6 CIDRs" $key) }} +{{- end }} +{{- end }} +{{- end }} +{{- range $network.probePorts }} +{{- if or (not (has (.protocol | default "") (list "TCP" "UDP" "SCTP"))) (not (regexMatch "^[0-9]+$" (toString .port))) (lt (int .port) 1) (gt (int .port) 65535) }} +{{- fail "networkPolicies.probePorts requires protocol TCP/UDP/SCTP and an integer port from 1 to 65535" }} +{{- end }} +{{- end }} +{{- end }} {{- if .Values.api.standalone }} {{- if ne (include "gameplane.apiEnabled" .) "true" }} {{- fail "api.standalone requires api.enabled=true" }} diff --git a/charts/gameplane/values.yaml b/charts/gameplane/values.yaml index 8acb5ba60..4acd15c53 100644 --- a/charts/gameplane/values.yaml +++ b/charts/gameplane/values.yaml @@ -552,6 +552,9 @@ gamesNamespace: gameplane-games networkPolicies: enabled: true + # Fail closed unless actual kubelet source CIDRs/probe ports and apiserver + # endpoint CIDRs are supplied. Defaults retain the legacy broad allowances. + strict: false # CIDRs that hold kubelet source IPs for liveness/readiness probes. # If unset, the chart falls back to RFC1918 + link-local ranges, # which cover most clusters without opening the public internet. diff --git a/charts/tests/check_network_isolation.py b/charts/tests/check_network_isolation.py new file mode 100644 index 000000000..7c180d143 --- /dev/null +++ b/charts/tests/check_network_isolation.py @@ -0,0 +1,49 @@ +"""CI-only rendered effective policy tuple and strict-profile regressions.""" +from check_standalone import render, one + + +def main(): + values = {"networkPolicies": {"strict": True, + "kubeletCIDRs": ["192.0.2.10/32"], + "probePorts": [{"protocol": "TCP", "port": 8090}], + "apiServerCIDRs": ["198.51.100.20/32"]}} + objects = render(values) + probe = one(objects, "NetworkPolicy", "allow-kubelet-probes")["spec"]["ingress"] + assert probe == [{"from": [{"ipBlock": {"cidr": "192.0.2.10/32"}}], + "ports": [{"protocol": "TCP", "port": 8090}]}] + for name in ("allow-agent-to-apiserver", "allow-sentinel-to-apiserver"): + egress = one(objects, "NetworkPolicy", name)["spec"]["egress"] + assert egress == [{"to": [{"ipBlock": {"cidr": "198.51.100.20/32"}}], + "ports": [{"protocol": "TCP", "port": 443}, {"protocol": "TCP", "port": 6443}]}] + excluded = [{"key": "gameplane.local/sentinel-isolation", "operator": "NotIn", "values": ["true"]}] + for name, direction, peer in (("allow-sentinel-to-game", "ingress", "from"), + ("allow-sentinel-to-game-egress", "egress", "to")): + policy = one(objects, "NetworkPolicy", name)["spec"] + assert policy["podSelector"]["matchExpressions"] == excluded + assert policy[direction][0][peer][0]["podSelector"]["matchExpressions"] == excluded + import copy + for key in ("kubeletCIDRs", "apiServerCIDRs", "probePorts"): + invalid = copy.deepcopy(values) + invalid["networkPolicies"][key] = [] + render(invalid, failure="requires a non-empty networkPolicies." + key) + for cidr in ("", "10.0.0.1", "999.0.0.1/32", "10.0.0.1/33", "2001:::1/64", "2001:db8/64", "::/129"): + invalid = copy.deepcopy(values) + invalid["networkPolicies"]["apiServerCIDRs"] = [cidr] + render(invalid, failure="must be explicit IPv4/IPv6 CIDRs") + for cidr in ("::/0", "::1/128", "2001:db8::/64", "2001:db8:1:2:3:4:5:6/128"): + valid = copy.deepcopy(values) + valid["networkPolicies"]["apiServerCIDRs"] = [cidr] + render(valid) + for port in (0, 65536, 1.5, "all"): + invalid = copy.deepcopy(values) + invalid["networkPolicies"]["probePorts"][0]["port"] = port + render(invalid, failure="integer port") + render({"networkPolicies": {"strict": True, "enabled": False}}, failure="requires networkPolicies.enabled=true") + # Legacy/default installs still render the documented broad fallback. + fallback = one(render(), "NetworkPolicy", "allow-kubelet-probes")["spec"]["ingress"][0] + assert "ports" not in fallback and {p["ipBlock"]["cidr"] for p in fallback["from"]} == { + "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "169.254.0.0/16"} + + +if __name__ == "__main__": + main() diff --git a/deploy/kind/e2e.sh b/deploy/kind/e2e.sh index 8f5df5211..3cf78410c 100755 --- a/deploy/kind/e2e.sh +++ b/deploy/kind/e2e.sh @@ -27,6 +27,11 @@ ACTION="${1:-up}" CLUSTER="${2:-gameplane-e2e}" TAG="${3:-${GAMEPLANE_E2E_TAG:-e2e}}" PROFILE="${4:-default}" +CNI="${GAMEPLANE_E2E_CNI:-kindnet}" +case "${CNI}" in +kindnet | cilium) ;; +*) echo "unsupported E2E CNI: ${CNI}" >&2; exit 2 ;; +esac case "${PROFILE}" in default | postgres | management) ;; *) @@ -188,14 +193,21 @@ up) echo "removing pre-existing cluster ${CLUSTER} for a clean slate" kind delete cluster --name "${CLUSTER}" || true fi - echo "creating kind cluster ${CLUSTER} (${PROFILE})" + echo "creating kind cluster ${CLUSTER} (${PROFILE}, ${CNI})" + kind_networking="" + kind_wait=90s + if [ "${CNI}" = cilium ]; then + kind_networking=$'networking:\n disableDefaultCNI: true' + kind_wait=0s # Nodes become Ready after Cilium is installed below. + fi extra_nodes="" if [[ "${PROFILE}" == "management" ]]; then extra_nodes=$' - role: worker\n - role: worker' fi - kind create cluster --name "${CLUSTER}" --wait 90s --config - </dev/null + if [ "${CNI}" = cilium ]; then + # Pin the enforcing CNI; kindnet does not implement NetworkPolicy. + # https://docs.cilium.io/en/stable/installation/kind/ + helm upgrade --install cilium cilium --repo https://helm.cilium.io/ \ + --version 1.20.3 --namespace kube-system \ + --set image.pullPolicy=IfNotPresent --set ipam.mode=kubernetes \ + --set policyCIDRMatchMode=nodes \ + --set operator.replicas=1 --wait --timeout 5m + kubectl wait nodes --all --for=condition=Ready --timeout=120s + fi + install_metallb apply_metallb_pools @@ -363,6 +386,17 @@ EOF deployment/gameplane-test-fakeoidc --timeout=120s DB_SETS=() + NETWORK_SETS=() + if [ "${CNI}" = cilium ]; then + # Single-node IPv4 fixture: use actual endpoint/source addresses. + node_ip="$(kubectl get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}')" + api_ip="$(kubectl get endpoints kubernetes -n default -o jsonpath='{.subsets[0].addresses[0].ip}')" + NETWORK_SETS=(--set networkPolicies.strict=true + --set-string "networkPolicies.kubeletCIDRs[0]=${node_ip}/32" + --set-string "networkPolicies.apiServerCIDRs[0]=${api_ip}/32" + --set networkPolicies.probePorts[0].protocol=TCP + --set networkPolicies.probePorts[0].port=25566) + fi if [ "${PROFILE}" = postgres ]; then dsn="$(kubectl get secret gameplane-postgres --namespace gameplane-system \ -o jsonpath='{.data.dsn}' | base64 -d)" @@ -408,6 +442,7 @@ EOF --set "defaultModuleSource.enabled=false" \ --set "api.telemetry.receiver.enabled=true" \ ${DB_SETS[@]+"${DB_SETS[@]}"} \ + ${NETWORK_SETS[@]+"${NETWORK_SETS[@]}"} \ --wait --timeout 5m echo diff --git a/docs/module-authoring.md b/docs/module-authoring.md index 1141965bb..55b2436aa 100644 --- a/docs/module-authoring.md +++ b/docs/module-authoring.md @@ -12,6 +12,15 @@ materializes into an in-cluster `GameTemplate`. ## Quickstart: Authoring with `gp-module` CLI & Web Builder +For a validated wake proxy, add `sentinelIsolation: true` under the template's +`spec`. The operator permits that server's sentinel to reach only its own +advertised container ports, preserving each port's TCP/UDP/SCTP protocol. +Leave management ports unadvertised. A template with no advertised ports grants +no sentinel access. Omission preserves legacy behavior; adoption in the +GameplanePanel/module repository is separate from the operator upgrade. +Test player joins, status queries and sleep/wake cycles with an enforcing CNI +before publishing the opt-in. See [network policy migration](networking.md#networkpolicy-isolation-and-migration). + Gameplane provides a unified toolkit for creating, verifying, previewing, and packaging modules: - **Command-Line Interface (`gp-module`)**: First-class Go CLI with Makefile targets. diff --git a/docs/networking.md b/docs/networking.md index a8415fb72..fd3259012 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -2,6 +2,66 @@ Gameplane exposes game servers through Kubernetes Services, with optional integration with your cluster's load-balancer address manager to assign specific public addresses or address pools to each server. +## NetworkPolicy isolation and migration + +NetworkPolicies are additive. An allowance in one policy cannot be narrowed by +another policy. Enforcement also requires a CNI that implements NetworkPolicy. + +Modules may set `GameTemplate.spec.sentinelIsolation: true`. The operator then +labels the game and sentinel pod templates and creates two owned policies: +`-sentinel-ingress` and `-sentinel-egress`. They allow a sentinel +to reach only its own server's advertised container ports, with the declared +protocol. Zero advertised ports creates no allowance. Management ports must +remain `advertise: false`. The legacy chart policies exclude opted-in sources +and destinations. Templates without the field retain the existing broad +sentinel-to-game allowance; module maintainers can adopt it after validating +their proxy traffic. + +Upgrade the CRDs, operator and chart together before opting in a template. +Existing game pods roll out to acquire the isolation label; StatefulSet +selectors stay unchanged. Do not claim isolation until both game and sentinel +pods have the label and the per-server policies exist. Other administrator-added +policies can still grant wider access. Disabling chart NetworkPolicies or using +a non-enforcing CNI does not provide this boundary. + +Probe and API-server narrowing is a separate installation option: + +```yaml +networkPolicies: + enabled: true + strict: true + kubeletCIDRs: ["192.0.2.10/32"] # Replace with observed kubelet sources. + probePorts: + - {protocol: TCP, port: 25565} # Replace with all configured probe ports. + apiServerCIDRs: ["198.51.100.20/32"] # Replace with actual API endpoints. +``` + +Strict mode rejects missing/empty lists, malformed CIDRs and invalid probe +ports. It never supplies broad fallback ranges. API egress is limited to the +listed addresses on TCP 443/6443. Choose addresses based on your CNI's policy +evaluation point: service DNAT or source SNAT can change which address it sees. +Include every control-plane endpoint and kubelet source that can host a server; +update these lists when nodes or endpoints change. CNI host-traffic exceptions +may still apply. + +On Cilium, CIDR rules exclude node identities by default. For a self-hosted +API server on a node, configure `policyCIDRMatchMode=nodes` before using these +plain NetworkPolicy endpoint CIDRs, or provide equivalent Cilium-specific +API-server policy. The CI fixture uses nodes mode. See +[Cilium CIDR policy semantics](https://docs.cilium.io/en/stable/security/policy/layer3/#selecting-pods-or-nodes-with-cidr-ipblock) +and validate kubelet source handling for your Cilium configuration as well. + +Before enabling strict mode, inventory the actual endpoints and configured +game probes, observe their traffic in the CNI, then validate game readiness, +agent heartbeats, wake requests and API/operator agent access in a staging +namespace. The default profile retains RFC1918/link-local API egress and +unrestricted probe ports for compatibility; it does not isolate those paths. +Narrowing sentinel traffic alone cannot remove those separate grants. + +The dedicated CI `network-isolation` bucket installs pinned Cilium, exercises +real allowed/denied connections, and upgrades existing legacy templates. +The default kindnet fixture only verifies policy shape. + ## Address Pool Overview When a GameServer is exposed as a LoadBalancer (`spec.networking.expose: LoadBalancer`), it can request a specific external address or address pool. The operator translates this request to your cluster's load-balancer flavor so the same CRD works whether you run MetalLB, Cilium, or no address manager at all. diff --git a/hack/ci_scope.py b/hack/ci_scope.py index 45a1887a4..e887463dd 100644 --- a/hack/ci_scope.py +++ b/hack/ci_scope.py @@ -29,8 +29,8 @@ FORCE_ALL_PREFIXES = (".github/actions/",) # Mirrors the e2e-go matrix buckets in .github/workflows/ci.yaml. E2E_BUCKETS = ["operator", "api-auth", "api-roles", "api-rbac", "api-agent", "api-mods", - "telemetry", "optional-components", "management-placement"] -E2E_AMD64_ONLY = {"management-placement"} + "telemetry", "optional-components", "network-isolation", "management-placement"] +E2E_AMD64_ONLY = {"network-isolation", "management-placement"} SUITES_FILE = "test/e2e/suite-components.json" # Paths no Go e2e test reads. Their own jobs keep their paths-filter gates. NO_E2E_FILES = { diff --git a/hack/test_ci_scope.py b/hack/test_ci_scope.py index 3814677aa..03984e022 100644 --- a/hack/test_ci_scope.py +++ b/hack/test_ci_scope.py @@ -408,6 +408,17 @@ def select(self, *paths): """Return the e2e outputs for a partial-scope change to paths.""" return e2e_outputs(ROOT, list(paths), None) + def test_audit_network_isolation_has_a_run_pattern(self): + """Affected components and the owning test file select the audit bucket.""" + for path in ("operator/internal/controller/gameserver_controller.go", + "api/cmd/main.go", + "agent/cmd/main.go", + "test/e2e/network_isolation_e2e_test.go"): + with self.subTest(path=path): + values = self.select(path) + self.assertIn("TestNetworkIsolation_EnforcedAndUpgrade", + values["e2e-run"].get("network-isolation", "")) + def test_audit_management_placement_has_a_run_pattern(self): """Affected components and the owning test file select the audit bucket.""" for path in ("operator/cmd/main.go", diff --git a/operator/api/v1alpha1/gametemplate_types.go b/operator/api/v1alpha1/gametemplate_types.go index bea234aea..38a906125 100644 --- a/operator/api/v1alpha1/gametemplate_types.go +++ b/operator/api/v1alpha1/gametemplate_types.go @@ -89,6 +89,12 @@ type GameTemplateSpec struct { // +optional Ports []GamePort `json:"ports,omitempty"` + // SentinelIsolation restricts a wake sentinel to its own server's advertised + // ports. Templates opt in after module connectivity validation; absent or + // false retains the legacy namespace-wide sentinel allowances. + // +optional + SentinelIsolation bool `json:"sentinelIsolation,omitempty"` + // Storage describes the default persistent storage layout. // +optional Storage GameStorageSpec `json:"storage,omitempty"` diff --git a/operator/config/crd/gameplane.local_gametemplates.yaml b/operator/config/crd/gameplane.local_gametemplates.yaml index c6428c858..9cefe0a11 100644 --- a/operator/config/crd/gameplane.local_gametemplates.yaml +++ b/operator/config/crd/gameplane.local_gametemplates.yaml @@ -2113,6 +2113,12 @@ spec: minimum: 0 type: integer type: object + sentinelIsolation: + description: |- + SentinelIsolation restricts a wake sentinel to its own server's advertised + ports. Templates opt in after module connectivity validation; absent or + false retains the legacy namespace-wide sentinel allowances. + type: boolean storage: description: Storage describes the default persistent storage layout. properties: diff --git a/operator/internal/controller/gameserver_controller.go b/operator/internal/controller/gameserver_controller.go index 4a258af72..8b35168d5 100644 --- a/operator/internal/controller/gameserver_controller.go +++ b/operator/internal/controller/gameserver_controller.go @@ -268,6 +268,11 @@ func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, err } + if tmpl.Spec.SentinelIsolation && len(networkPolicyPortsFromTemplate(&tmpl)) == 0 { + return ctrl.Result{}, r.setPhase(ctx, &gs, + "sentinelIsolation requires at least one advertised port; advertise a game port or disable sentinelIsolation in the GameTemplate") + } + // Resolve spec.config against the template's configSchema before // touching any children: invalid config must fail loudly (mirroring // the missing-template path) instead of materializing a pod that @@ -306,6 +311,9 @@ func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) logger.Error(err, "reconcile ingress NetworkPolicy") return ctrl.Result{}, err } + if err := r.reconcileSentinelIsolation(ctx, &gs, &tmpl); err != nil { + return ctrl.Result{}, err + } if err := r.reconcileAgentTLS(ctx, &gs); err != nil { logger.Error(err, "reconcile agent TLS") return ctrl.Result{}, err @@ -1502,7 +1510,13 @@ func (r *GameServerReconciler) reconcileStatefulSet( ss.Spec.Replicas = &actualReplicas ss.Spec.ServiceName = gs.Name ss.Spec.Selector = &metav1.LabelSelector{MatchLabels: labels} - ss.Spec.Template.Labels = labels + ss.Spec.Template.Labels = make(map[string]string, len(labels)+1) + for key, value := range labels { + ss.Spec.Template.Labels[key] = value + } + if tmpl.Spec.SentinelIsolation { + ss.Spec.Template.Labels[sentinelIsolationLabel] = "true" + } // Stamp (or clear) the config fingerprint without touching // annotations other actors may have set on the pod template. ann := ss.Spec.Template.Annotations diff --git a/operator/internal/controller/gameserver_sentinel.go b/operator/internal/controller/gameserver_sentinel.go index 5415fbbf8..c2549fc75 100644 --- a/operator/internal/controller/gameserver_sentinel.go +++ b/operator/internal/controller/gameserver_sentinel.go @@ -230,6 +230,9 @@ func (r *GameServerReconciler) reconcileSentinel( sentinelLabel: sentinelValue, "app.kubernetes.io/instance": gs.Name, } + if tmpl.Spec.SentinelIsolation { + dep.Spec.Template.Labels[sentinelIsolationLabel] = "true" + } // Container ports: advertised ports only, mirroring buildGameContainer. ports := make([]corev1.ContainerPort, 0) diff --git a/operator/internal/controller/gameserver_wipe_inventory_test.go b/operator/internal/controller/gameserver_wipe_inventory_test.go index e4075147d..4805da30a 100644 --- a/operator/internal/controller/gameserver_wipe_inventory_test.go +++ b/operator/internal/controller/gameserver_wipe_inventory_test.go @@ -106,11 +106,36 @@ func (r *wipeInventoryReader) Get(ctx context.Context, key client.ObjectKey, obj func (r *wipeInventoryReader) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { if _, ok := list.(*corev1.PodList); ok { - r.podLists++ + options := (&client.ListOptions{}).ApplyOptions(opts) + // Wipe inventory deliberately reads every Pod in the namespace so + // unlabeled legacy workers remain visible. Filtered safety queries, + // such as sentinel opt-out drainage, are separate inventories. + if (options.LabelSelector == nil || options.LabelSelector.Empty()) && + (options.FieldSelector == nil || options.FieldSelector.Empty()) { + r.podLists++ + } } return r.Reader.List(ctx, list, opts...) } +func TestWipeInventoryReaderSeparatesFilteredPodQueries(t *testing.T) { + reader := &wipeInventoryReader{Reader: fake.NewClientBuilder().WithScheme(wipeScheme(t)).Build()} + for range 2 { + var pods corev1.PodList + if err := reader.List(t.Context(), &pods, client.InNamespace("games")); err != nil { + t.Fatal(err) + } + if err := reader.List(t.Context(), &pods, client.InNamespace("games"), client.MatchingLabels{ + "app.kubernetes.io/instance": "server", sentinelIsolationLabel: "true", + }); err != nil { + t.Fatal(err) + } + } + if reader.podLists != 2 { + t.Fatalf("duplicate unfiltered inventories must count independently of sentinel queries: got %d", reader.podLists) + } +} + func TestWipeInventoryIsSharedWithWorkloadMutation(t *testing.T) { s := wipeScheme(t) gs := wipeGameServer(false, "") diff --git a/operator/internal/controller/sentinel_isolation.go b/operator/internal/controller/sentinel_isolation.go new file mode 100644 index 000000000..c01e161f1 --- /dev/null +++ b/operator/internal/controller/sentinel_isolation.go @@ -0,0 +1,108 @@ +package controller + +import ( + "context" + "fmt" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +const sentinelIsolationLabel = "gameplane.local/sentinel-isolation" + +// The chart's legacy policies exclude opted-in pods on both sides. Policies +// are additive: these per-server grants cannot subtract a namespace-wide grant. +func (r *GameServerReconciler) reconcileSentinelIsolation(ctx context.Context, gs *gameplanev1alpha1.GameServer, tmpl *gameplanev1alpha1.GameTemplate) error { + if !tmpl.Spec.SentinelIsolation { + // Workload reconciliation removes the opt-in labels by rolling both + // pod templates. Keep both grants until every live pod has switched + // back to the legacy policies, including during a partial rollout. + reader := r.APIReader + if reader == nil { + reader = r.Client + } + // An empty Pod list alone is insufficient: an old workload template + // could still create labeled replacements after that observation. + var game appsv1.StatefulSet + if err := reader.Get(ctx, client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name}, &game); err != nil { + if !apierrors.IsNotFound(err) { + return err + } + } else if game.Spec.Template.Labels[sentinelIsolationLabel] == "true" || game.Status.ObservedGeneration < game.Generation || game.Status.CurrentRevision != game.Status.UpdateRevision { + return nil + } + var sentinel appsv1.Deployment + if err := reader.Get(ctx, client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-waker"}, &sentinel); err != nil { + if !apierrors.IsNotFound(err) { + return err + } + } else if sentinel.Spec.Template.Labels[sentinelIsolationLabel] == "true" || sentinel.Status.ObservedGeneration < sentinel.Generation || sentinel.Status.UpdatedReplicas != sentinel.Status.Replicas { + return nil + } + var pods corev1.PodList + if err := reader.List(ctx, &pods, client.InNamespace(gs.Namespace), client.MatchingLabels{ + "app.kubernetes.io/instance": gs.Name, sentinelIsolationLabel: "true", + }); err != nil { + return err + } + for i := range pods.Items { + name := pods.Items[i].Labels["app.kubernetes.io/name"] + if name == "gameplane-game" || name == "gameplane-waker" { + return nil + } + } + } + ports := networkPolicyPortsFromTemplate(tmpl) + for _, ingress := range []bool{true, false} { + suffix, target, peer := "egress", "gameplane-waker", "gameplane-game" + if ingress { + suffix, target, peer = "ingress", "gameplane-game", "gameplane-waker" + } + np := &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: gs.Name + "-sentinel-" + suffix, Namespace: gs.Namespace}} + if !tmpl.Spec.SentinelIsolation || len(ports) == 0 { + if err := r.Get(ctx, client.ObjectKeyFromObject(np), np); err != nil { + if client.IgnoreNotFound(err) != nil { + return err + } + continue + } + if metav1.IsControlledBy(np, gs) { + uid := np.UID + if err := client.IgnoreNotFound(r.Delete(ctx, np, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}})); err != nil { + return err + } + } + continue + } + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, np, func() error { + if np.ResourceVersion != "" && !metav1.IsControlledBy(np, gs) { + return fmt.Errorf("sentinel policy %s is not owned by this GameServer", np.Name) + } + np.Spec = networkingv1.NetworkPolicySpec{PodSelector: metav1.LabelSelector{MatchLabels: map[string]string{ + "app.kubernetes.io/name": target, "app.kubernetes.io/instance": gs.Name, + }}} + peers := []networkingv1.NetworkPolicyPeer{{PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{ + "app.kubernetes.io/name": peer, "app.kubernetes.io/instance": gs.Name, + }}}} + if ingress { + np.Spec.PolicyTypes = []networkingv1.PolicyType{networkingv1.PolicyTypeIngress} + np.Spec.Ingress = []networkingv1.NetworkPolicyIngressRule{{From: peers, Ports: ports}} + } else { + np.Spec.PolicyTypes = []networkingv1.PolicyType{networkingv1.PolicyTypeEgress} + np.Spec.Egress = []networkingv1.NetworkPolicyEgressRule{{To: peers, Ports: ports}} + } + return controllerutil.SetControllerReference(gs, np, r.Scheme) + }) + if err != nil { + return err + } + } + return nil +} diff --git a/operator/internal/controller/sentinel_isolation_test.go b/operator/internal/controller/sentinel_isolation_test.go new file mode 100644 index 000000000..d28f70f9e --- /dev/null +++ b/operator/internal/controller/sentinel_isolation_test.go @@ -0,0 +1,234 @@ +package controller + +import ( + "context" + "strings" + "testing" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +func TestSentinelIsolationPolicies(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = appsv1.AddToScheme(scheme) + if err := networkingv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := gameplanev1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + c := fake.NewClientBuilder().WithScheme(scheme).Build() + r := &GameServerReconciler{Client: c, Scheme: scheme} + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "alpha", Namespace: "games", UID: "original"}} + tmpl := singlePortTemplate(true, "minecraft") + tmpl.Spec.SentinelIsolation = true + tmpl.Spec.Ports = append(tmpl.Spec.Ports, + gameplanev1alpha1.GamePort{Name: "query", ContainerPort: 19132, Protocol: corev1.ProtocolUDP, Advertise: true}, + gameplanev1alpha1.GamePort{Name: "management", ContainerPort: 25575, Protocol: corev1.ProtocolTCP}) + if err := r.reconcileSentinelIsolation(t.Context(), gs, tmpl); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name, target, peer string + ingress bool + }{ + {"alpha-sentinel-ingress", "gameplane-game", "gameplane-waker", true}, + {"alpha-sentinel-egress", "gameplane-waker", "gameplane-game", false}, + } { + var policy networkingv1.NetworkPolicy + if err := c.Get(t.Context(), client.ObjectKey{Namespace: "games", Name: tc.name}, &policy); err != nil { + t.Fatal(err) + } + if !metav1.IsControlledBy(&policy, gs) { + t.Fatal("missing server ownership") + } + selector := policy.Spec.PodSelector.MatchLabels + if selector["app.kubernetes.io/name"] != tc.target || selector["app.kubernetes.io/instance"] != "alpha" { + t.Fatalf("wrong target: %v", selector) + } + var peers []networkingv1.NetworkPolicyPeer + var ports []networkingv1.NetworkPolicyPort + if tc.ingress { + if len(policy.Spec.Ingress) != 1 || len(policy.Spec.Egress) != 0 { + t.Fatal("wrong ingress policy shape") + } + peers, ports = policy.Spec.Ingress[0].From, policy.Spec.Ingress[0].Ports + } else { + if len(policy.Spec.Egress) != 1 || len(policy.Spec.Ingress) != 0 { + t.Fatal("wrong egress policy shape") + } + peers, ports = policy.Spec.Egress[0].To, policy.Spec.Egress[0].Ports + } + if len(peers) != 1 || peers[0].NamespaceSelector != nil || peers[0].IPBlock != nil || peers[0].PodSelector.MatchLabels["app.kubernetes.io/name"] != tc.peer || peers[0].PodSelector.MatchLabels["app.kubernetes.io/instance"] != "alpha" { + t.Fatalf("cross-server peer grant: %+v", peers) + } + if len(ports) != 2 || ports[0].Port.IntVal != 25565 || *ports[0].Protocol != corev1.ProtocolTCP || ports[1].Port.IntVal != 19132 || *ports[1].Protocol != corev1.ProtocolUDP { + t.Fatalf("wrong protocol/port tuples: %+v", ports) + } + } + // Zero advertised ports removes both grants instead of opening all ports. + tmpl.Spec.Ports = nil + if err := r.reconcileSentinelIsolation(t.Context(), gs, tmpl); err != nil { + t.Fatal(err) + } + var policies networkingv1.NetworkPolicyList + if err := c.List(t.Context(), &policies); err != nil || len(policies.Items) != 0 { + t.Fatalf("zero-port grants remain: %v %+v", err, policies.Items) + } + // Absent/default false preserves legacy behavior and creates no policies. + tmpl = singlePortTemplate(true, "minecraft") + if err := r.reconcileSentinelIsolation(t.Context(), gs, tmpl); err != nil { + t.Fatal(err) + } + if err := c.List(t.Context(), &policies); err != nil || len(policies.Items) != 0 { + t.Fatalf("legacy template changed: %v %+v", err, policies.Items) + } +} + +func TestSentinelIsolationDoesNotDeleteForeignPolicy(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = appsv1.AddToScheme(scheme) + _ = networkingv1.AddToScheme(scheme) + _ = gameplanev1alpha1.AddToScheme(scheme) + foreign := &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "alpha-sentinel-ingress", Namespace: "games"}} + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(foreign).Build() + r := &GameServerReconciler{Client: c, Scheme: scheme} + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "alpha", Namespace: "games", UID: "original"}} + if err := r.reconcileSentinelIsolation(t.Context(), gs, singlePortTemplate(false, "none")); err != nil { + t.Fatal(err) + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(foreign), foreign); err != nil { + t.Fatal("deleted foreign policy", err) + } +} + +type staleSentinelPodClient struct{ client.Client } + +func (c staleSentinelPodClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + if pods, ok := list.(*corev1.PodList); ok { + pods.Items = nil + return nil + } + return c.Client.List(ctx, list, opts...) +} + +func TestSentinelIsolationOptOutWaitsForBothLivePodLabels(t *testing.T) { + scheme := testScheme(t) + _ = networkingv1.AddToScheme(scheme) + c := fake.NewClientBuilder().WithScheme(scheme).Build() + r := &GameServerReconciler{Client: staleSentinelPodClient{c}, APIReader: c, Scheme: scheme} + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "alpha", Namespace: "games", UID: "original"}} + tmpl := singlePortTemplate(true, "minecraft") + tmpl.Spec.SentinelIsolation = true + if err := r.reconcileSentinelIsolation(t.Context(), gs, tmpl); err != nil { + t.Fatal(err) + } + pods := []*corev1.Pod{ + {ObjectMeta: metav1.ObjectMeta{Name: "alpha-0", Namespace: gs.Namespace, Labels: map[string]string{"app.kubernetes.io/name": "gameplane-game", "app.kubernetes.io/instance": gs.Name, sentinelIsolationLabel: "true"}}}, + {ObjectMeta: metav1.ObjectMeta{Name: "alpha-waker", Namespace: gs.Namespace, Labels: map[string]string{"app.kubernetes.io/name": "gameplane-waker", "app.kubernetes.io/instance": gs.Name, sentinelIsolationLabel: "true"}}}, + } + for _, pod := range pods { + if err := c.Create(t.Context(), pod); err != nil { + t.Fatal(err) + } + } + tmpl.Spec.SentinelIsolation = false + for remaining := 2; remaining >= 0; remaining-- { + if err := r.reconcileSentinelIsolation(t.Context(), gs, tmpl); err != nil { + t.Fatal(err) + } + var policies networkingv1.NetworkPolicyList + if err := c.List(t.Context(), &policies); err != nil { + t.Fatal(err) + } + want := 2 + if remaining == 0 { + want = 0 + } + if len(policies.Items) != want { + t.Fatalf("%d labeled pods remain: want %d grants, got %d", remaining, want, len(policies.Items)) + } + if remaining > 0 { + pod := pods[2-remaining] + delete(pod.Labels, sentinelIsolationLabel) + if err := c.Update(t.Context(), pod); err != nil { + t.Fatal(err) + } + } + } +} + +func TestSentinelIsolationRejectsNoAdvertisedPortsBeforeCreatingWorkloads(t *testing.T) { + scheme := testScheme(t) + _ = batchv1.AddToScheme(scheme) + tmpl := singlePortTemplate(false, "none") + tmpl.Name = "isolated" + tmpl.Spec.SentinelIsolation = true + tmpl.Spec.Ports = []gameplanev1alpha1.GamePort{{Name: "management", ContainerPort: 25575, Protocol: corev1.ProtocolTCP}} + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "alpha", Namespace: "games", UID: "original"}} + gs.Spec.TemplateRef.Name = tmpl.Name + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(gs).WithObjects(gs, tmpl).Build() + r := &GameServerReconciler{Client: c, APIReader: c, Scheme: scheme} + if _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(gs)}); err != nil { + t.Fatal(err) + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), gs); err != nil { + t.Fatal(err) + } + condition := meta.FindStatusCondition(gs.Status.Conditions, "Ready") + if gs.Status.Phase != gameplanev1alpha1.GameServerPhaseFailed || condition == nil || condition.Status != metav1.ConditionFalse || !strings.Contains(condition.Message, "sentinelIsolation requires at least one advertised port") { + t.Fatalf("missing actionable isolation failure: %+v", gs.Status) + } + var workloads appsv1.StatefulSetList + if err := c.List(t.Context(), &workloads); err != nil || len(workloads.Items) != 0 { + t.Fatalf("invalid template created workloads: %v %+v", err, workloads) + } +} + +func TestSentinelIsolationOptOutWaitsForWorkloadControllers(t *testing.T) { + scheme := testScheme(t) + _ = networkingv1.AddToScheme(scheme) + for _, test := range []struct { + name string + object client.Object + }{ + {"game-old-template", &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "alpha", Namespace: "games"}, Spec: appsv1.StatefulSetSpec{Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{sentinelIsolationLabel: "true"}}}}}}, + {"game-unobserved", &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "alpha", Namespace: "games", Generation: 2}, Status: appsv1.StatefulSetStatus{ObservedGeneration: 1}}}, + {"game-old-revision", &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "alpha", Namespace: "games"}, Status: appsv1.StatefulSetStatus{CurrentRevision: "old", UpdateRevision: "new"}}}, + {"sentinel-old-template", &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "alpha-waker", Namespace: "games"}, Spec: appsv1.DeploymentSpec{Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{sentinelIsolationLabel: "true"}}}}}}, + {"sentinel-unobserved", &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "alpha-waker", Namespace: "games", Generation: 2}, Status: appsv1.DeploymentStatus{ObservedGeneration: 1}}}, + {"sentinel-old-replicas", &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "alpha-waker", Namespace: "games"}, Status: appsv1.DeploymentStatus{Replicas: 2, UpdatedReplicas: 1}}}, + } { + t.Run(test.name, func(t *testing.T) { + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(test.object).Build() + r := &GameServerReconciler{Client: c, APIReader: c, Scheme: scheme} + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "alpha", Namespace: "games", UID: "original"}} + tmpl := singlePortTemplate(true, "minecraft") + tmpl.Spec.SentinelIsolation = true + if err := r.reconcileSentinelIsolation(t.Context(), gs, tmpl); err != nil { + t.Fatal(err) + } + tmpl.Spec.SentinelIsolation = false + if err := r.reconcileSentinelIsolation(t.Context(), gs, tmpl); err != nil { + t.Fatal(err) + } + var policies networkingv1.NetworkPolicyList + if err := c.List(t.Context(), &policies); err != nil || len(policies.Items) != 2 { + t.Fatalf("old workload could recreate labeled pods without grants: %v %+v", err, policies.Items) + } + }) + } +} diff --git a/operator/specs.md b/operator/specs.md index 3085dac8c..3492b29b7 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -180,6 +180,7 @@ Primary reconcilers register with the manager in `cmd/main.go` and handle CRD li - Graceful stop: if template has Lifecycle.Stop sequence, orchestrate via agent before scaling down. - Node affinity: honor spec.nodeSelector and spec.affinity preferences. - Ingress NetworkPolicy: enforce per-template advertised ports, admit CIDRs from `--game-ingress-from-cidr`. + - Sentinel NetworkPolicies: `GameTemplate.spec.sentinelIsolation` defaults false. When true, reconcile owned `-sentinel-ingress` and `-sentinel-egress` policies joining only the same namespace/server instance on advertised protocol/port tuples. An opted-in template with zero advertised ports fails before child reconciliation with an actionable `Ready=False` condition; no unrestricted grant is created. Set `gameplane.local/sentinel-isolation=true` on both pod templates, without changing immutable workload selectors. On opt-out, roll the labels off both workloads and retain the per-server grants until uncached reads confirm both controllers observed the unlabeled templates and completed their rollouts, and neither game nor sentinel pods still carry the label. The matching chart excludes opted-in sources and destinations from legacy grants; other policies remain additive. - Fixed-name child cleanup: Delete fixed-name objects (`-config` Secret, `-files` Secret, `-rcon` Secret, `-auto` BackupSchedule, `-wipe` Job) only when the GameServer controls them via OwnerReference; unowned objects with these names stay in place (same rule as game-ingress NetworkPolicy). Wipe cleanup carries the verified Job UID as a delete precondition; a foreign Job cannot acknowledge a pending wipe or resume its server. - **Conflict requeue:** When reconcileStatefulSet or reconcileWipe encounters an apierrors.IsConflict error (concurrent update races), requeue without error logging — conflicts are expected and transient. The Module, NetworkCapture and Backup reconcilers apply the same rule to every error they return, via `requeueOnConflict` (`conflict.go`). - Load-balancer address management: translate spec.networking.addressPool / .address preferences onto the Service based on the cluster's address-manager flavor (MetalLB, Cilium, or none), report assignment status via the AddressAssignment condition. diff --git a/test/e2e/buckets.sh b/test/e2e/buckets.sh index b803e2f42..66631bf1e 100755 --- a/test/e2e/buckets.sh +++ b/test/e2e/buckets.sh @@ -414,8 +414,12 @@ EOF unbucketed() { :; } +bucket_network_isolation() { + printf '%s\n' TestNetworkIsolation_EnforcedAndUpgrade TestNetworkIsolation_ProbeOutput +} + bucket_names() { - printf '%s\n' operator api-auth api-roles api-rbac api-agent api-mods ratelimit bot-fast bot-heavy multicluster upgrade telemetry capture-overhead optional-components management-placement + printf '%s\n' operator api-auth api-roles api-rbac api-agent api-mods ratelimit bot-fast bot-heavy multicluster upgrade telemetry capture-overhead optional-components network-isolation management-placement } list_bucket() { @@ -434,6 +438,7 @@ list_bucket() { telemetry) bucket_telemetry ;; capture-overhead) bucket_capture_overhead ;; optional-components) bucket_optional_components ;; + network-isolation) bucket_network_isolation ;; management-placement) bucket_management_placement ;; *) echo "unknown bucket: $1 (known: $(bucket_names | tr '\n' ' '))" >&2 diff --git a/test/e2e/ci_selection_e2e_test.go b/test/e2e/ci_selection_e2e_test.go index a52268ad2..8cccdf589 100644 --- a/test/e2e/ci_selection_e2e_test.go +++ b/test/e2e/ci_selection_e2e_test.go @@ -22,7 +22,7 @@ import ( // ciSelectionBuckets mirrors E2E_BUCKETS in hack/ci_scope.py (the e2e-go matrix). var ciSelectionBuckets = []string{ "operator", "api-auth", "api-roles", "api-rbac", "api-agent", "api-mods", - "telemetry", "optional-components", "management-placement", + "telemetry", "optional-components", "network-isolation", "management-placement", } // ciSelection is the decoded output of one `ci_scope.py e2e` run. diff --git a/test/e2e/network_isolation_e2e_test.go b/test/e2e/network_isolation_e2e_test.go new file mode 100644 index 000000000..412b9b6ab --- /dev/null +++ b/test/e2e/network_isolation_e2e_test.go @@ -0,0 +1,230 @@ +//go:build e2e + +package e2e + +import ( + "context" + "fmt" + "reflect" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" +) + +// This bucket uses Cilium, not kindnet. Exercise the effective union of all +// installed policies, including an existing legacy server's opt-in rollout. +func TestNetworkIsolation_EnforcedAndUpgrade(t *testing.T) { + ctx := t.Context() + const ns = "gameplane-games" + if _, err := envInstance.K8s.AppsV1().DaemonSets("kube-system").Get(ctx, "cilium", metav1.GetOptions{}); err != nil { + t.Fatalf("network-isolation bucket requires the enforcing Cilium fixture: %v", err) + } + stamp := time.Now().UnixNano() + legacyTemplate := fmt.Sprintf("net-legacy-%d", stamp) + isolatedTemplate := fmt.Sprintf("net-isolated-%d", stamp) + isolationTemplate(t, legacyTemplate, false) + isolationTemplate(t, isolatedTemplate, true) + a, b, c := fmt.Sprintf("net-a-%d", stamp), fmt.Sprintf("net-b-%d", stamp), fmt.Sprintf("net-c-%d", stamp) + for _, pair := range [][2]string{{a, legacyTemplate}, {b, legacyTemplate}, {c, isolatedTemplate}} { + applyBusyboxGameServer(t, ns, pair[0], pair[1]) + } + aIP := isolationGameIP(t, a, false) + bIP := isolationGameIP(t, b, false) + cIP := isolationGameIP(t, c, true) + probeA := isolationProbe(t, a, false) + probeC := isolationProbe(t, c, true) + + // Compatibility: absent-field templates keep their existing broad grant. + isolationHTTP(t, probeA, bIP, 25575, true) + isolationHTTP(t, probeC, cIP, 25566, true) + isolationHTTP(t, probeC, cIP, 25575, false) + isolationHTTP(t, probeC, aIP, 25566, false) + isolationHTTP(t, probeA, cIP, 25566, false) + + ss, err := envInstance.K8s.AppsV1().StatefulSets(ns).Get(ctx, a, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + oldSelector := ss.Spec.Selector.DeepCopy() + _, err = envInstance.Dyn.Resource(gameTemplateGVR).Patch(ctx, legacyTemplate, types.MergePatchType, + []byte(`{"spec":{"sentinelIsolation":true}}`), metav1.PatchOptions{}) + if err != nil { + t.Fatal(err) + } + aIP, bIP = isolationGameIP(t, a, true), isolationGameIP(t, b, true) + ss, err = envInstance.K8s.AppsV1().StatefulSets(ns).Get(ctx, a, metav1.GetOptions{}) + if err != nil || !reflect.DeepEqual(oldSelector, ss.Spec.Selector) { + t.Fatalf("opt-in must preserve the immutable StatefulSet selector: %v", err) + } + _, err = envInstance.K8s.CoreV1().Pods(ns).Patch(ctx, probeA, types.MergePatchType, + []byte(`{"metadata":{"labels":{"gameplane.local/sentinel-isolation":"true"}}}`), metav1.PatchOptions{}) + if err != nil { + t.Fatal(err) + } + isolationHTTP(t, probeA, aIP, 25566, true) + isolationHTTP(t, probeA, aIP, 25575, false) + isolationHTTP(t, probeA, bIP, 25566, false) + isolationHTTP(t, probeA, bIP, 25575, false) + + // Kubelet HTTP readiness and agent heartbeats still work with strict CIDRs. + // Also exercise an actual API -> agent mTLS call through its Service. + envInstance.BootstrapAdmin(t, adminUsername, adminPassword) + cli := envInstance.APIClient(t, adminUsername, adminPassword) + defer cli.Close() + requireAgentReady(t, ns, a) + waitAgentReachable(t, cli, a) + envInstance.Eventually(t, 90*time.Second, func() (bool, string) { + gs, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Get(ctx, a, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + hb, _, _ := unstructured.NestedString(gs.Object, "status", "agent", "lastHeartbeat") + stamp, err := time.Parse(time.RFC3339, hb) + return err == nil && time.Since(stamp) < 45*time.Second, "waiting for fresh agent heartbeat" + }) +} + +func isolationTemplate(t *testing.T, name string, isolated bool) { + t.Helper() + spec := map[string]any{ + "displayName": name, "game": "busybox", "version": "1", "image": "busybox:1.36", + "command": []any{"sh", "-c", "mkdir -p /tmp/http; echo reachable >/tmp/http/index.html; httpd -p 25575 -h /tmp/http; exec httpd -f -p 25566 -h /tmp/http"}, + "ports": []any{ + map[string]any{"name": "game", "containerPort": int64(25566), "protocol": "TCP", "advertise": true}, + map[string]any{"name": "management", "containerPort": int64(25575), "protocol": "TCP", "advertise": false}, + }, + "probes": map[string]any{"readiness": map[string]any{ + "httpGet": map[string]any{"path": "/", "port": int64(25566)}, "periodSeconds": int64(2), + }}, + } + if isolated { + spec["sentinelIsolation"] = true + } + obj := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "gameplane.local/v1alpha1", "kind": "GameTemplate", + "metadata": map[string]any{"name": name}, "spec": spec, + }} + if _, err := envInstance.Dyn.Resource(gameTemplateGVR).Create(t.Context(), obj, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = envInstance.Dyn.Resource(gameTemplateGVR).Delete(context.Background(), name, metav1.DeleteOptions{}) + }) +} + +func isolationGameIP(t *testing.T, name string, isolated bool) string { + t.Helper() + var ip string + envInstance.Eventually(t, 3*time.Minute, func() (bool, string) { + pod, err := envInstance.K8s.CoreV1().Pods("gameplane-games").Get(t.Context(), name+"-0", metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + if pod.DeletionTimestamp != nil || (pod.Labels["gameplane.local/sentinel-isolation"] == "true") != isolated { + return false, "waiting for rollout with expected isolation label" + } + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue && pod.Status.PodIP != "" { + ip = pod.Status.PodIP + return true, "" + } + } + return false, "waiting for HTTP readiness probe" + }) + return ip +} + +// A synthetic client carries the exact real sentinel selectors. It exposes no +// listener and owns no Service/Deployment, so the running game is unaffected. +func isolationProbe(t *testing.T, server string, isolated bool) string { + t.Helper() + const ns = "gameplane-games" + name := server + "-probe" + labels := map[string]string{"app.kubernetes.io/name": "gameplane-waker", "app.kubernetes.io/instance": server} + if isolated { + labels["gameplane.local/sentinel-isolation"] = "true" + } + noToken := false + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: labels}, + Spec: corev1.PodSpec{AutomountServiceAccountToken: &noToken, + Containers: []corev1.Container{{Name: "probe", Image: "busybox:1.36", Command: []string{"sleep", "3600"}}}}, + } + if _, err := envInstance.K8s.CoreV1().Pods(ns).Create(t.Context(), pod, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = envInstance.K8s.CoreV1().Pods(ns).Delete(context.Background(), name, metav1.DeleteOptions{}) + }) + envInstance.Eventually(t, 90*time.Second, func() (bool, string) { + got, err := envInstance.K8s.CoreV1().Pods(ns).Get(t.Context(), name, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + return got.Status.Phase == corev1.PodRunning, "waiting for probe pod" + }) + return name +} + +func isolationHTTP(t *testing.T, source, ip string, port int, allowed bool) { + t.Helper() + // Every exec must succeed regardless of network outcome. Otherwise a broken + // kubectl/CNI fixture could masquerade as the expected denied connection. + target := fmt.Sprintf("http://%s:%d/", ip, port) + envInstance.Eventually(t, 45*time.Second, func() (bool, string) { + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + out, err := envInstance.Kubectl(ctx, "exec", "-n", "gameplane-games", source, "--", + "sh", "-c", `if wget -q -T 2 -O /dev/null "$1"; then echo ALLOWED; else echo DENIED; fi`, "probe", target) + if err != nil { + return false, "probe execution failed: " + err.Error() + out + } + got := isolationProbeVerdict(out) + want := "DENIED" + if allowed { + want = "ALLOWED" + } + return got == want, fmt.Sprintf("%s -> %s: got %q, want %s; output: %q", source, target, got, want, out) + }) +} + +// Kubectl combines stdout and stderr, so wget's expected timeout diagnostic +// can precede the result. Require one exact marker; never accept missing or +// contradictory results, and retain the full output in the failure message. +func isolationProbeVerdict(out string) string { + var verdict string + for line := range strings.SplitSeq(out, "\n") { + line = strings.TrimSpace(line) + if line != "ALLOWED" && line != "DENIED" { + continue + } + if verdict != "" { + return "" + } + verdict = line + } + return verdict +} + +func TestNetworkIsolation_ProbeOutput(t *testing.T) { + t.Parallel() + for _, tc := range []struct{ name, output, want string }{ + {"allowed", "ALLOWED\n", "ALLOWED"}, + {"denied with wget timeout", "wget: download timed out\nDENIED\n", "DENIED"}, + {"denied with carriage return", "wget: connection refused\r\nDENIED\r\n", "DENIED"}, + {"diagnostic only", "wget: download timed out\n", ""}, + {"marker substring", "unexpected ALLOWED response\n", ""}, + {"contradictory markers", "ALLOWED\nDENIED\n", ""}, + {"duplicate markers", "DENIED\nDENIED\n", ""}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := isolationProbeVerdict(tc.output); got != tc.want { + t.Fatalf("probe verdict = %q, want %q for %q", got, tc.want, tc.output) + } + }) + } +} diff --git a/test/e2e/specs.md b/test/e2e/specs.md index 4fb60a76f..4cfa9d0d3 100644 --- a/test/e2e/specs.md +++ b/test/e2e/specs.md @@ -6,6 +6,13 @@ ## Purpose +The amd64 `network-isolation` bucket requires the Cilium fixture +(`GAMEPLANE_E2E_CNI=cilium`) and strict chart CIDRs derived from the single-node +cluster. It tests actual legacy versus isolated sentinel connections, an +existing template's opt-in rollout without selector changes, HTTP kubelet +readiness, fresh agent heartbeats, and API-to-agent mTLS access. It uses one +admin login and runs separately from the kindnet shape-only checks. + The `test/e2e` module is Gameplane's end-to-end test suite: it stands up a real (kind-based) Kubernetes cluster, installs the Helm chart, and drives the operator/api/agent/sentinel stack as real components rather than mocks — CRDs applied through the typed/dynamic client, HTTP calls through a real port-forward, RCON/wire-protocol joins against real game server containers. Where the other modules' test tiers (unit, envtest) prove a component's own logic in isolation, this module proves the whole product works together on a cluster shaped like the ones users actually run. It was one of three `go.work` modules without CI lint coverage until this feature (`api`, `agent`, and `test/e2e` — see FR-001); all 13 `go.work` members (`netguard`, `gameaction`, `gameproto`, `operator`, `agent`, `api`, `sentinel`, `audit-syslog-bridge`, `telemetry-receiver`, `mcp-server`, `svcutil`, `tunnel`, and `test/e2e`) are golangci-lint gated in CI now — see the `lint` job's matrix in `.github/workflows/ci.yaml`. `test/e2e` lints with `--build-tags=e2e`, same tier as `operator`/`api`'s `--build-tags=envtest`, run as their own step so the catch-all lint step excludes all three and nothing is linted twice. @@ -69,10 +76,10 @@ The golangci-lint job mirrors this: it must be invoked with `--build-tags=e2e` o ## The bucket contract — the module's most important invariant -The `management-placement` audit bucket is registered in the change-based selector, -with explicit component mapping and an amd64-only selection-summary row. Its +The `network-isolation` and `management-placement` audit buckets are registered in the change-based selector, +with explicit component mapping and amd64-only selection-summary rows. Each matrix leg skips cluster setup only when its selected test pattern is empty; -full-scope and affected-component runs must include it. +full-scope and affected-component runs must include the affected buckets. `buckets.sh` is the single source of truth for how CI's e2e suite is sharded into parallel jobs (`operator`, `api-auth`, `api-roles`, `api-rbac`, `api-agent`, `api-mods`, `ratelimit`, `bot-fast`, `bot-heavy`, `multicluster`, `upgrade`). Each bucket is a flat list of **exact Go function names** (e.g. `TestGameServer_MinecraftJavaBot_Joined`), and `buckets.sh regex ` turns that list into an **anchored** `^(A|B|...)$` regex passed to `go test -run`. diff --git a/test/e2e/suite-components.json b/test/e2e/suite-components.json index 238af8f04..de4a034e7 100644 --- a/test/e2e/suite-components.json +++ b/test/e2e/suite-components.json @@ -16,6 +16,7 @@ "test/e2e/suite-components.json" ], "suites": [ + { "tests": "^TestNetworkIsolation_", "components": ["operator", "api", "agent"] }, { "tests": "^TestManagementPlacementAndDisruption$", "components": ["operator", "api", "agent"] }, { "tests": "^TestHelmInstall_", "components": ["*"] }, { "tests": "^TestCRD_Validation_", "components": ["operator"] },