diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0dda76149..440c9e769 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1824,17 +1824,21 @@ jobs: # (operator does zero logins and is wait-dominated, so it runs # wide; the api buckets stay within the login limiter's budget). # The arm64 job below uses the same steps and per-bucket settings. - bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, management-placement] + bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, wake-hostport, management-placement] arch: [amd64] include: - bucket: operator parallel: 8 test_timeout: 20m job_timeout: 60 # Boots a kind cluster, installs the chart, runs the bucket's suite. + - bucket: wake-hostport + parallel: 1 + test_timeout: 15m + job_timeout: 60 # Boots two-node kind, installs the chart and exercises repeated Hostport sleep/wake handoffs. - bucket: management-placement parallel: 1 test_timeout: 15m - job_timeout: 60 + job_timeout: 60 # Boots management kind, installs the chart and checks placement/disruption behavior. - bucket: api-auth parallel: 4 test_timeout: 15m @@ -1892,7 +1896,7 @@ jobs: - name: bring up cluster + helm install if: env.RUN_REGEX != '' || env.TAIL_REGEX != '' env: - GAMEPLANE_E2E_PROFILE: ${{ matrix.bucket == 'management-placement' && 'management' || 'default' }} + GAMEPLANE_E2E_PROFILE: ${{ matrix.bucket == 'wake-hostport' && 'two-node' || matrix.bucket == 'management-placement' && 'management' || 'default' }} run: ./deploy/kind/e2e.sh up "$CLUSTER" e2e "$GAMEPLANE_E2E_PROFILE" - name: run go e2e (${{ matrix.bucket }}) if: env.RUN_REGEX != '' @@ -2697,7 +2701,7 @@ jobs: 'go-e2e-unit': (n) => n === 'go (test/e2e)', 'api-postgres': (n) => n === 'api (postgres)', 'e2e-buckets': (n) => n === 'e2e bucket coverage', - 'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|management-placement) \//.test(n) && n.includes('/ amd64 '), + 'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|wake-hostport|management-placement) \//.test(n) && n.includes('/ amd64 '), 'e2e-go-arm64': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//.test(n) && n.includes('/ arm64 '), 'e2e-multicluster': (n) => n.startsWith('e2e multicluster /') && n.includes('/ amd64 '), 'e2e-multicluster-arm64': (n) => n.startsWith('e2e multicluster /') && n.includes('/ arm64 '), @@ -2850,7 +2854,7 @@ jobs: const bucketSet = new Set(); for (const j of apiJobs) { const n = j.name || ''; - const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|management-placement) \//); + const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|wake-hostport|management-placement) \//); if (m) bucketSet.add(m[1]); if (/^e2e multicluster \//.test(n)) bucketSet.add('multicluster'); if (/^e2e upgrade \//.test(n)) bucketSet.add('upgrade'); diff --git a/charts/gameplane/templates/operator.yaml b/charts/gameplane/templates/operator.yaml index f472872a2..584cd2f86 100644 --- a/charts/gameplane/templates/operator.yaml +++ b/charts/gameplane/templates/operator.yaml @@ -17,6 +17,10 @@ metadata: name: {{ .Release.Name }}-operator-watch labels: {{- include "gameplane.labels" . | nindent 4 }} rules: + # Direct node identity/readiness checks for durable Hostport wake endpoints. + - apiGroups: [""] + resources: [nodes] + verbs: [get] - apiGroups: ["gameplane.local"] resources: [gameservers, gametemplates, backups, backupschedules, restores, modules, modulesources, networkcaptures] verbs: [get, list, watch] diff --git a/deploy/kind/e2e.sh b/deploy/kind/e2e.sh index 8f5df5211..d06330c0a 100755 --- a/deploy/kind/e2e.sh +++ b/deploy/kind/e2e.sh @@ -6,7 +6,8 @@ # deploy/kind/e2e.sh down [cluster-name] # # Diffs from dev-up (deploy/kind/up.sh): -# - Single-node by default; management profile adds two scheduling workers. +# - Single-node by default; two-node adds one worker for Hostport handoff, +# and management adds two workers for scheduling and disruption checks. # - Skips ingress-nginx (the dashboard isn't exercised here). # - Loads pre-built gameplane-test/{operator,api,agent,sentinel,capture-sidecar,telemetry-receiver,fakeoidc}: # images (and gameplane-test/gameprobe: when present). @@ -28,9 +29,9 @@ CLUSTER="${2:-gameplane-e2e}" TAG="${3:-${GAMEPLANE_E2E_TAG:-e2e}}" PROFILE="${4:-default}" case "${PROFILE}" in -default | postgres | management) ;; +default | postgres | two-node | management) ;; *) - echo "unknown profile '${PROFILE}' (want default, postgres or management)" >&2 + echo "unknown profile '${PROFILE}' (want default, postgres, two-node or management)" >&2 exit 2 ;; esac @@ -190,7 +191,9 @@ up) fi echo "creating kind cluster ${CLUSTER} (${PROFILE})" extra_nodes="" - if [[ "${PROFILE}" == "management" ]]; then + if [[ "${PROFILE}" == "two-node" ]]; then + extra_nodes=" - role: worker" + elif [[ "${PROFILE}" == "management" ]]; then extra_nodes=$' - role: worker\n - role: worker' fi kind create cluster --name "${CLUSTER}" --wait 90s --config - < time.Second { + t.Fatal("wake PATCH exceeded its own timeout") + } +} + +type contextBoundWake struct{ started chan struct{} } + +func (w *contextBoundWake) RequestWake(ctx context.Context) error { + close(w.started) + <-ctx.Done() + return ctx.Err() +} + +func TestJoinBudgetIncludesWakeRequest(t *testing.T) { + server, client := net.Pipe() + defer server.Close() + defer client.Close() + wake := &contextBoundWake{started: make(chan struct{})} + done := make(chan struct{}) + go func() { + defer close(done) + handleJoin(t.Context(), server, bufio.NewReader(server), wake, joinParams{upstreamAddr: "127.0.0.1:1", deadline: 25 * time.Millisecond, pollInterval: time.Millisecond}) + }() + <-wake.started + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("join deadline did not bound wake request") + } +} + +func TestUDPWakeWorkerCoalescesAndStops(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + wake := &contextBoundWake{started: make(chan struct{})} + worker := newUDPWakeWorker(ctx, wake) + worker.request() + <-wake.started + for range 1000 { + worker.request() + } + if len(worker.requests) > 1 { + t.Fatal("unbounded UDP wake backlog") + } + cancel() + select { + case <-worker.done: + case <-time.After(time.Second): + t.Fatal("UDP wake worker did not stop") + } +} + +func TestSuccessfulGameplayOutlivesJoinBudget(t *testing.T) { + upstream, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer upstream.Close() + accepted := make(chan struct{}) + go func() { + conn, acceptErr := upstream.Accept() + if acceptErr != nil { + return + } + defer conn.Close() + close(accepted) + _, _ = io.Copy(conn, conn) + }() + server, player := tcpPipe(t) + defer player.Close() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + done := make(chan struct{}) + go func() { + defer close(done) + defer server.Close() + handleJoin(ctx, server, bufio.NewReader(server), newCountingWaker(nil), joinParams{upstreamAddr: upstream.Addr().String(), deadline: 100 * time.Millisecond, pollInterval: time.Millisecond}) + }() + select { + case <-accepted: + case <-time.After(time.Second): + t.Fatal("join did not reach game") + } + // The session must retain the drain context after the short join context + // is cancelled, including after its original deadline has elapsed. + time.Sleep(200 * time.Millisecond) + _ = player.SetDeadline(time.Now().Add(time.Second)) + if _, err := player.Write([]byte("still playing")); err != nil { + t.Fatal(err) + } + got := make([]byte, len("still playing")) + if _, err := io.ReadFull(player, got); err != nil || string(got) != "still playing" { + t.Fatalf("gameplay after join budget: %q, %v", got, err) + } + cancel() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("session did not stop on drain cancellation") + } +} + +type observedPacketConn struct { + net.PacketConn + reads chan struct{} +} + +func (c observedPacketConn) ReadFrom(p []byte) (int, net.Addr, error) { + n, addr, err := c.PacketConn.ReadFrom(p) + if err == nil { + c.reads <- struct{}{} + } + return n, addr, err +} + +func TestUDPReceiveContinuesWhileWakeIsStalled(t *testing.T) { + listener, err := (&net.ListenConfig{}).ListenPacket(t.Context(), "udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + client, err := (&net.Dialer{}).DialContext(t.Context(), "udp", listener.LocalAddr().String()) + if err != nil { + t.Fatal(err) + } + defer client.Close() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + wake := &contextBoundWake{started: make(chan struct{})} + reads := make(chan struct{}, 128) + done := make(chan struct{}) + go func() { + defer close(done) + serveUDPPackets(ctx, observedPacketConn{listener, reads}, PortConfig{}, wake, Config{UDPThreshold: 1, UDPWindow: time.Minute, UDPMaxSources: 10}, make(chan error, 1)) + }() + if _, err := client.Write([]byte("wake")); err != nil { + t.Fatal(err) + } + select { + case <-wake.started: + case <-time.After(time.Second): + t.Fatal("wake did not start") + } + started := time.Now() + for range 100 { + if _, err := client.Write([]byte("game packet")); err != nil { + t.Fatal(err) + } + } + for range 101 { + select { + case <-reads: + case <-time.After(time.Second): + t.Fatal("UDP receive blocked behind stalled wake PATCH") + } + } + t.Logf("received 101 datagrams while wake blocked in %s", time.Since(started)) + cancel() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("UDP listener/worker did not stop") + } +} diff --git a/test/e2e/buckets.sh b/test/e2e/buckets.sh index b803e2f42..3a42f525f 100755 --- a/test/e2e/buckets.sh +++ b/test/e2e/buckets.sh @@ -407,6 +407,11 @@ EOF # Tests that exist in the suite but deliberately run in NO bucket. Every # entry needs a reason; `verify` fails on any unlisted stray so additions # here are a conscious, reviewed act. Currently empty. +bucket_wake_hostport() { cat <<'EOF' +TestGameServer_HostportWakeStableEndpoint +EOF +} + bucket_management_placement() { cat <<'EOF' TestManagementPlacementAndDisruption EOF @@ -415,7 +420,7 @@ EOF unbucketed() { :; } bucket_names() { - printf '%s\n' operator api-auth api-roles api-rbac api-agent api-mods ratelimit bot-fast bot-heavy multicluster upgrade telemetry capture-overhead optional-components management-placement + printf '%s\n' operator api-auth api-roles api-rbac api-agent api-mods ratelimit bot-fast bot-heavy multicluster upgrade telemetry capture-overhead optional-components wake-hostport management-placement } list_bucket() { @@ -434,6 +439,7 @@ list_bucket() { telemetry) bucket_telemetry ;; capture-overhead) bucket_capture_overhead ;; optional-components) bucket_optional_components ;; + wake-hostport) bucket_wake_hostport ;; management-placement) bucket_management_placement ;; *) echo "unknown bucket: $1 (known: $(bucket_names | tr '\n' ' '))" >&2 diff --git a/test/e2e/ci_selection_e2e_test.go b/test/e2e/ci_selection_e2e_test.go index a52268ad2..1eab3b245 100644 --- a/test/e2e/ci_selection_e2e_test.go +++ b/test/e2e/ci_selection_e2e_test.go @@ -22,7 +22,7 @@ import ( // ciSelectionBuckets mirrors E2E_BUCKETS in hack/ci_scope.py (the e2e-go matrix). var ciSelectionBuckets = []string{ "operator", "api-auth", "api-roles", "api-rbac", "api-agent", "api-mods", - "telemetry", "optional-components", "management-placement", + "telemetry", "optional-components", "wake-hostport", "management-placement", } // ciSelection is the decoded output of one `ci_scope.py e2e` run. diff --git a/test/e2e/hostport_wake_e2e_test.go b/test/e2e/hostport_wake_e2e_test.go new file mode 100644 index 000000000..4cef3451b --- /dev/null +++ b/test/e2e/hostport_wake_e2e_test.go @@ -0,0 +1,241 @@ +//go:build e2e + +package e2e + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" +) + +// This bucket owns its two-node cluster. It deliberately retains terminating +// Pods and changes node scheduling; it must not run with other game tests. +func TestGameServer_HostportWakeStableEndpoint(t *testing.T) { + ctx := t.Context() + // The chart owns this namespace and scopes operator Secret access to it. + const ns, port = "gameplane-games", 12345 + name := fmt.Sprintf("hostport-wake-%d", time.Now().UnixNano()) + tmpl := name + "-template" + nodes, err := envInstance.K8s.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + t.Fatal(err) + } + if len(nodes.Items) < 2 { + t.Fatal("wake-hostport requires the two-node kind profile") + } + var endpoint corev1.Node + for _, node := range nodes.Items { + if _, control := node.Labels["node-role.kubernetes.io/control-plane"]; !control { + endpoint = node + break + } + } + if endpoint.Name == "" { + t.Fatal("no worker node") + } + for _, node := range nodes.Items { + patch := []byte(`{"metadata":{"labels":{"gameplane.local/wake-pool":"test"}}}`) + if _, err := envInstance.K8s.CoreV1().Nodes().Patch(ctx, node.Name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + } + // The game and sentinel both need this dedicated-node toleration. + taints := append(append([]corev1.Taint{}, endpoint.Spec.Taints...), corev1.Taint{Key: "gameplane.local/dedicated", Value: "games", Effect: corev1.TaintEffectNoSchedule}) + patchHostportNode(t, endpoint.Name, map[string]any{"taints": taints}) + t.Cleanup(func() { + patchHostportNode(t, endpoint.Name, map[string]any{"unschedulable": false, "taints": endpoint.Spec.Taints}) + }) + address := "" + for _, addr := range endpoint.Status.Addresses { + if addr.Type == corev1.NodeInternalIP { + address = net.JoinHostPort(addr.Address, fmt.Sprint(port)) + break + } + } + if address == "" { + t.Fatal("worker has no InternalIP") + } + + applyBusyboxTemplate(t, tmpl) + // During deletion the shell keeps httpd alive for 12 seconds. This makes + // the old port genuinely occupied while the sleep handoff is observed. + templatePatch := map[string]any{"spec": map[string]any{ + "command": []string{"sh", "-c", "mkdir -p /tmp/site; echo endpoint-ok >/tmp/site/index.html; httpd -p 12345 -h /tmp/site; trap 'sleep 12; exit 0' TERM; while :; do sleep 1; done"}, + "ports": []any{map[string]any{"name": "game", "containerPort": port, "advertise": true, "protocol": "TCP", "wakeProtocol": "generic"}}, + }} + data, err := json.Marshal(templatePatch) + if err != nil { + t.Fatal(err) + } + if _, err := envInstance.Dyn.Resource(gameTemplateGVR).Patch(ctx, tmpl, types.MergePatchType, data, metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + gs := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "gameplane.local/v1alpha1", "kind": "GameServer", + "metadata": map[string]any{"name": name, "namespace": ns}, + "spec": map[string]any{"templateRef": map[string]any{"name": tmpl}, "networking": map[string]any{"expose": "Hostport"}, + "nodeSelector": map[string]any{"kubernetes.io/hostname": endpoint.Labels["kubernetes.io/hostname"]}, + "tolerations": []any{map[string]any{"key": "gameplane.local/dedicated", "operator": "Equal", "value": "games", "effect": "NoSchedule"}, map[string]any{"key": "node-role.kubernetes.io/control-plane", "operator": "Exists", "effect": "NoSchedule"}}, + "idle": map[string]any{"enabled": true, "afterMinutes": int64(60), "wakeOnConnect": true}}, + }} + if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Create(ctx, gs, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Delete(context.Background(), name, metav1.DeleteOptions{}) + }) + waitPVCBound(t, ns, name+"-data", 90*time.Second) + httpClient := &http.Client{Timeout: 2 * time.Second, Transport: &http.Transport{DisableKeepAlives: true}} + defer httpClient.CloseIdleConnections() + gameReachable := func() (bool, string) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://"+address, nil) + if err != nil { + return false, err.Error() + } + resp, err := httpClient.Do(request) + if err != nil { + return false, err.Error() + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + return err == nil && strings.Contains(string(body), "endpoint-ok"), string(body) + } + for cycle := range 2 { + envInstance.Eventually(t, 2*time.Minute, gameReachable) + pod, err := envInstance.K8s.CoreV1().Pods(ns).Get(ctx, name+"-0", metav1.GetOptions{}) + if err != nil || pod.Spec.NodeName != endpoint.Name { + t.Fatalf("game moved: pod=%v err=%v", pod, err) + } + patchHostportServer(t, ns, name, map[string]any{"metadata": map[string]any{"annotations": map[string]any{"gameplane.local/idle-asleep-since": time.Now().UTC().Format(time.RFC3339)}}}) + envInstance.Eventually(t, 20*time.Second, func() (bool, string) { + pod, err := envInstance.K8s.CoreV1().Pods(ns).Get(ctx, name+"-0", metav1.GetOptions{}) + return err == nil && !pod.DeletionTimestamp.IsZero(), fmt.Sprint(err) + }) + if ok, msg := gameReachable(); !ok { + t.Fatalf("fixture did not retain occupied old port during termination: %s", msg) + } + if _, err := envInstance.K8s.AppsV1().Deployments(ns).Get(ctx, name+"-waker", metav1.GetOptions{}); !apierrors.IsNotFound(err) { + t.Fatalf("sentinel created before old game released port: %v", err) + } + assertHostportCondition(t, ns, name, "GameDraining", false) + if cycle == 0 { + // Once the endpoint is recorded, broaden the worker pool to both + // nodes. Only the persisted required affinity can retain identity. + patchHostportServer(t, ns, name, map[string]any{"spec": map[string]any{"nodeSelector": map[string]any{"kubernetes.io/hostname": nil, "gameplane.local/wake-pool": "test"}}}) + patchHostportNode(t, endpoint.Name, map[string]any{"unschedulable": true}) + envInstance.Eventually(t, 90*time.Second, func() (bool, string) { return hostportConditionIs(t, ns, name, "EndpointUnschedulable", false) }) + // The alternate node is available; the listener must stay Pending + // on the pinned endpoint instead of silently moving there. + assertHostportPodsOnNode(t, ns, name, endpoint.Name) + patchHostportNode(t, endpoint.Name, map[string]any{"unschedulable": false}) + } + envInstance.Eventually(t, 90*time.Second, func() (bool, string) { return hostportConditionIs(t, ns, name, "ListenerReady", true) }) + assertHostportPodsOnNode(t, ns, name, endpoint.Name) + wakers, err := envInstance.K8s.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{LabelSelector: "app.kubernetes.io/name=gameplane-waker,app.kubernetes.io/instance=" + name}) + if err != nil || len(wakers.Items) != 1 { + t.Fatalf("sentinel Pods: %v, %v", wakers, err) + } + waker := wakers.Items[0] + finalizer := []byte(`{"metadata":{"finalizers":["test.gameplane.local/hold-handoff"]}}`) + if _, err := envInstance.K8s.CoreV1().Pods(ns).Patch(ctx, waker.Name, types.MergePatchType, finalizer, metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _, _ = envInstance.K8s.CoreV1().Pods(ns).Patch(context.Background(), waker.Name, types.MergePatchType, []byte(`{"metadata":{"finalizers":[]}}`), metav1.PatchOptions{}) + }) + // A connection to the same node address must reach the sentinel and + // wake the game. Hostport clients reconnect after this intentional bounce. + conn, err := (&net.Dialer{Timeout: 3 * time.Second}).DialContext(ctx, "tcp", address) + if err != nil { + t.Fatalf("sleeping endpoint unreachable: %v", err) + } + _ = conn.Close() + envInstance.Eventually(t, 30*time.Second, func() (bool, string) { return hostportConditionIs(t, ns, name, "SentinelDraining", false) }) + envInstance.Consistently(t, 3*time.Second, 200*time.Millisecond, func() (bool, string) { + _, err := envInstance.K8s.CoreV1().Pods(ns).Get(ctx, name+"-0", metav1.GetOptions{}) + return apierrors.IsNotFound(err), "game scheduled before terminating sentinel Pod disappeared" + }) + if _, err := envInstance.K8s.CoreV1().Pods(ns).Patch(ctx, waker.Name, types.MergePatchType, []byte(`{"metadata":{"finalizers":[]}}`), metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + envInstance.Eventually(t, 2*time.Minute, gameReachable) + assertHostportPodsOnNode(t, ns, name, endpoint.Name) + t.Logf("cycle %d kept reachable endpoint %s on %s", cycle+1, address, endpoint.Name) + } + // Real node deletion (including replacement under the same name) must + // report unavailable instead of migrating to the otherwise eligible node. + patchHostportServer(t, ns, name, map[string]any{"metadata": map[string]any{"annotations": map[string]any{"gameplane.local/idle-asleep-since": time.Now().UTC().Format(time.RFC3339)}}}) + if err := envInstance.K8s.CoreV1().Nodes().Delete(ctx, endpoint.Name, metav1.DeleteOptions{}); err != nil { + t.Fatal(err) + } + envInstance.Eventually(t, 45*time.Second, func() (bool, string) { return hostportConditionIs(t, ns, name, "EndpointNodeUnavailable", false) }) + assertHostportPodsOnNode(t, ns, name, endpoint.Name) +} + +func patchHostportServer(t *testing.T, ns, name string, patch map[string]any) { + t.Helper() + data, err := json.Marshal(patch) + if err != nil { + t.Fatal(err) + } + if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Patch(context.Background(), name, types.MergePatchType, data, metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } +} + +func patchHostportNode(t *testing.T, name string, spec map[string]any) { + t.Helper() + data, err := json.Marshal(map[string]any{"spec": spec}) + if err != nil { + t.Fatal(err) + } + if _, err := envInstance.K8s.CoreV1().Nodes().Patch(context.Background(), name, types.MergePatchType, data, metav1.PatchOptions{}); err != nil && !apierrors.IsNotFound(err) { + t.Error(err) + } +} + +func hostportConditionIs(t *testing.T, ns, name, reason string, ready bool) (bool, string) { + t.Helper() + gs, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Get(t.Context(), name, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + conditions, _, _ := unstructured.NestedSlice(gs.Object, "status", "conditions") + for _, item := range conditions { + condition, ok := item.(map[string]any) + if ok && condition["type"] == "HostportWakeReady" { + return condition["reason"] == reason && (condition["status"] == "True") == ready, fmt.Sprint(condition) + } + } + return false, "HostportWakeReady not yet present" +} + +func assertHostportCondition(t *testing.T, ns, name, reason string, ready bool) { + t.Helper() + envInstance.Eventually(t, 15*time.Second, func() (bool, string) { return hostportConditionIs(t, ns, name, reason, ready) }) +} + +func assertHostportPodsOnNode(t *testing.T, ns, name, node string) { + t.Helper() + pods, err := envInstance.K8s.CoreV1().Pods(ns).List(t.Context(), metav1.ListOptions{LabelSelector: "app.kubernetes.io/instance=" + name}) + if err != nil { + t.Fatal(err) + } + for _, pod := range pods.Items { + if pod.Spec.NodeName != "" && pod.Spec.NodeName != node { + t.Fatalf("listener %s moved to %s; endpoint remains %s", pod.Name, pod.Spec.NodeName, node) + } + } +} diff --git a/test/e2e/specs.md b/test/e2e/specs.md index 2954c54b1..42952f313 100644 --- a/test/e2e/specs.md +++ b/test/e2e/specs.md @@ -20,6 +20,19 @@ It was one of three `go.work` modules without CI lint coverage until this featur - Verify the installed operator identity can read Secrets only in its configured namespaces, can mutate game-namespace Secrets, and cannot read or write unrelated namespace Secrets (`TestHelmInstall_OperatorSecretNamespaceBoundary`, operator bucket). - Enforce, via `buckets.sh`, that every test in the suite is assigned to exactly one CI bucket so nothing silently drops out of coverage. +### Hostport wake acceptance + +The dedicated `wake-hostport` bucket runs on the `two-node` kind profile. It uses +a real Hostport HTTP listener and dedicated-node taint, broadens the scheduling +pool after recording the endpoint, and exercises two sleep/wake cycles at the +same node address. It observes the old game still serving during termination, +holds a terminating sentinel Pod to prove the reverse handoff waits, checks a +cordoned endpoint remains unavailable rather than relocating, and deletes the +node to exercise the explicit unavailable/replaced-node condition. This bucket +runs alone because it changes node scheduling. Its GameServer and template names +are unique per run; it uses the chart-owned game namespace so the operator's +configured Secret scope and RBAC apply. Runtime results come from CI. + ### Remote gateway parity acceptance The multicluster bucket runs `TestMultiCluster_GatewayParity` against two real @@ -68,10 +81,11 @@ The golangci-lint job mirrors this: it must be invoked with `--build-tags=e2e` o ## The bucket contract — the module's most important invariant -The `management-placement` audit bucket is registered in the change-based selector, -with explicit component mapping and an amd64-only selection-summary row. Its -matrix leg skips cluster setup only when its selected test pattern is empty; -full-scope and affected-component runs must include it. +The `wake-hostport` and `management-placement` audit buckets are registered in +the change-based selector with explicit component mapping and amd64-only +selection-summary rows. Each matrix leg skips cluster setup only when its +selected test pattern is empty; full-scope and affected-component runs must +include both buckets. `buckets.sh` is the single source of truth for how CI's e2e suite is sharded into parallel jobs (`operator`, `api-auth`, `api-roles`, `api-rbac`, `api-agent`, `api-mods`, `ratelimit`, `bot-fast`, `bot-heavy`, `multicluster`, `upgrade`). Each bucket is a flat list of **exact Go function names** (e.g. `TestGameServer_MinecraftJavaBot_Joined`), and `buckets.sh regex ` turns that list into an **anchored** `^(A|B|...)$` regex passed to `go test -run`. @@ -91,7 +105,7 @@ This is a name-matching contract, not a structural one: ## Conventions (already normative via `CLAUDE.md`, restated here for the module's own record) -- **`t.Parallel()`**: tests call it unless shared infrastructure requires serialization. The `ratelimit` bucket deliberately drains the shared login limiter; gateway parity temporarily changes the shared remote Helm release and trust. These are documented exceptions. +- **`t.Parallel()`**: tests call it unless shared infrastructure requires serialization. The `ratelimit` bucket deliberately drains the shared login limiter; gateway parity temporarily changes the shared remote Helm release and trust; `wake-hostport` changes shared node scheduling and deletes the endpoint node. These are documented exceptions. - **Per-test unique resource names**: CRs, namespaces, and users a test creates get a name unique to that test run (commonly time-based, e.g. `CreateUser`'s `fmt.Sprintf("%s-%d", prefix, time.Now().UnixNano())`) so parallel tests never collide on a shared cluster. - **`ociPushMu`**: a package-level mutex guarding tests that push through the shared, fixed-name `oras-push` Job (`module_e2e_test.go`, `module_verify_e2e_test.go`). Because the Job's name and namespace are fixed fixtures, two tests racing to apply/wait on it would stomp each other; `ociPushMu.Lock()`/`Unlock()` serializes them. - **`ensureResticRepo(t)`**: guards tests that run a backup against the shared restic repository fixture (`backup_e2e_test.go`, `backupschedule_e2e_test.go`, `failure_paths_e2e_test.go`) — restic repo initialization is not safe to race. diff --git a/test/e2e/suite-components.json b/test/e2e/suite-components.json index 238af8f04..c8125bcdf 100644 --- a/test/e2e/suite-components.json +++ b/test/e2e/suite-components.json @@ -16,6 +16,7 @@ "test/e2e/suite-components.json" ], "suites": [ + { "tests": "^TestGameServer_HostportWakeStableEndpoint$", "components": ["operator", "api", "agent", "sentinel", "gameproto"] }, { "tests": "^TestManagementPlacementAndDisruption$", "components": ["operator", "api", "agent"] }, { "tests": "^TestHelmInstall_", "components": ["*"] }, { "tests": "^TestCRD_Validation_", "components": ["operator"] },