fix: eliminate 502 errors during rolling deployments

Add gateway-level retry, health checking, and disruption protection so
clients never see errors when the API server is updated.

Closes #559

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: scotwells

config/apiserver/deployment.yaml

@@ -10,8 +10,8 @@ spec:
       app.kubernetes.io/part-of: milo-control-plane
   strategy:
     rollingUpdate:
-      maxSurge: 25%
-      maxUnavailable: 25%
+      maxSurge: 1
+      maxUnavailable: 0
     type: RollingUpdate
   template:
     metadata:
@@ -47,6 +47,7 @@ spec:
           - --token-auth-file=$(TOKEN_AUTH_FILE)
           - --anonymous-auth=$(ANONYMOUS_AUTH)
           - --v=$(LOG_LEVEL)
+          - --shutdown-delay-duration=$(SHUTDOWN_DELAY_DURATION)
         env:
           - name: LOG_LEVEL
             value: "4"
@@ -92,6 +93,8 @@ spec:
             value: ""
           - name: ANONYMOUS_AUTH
             value: "false"
+          - name: SHUTDOWN_DELAY_DURATION
+            value: "10s"
         livenessProbe:
           failureThreshold: 3
           httpGet:

config/apiserver/kustomization.yaml

@@ -3,3 +3,4 @@ kind: Kustomization
 resources:
   - deployment.yaml
   - service.yaml
+  - pdb.yaml

config/apiserver/pdb.yaml

@@ -0,0 +1,9 @@
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: milo-apiserver
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: milo-apiserver

config/ingress/gateway-api/backend-traffic-policy.yaml

@@ -0,0 +1,37 @@
+apiVersion: gateway.envoyproxy.io/v1alpha1
+kind: BackendTrafficPolicy
+metadata:
+  name: milo-apiserver
+  namespace: milo-system
+spec:
+  targetRefs:
+    - group: gateway.networking.k8s.io
+      kind: HTTPRoute
+      name: milo-apiserver
+  retry:
+    numRetries: 3
+    retryOn:
+      triggers:
+        - gateway-error
+        - connect-failure
+        - reset
+    perRetry:
+      backOff:
+        baseInterval: 100ms
+        maxInterval: 1s
+      timeout: 2s
+  healthCheck:
+    active:
+      type: HTTP
+      http:
+        path: /readyz
+      interval: 5s
+      timeout: 3s
+      unhealthyThreshold: 2
+      healthyThreshold: 1
+    passive:
+      consecutive5XxErrors: 2
+      consecutiveGatewayErrors: 1
+      interval: 3s
+      baseEjectionTime: 15s
+      maxEjectionPercent: 33

config/ingress/gateway-api/kustomization.yaml

@@ -4,3 +4,4 @@ kind: Kustomization
 resources:
   - httproute.yaml
   - backend-tls-policy.yaml
+  - backend-traffic-policy.yaml