fix(api): enforce singleton rollouts and database-aware readiness

.github/workflows/ci.yaml

@@ -2381,12 +2381,14 @@ jobs:
   # between buckets: its login rate limiter lives in memory and each bucket is
   # sized to one fresh limiter's budget (see test/e2e/buckets.sh). The restart
   # also means each later bucket runs on data the earlier ones left in Postgres.
+  # A final serial bucket migrates the Deployment strategy and exercises a
+  # PostgreSQL outage/recovery against that populated installation.
   e2e-postgres:
     name: e2e postgres / amd64 (kind)
     needs: [changes, build-images]
     if: (needs.changes.outputs.e2e == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'e2e-postgres')
     runs-on: ubuntu-latest
-    timeout-minutes: 75 # Boots a kind cluster with PostgreSQL, installs the chart, runs three buckets.
+    timeout-minutes: 75 # Boots PostgreSQL and runs auth, roles, RBAC, then availability.
     permissions:
       contents: read
     env:
@@ -2444,6 +2446,13 @@ jobs:
             GAMEPLANE_E2E_REUSE_CLUSTER=1 \
             GAMEPLANE_E2E_CLUSTER="$CLUSTER" \
             go test -tags=e2e -timeout 10m -v -parallel 4 -run "$RUN_REGEX" ./...
+      - name: run go e2e (PostgreSQL singleton upgrade and outage recovery)
+        run: |
+          RUN_REGEX="$(./test/e2e/buckets.sh regex postgres-availability)"
+          cd test/e2e && \
+            GAMEPLANE_E2E_REUSE_CLUSTER=1 \
+            GAMEPLANE_E2E_CLUSTER="$CLUSTER" \
+            go test -tags=e2e -timeout 15m -v -parallel 1 -run "$RUN_REGEX" ./...
       - name: dump cluster state on failure (e2e-postgres)
         if: failure()
         uses: ./.github/actions/dump-cluster-state
@@ -2854,6 +2863,10 @@ jobs:
               if (m) bucketSet.add(m[1]);
               if (/^e2e multicluster \//.test(n)) bucketSet.add('multicluster');
               if (/^e2e upgrade \//.test(n)) bucketSet.add('upgrade');
+              if (/^e2e postgres \//.test(n) && Array.isArray(j.steps)) {
+                const availability = j.steps.find((s) => s.name === 'run go e2e (PostgreSQL singleton upgrade and outage recovery)');
+                if (availability && availability.conclusion === 'success') bucketSet.add('postgres-availability');
+              }
               // The ratelimit bucket runs as a trailing step inside the
               // api-auth job leg (deliberately, so it drains the shared
               // login buckets last), not as its own job — so it never

api/cmd/health.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/go-chi/chi/v5"
+
+	"github.com/GameplanePanel/gameplane/api/internal/db"
+)
+
+// mountHealthRoutes separates process liveness from database availability.
+// Only one readiness ping can wait for a connection at a time, particularly
+// important for SQLite's single-connection pool. Concurrent probes fail closed
+// immediately instead of queueing more database work. No background work or
+// cached success can outlive the two-second request budget.
+func mountHealthRoutes(r chi.Router, store *db.Store) {
+	r.Get("/healthz", func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write([]byte("ok"))
+	})
+	pingSlot := make(chan struct{}, 1)
+	r.Get("/readyz", func(w http.ResponseWriter, req *http.Request) {
+		select {
+		case pingSlot <- struct{}{}:
+			defer func() { <-pingSlot }()
+		default:
+			http.Error(w, "not ready", http.StatusServiceUnavailable)
+			return
+		}
+		ctx, cancel := context.WithTimeout(req.Context(), 2*time.Second)
+		defer cancel()
+		if err := store.DB.PingContext(ctx); err != nil {
+			http.Error(w, "not ready", http.StatusServiceUnavailable)
+			return
+		}
+		_, _ = w.Write([]byte("ok"))
+	})
+}

api/cmd/health_test.go

@@ -0,0 +1,100 @@
+package main
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/go-chi/chi/v5"
+
+	"github.com/GameplanePanel/gameplane/api/internal/db"
+)
+
+func TestHealthRoutesDatabaseUnavailable(t *testing.T) {
+	store, err := db.Open(t.Context(), "sqlite", ":memory:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = store.Close() })
+	r := chi.NewRouter()
+	mountHealthRoutes(r, store)
+
+	check := func(path string, want int, body string) {
+		t.Helper()
+		rr := httptest.NewRecorder()
+		r.ServeHTTP(rr, httptest.NewRequestWithContext(t.Context(), http.MethodGet, path, nil))
+		if rr.Code != want || rr.Body.String() != body {
+			t.Fatalf("%s: status=%d body=%q, want %d %q", path, rr.Code, rr.Body.String(), want, body)
+		}
+	}
+	check("/readyz", http.StatusOK, "ok")
+	if err := store.Close(); err != nil {
+		t.Fatal(err)
+	}
+	check("/readyz", http.StatusServiceUnavailable, "not ready\n")
+	check("/healthz", http.StatusOK, "ok")
+}
+
+// Holding SQLite's only connection reproduces pool starvation without a fake
+// driver. Readiness must time out, cap concurrent pings, and recover when the
+// connection is released; liveness must never wait for that connection.
+func TestHealthRoutesBoundPingAndRecover(t *testing.T) {
+	store, err := db.Open(t.Context(), "sqlite", ":memory:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = store.Close() })
+	conn, err := store.DB.Conn(t.Context())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = conn.Close() }()
+	r := chi.NewRouter()
+	mountHealthRoutes(r, store)
+	ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)
+	defer cancel()
+	done := make(chan *httptest.ResponseRecorder, 1)
+	started := time.Now()
+	go func() {
+		rr := httptest.NewRecorder()
+		r.ServeHTTP(rr, httptest.NewRequestWithContext(ctx, http.MethodGet, "/readyz", nil))
+		done <- rr
+	}()
+	for store.DB.Stats().WaitCount == 0 {
+		select {
+		case <-ctx.Done():
+			t.Fatal("readiness did not attempt the database ping")
+		case <-time.After(time.Millisecond):
+		}
+	}
+	for _, path := range []string{"/readyz", "/healthz"} {
+		rr := httptest.NewRecorder()
+		before := time.Now()
+		r.ServeHTTP(rr, httptest.NewRequestWithContext(ctx, http.MethodGet, path, nil))
+		want := http.StatusOK
+		if path == "/readyz" {
+			want = http.StatusServiceUnavailable
+		}
+		if rr.Code != want || time.Since(before) > 500*time.Millisecond {
+			t.Fatalf("%s blocked behind the ping or returned %d", path, rr.Code)
+		}
+	}
+	select {
+	case rr := <-done:
+		if rr.Code != http.StatusServiceUnavailable || time.Since(started) > 2500*time.Millisecond {
+			t.Fatalf("ping did not fail within its two-second budget: status=%d elapsed=%s", rr.Code, time.Since(started))
+		}
+	case <-ctx.Done():
+		t.Fatal("readiness exceeded its own deadline")
+	}
+	if err := conn.Close(); err != nil {
+		t.Fatal(err)
+	}
+	rr := httptest.NewRecorder()
+	r.ServeHTTP(rr, httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/readyz", nil))
+	if rr.Code != http.StatusOK {
+		t.Fatalf("readiness did not recover: %d %s", rr.Code, rr.Body.String())
+	}
+}

api/cmd/main.go

@@ -313,9 +313,7 @@ func main() {
 	r.Use(bodyLimit(1 << 20)) // 1 MiB default; upload proxy raises its own ceiling
 	r.Use(audit.Middleware(auditor))
 
-	r.Get("/healthz", func(w http.ResponseWriter, _ *http.Request) {
-		_, _ = w.Write([]byte("ok"))
-	})
+	mountHealthRoutes(r, store)
 	// Prometheus metrics are served by the separate metrics listener below,
 	// never on this public router.
 

api/internal/auth/sessions.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/rand"
 	"crypto/sha256"
+	"database/sql"
 	"encoding/base64"
 	"encoding/hex"
 	"errors"
@@ -23,6 +24,11 @@ const (
 	sessionTTL    = 12 * time.Hour
 )
 
+var (
+	errSessionInvalid = errors.New("session invalid")
+	errSessionExpired = errors.New("session expired")
+)
+
 type ctxKey int
 
 const userCtxKey ctxKey = iota
@@ -136,7 +142,13 @@ func (s *SessionStore) Authenticate(next http.Handler) http.Handler {
 		}
 		u, csrf, err := s.lookup(req.Context(), cookie.Value)
 		if err != nil {
-			httperr.WriteMessage(w, req, messages.New("server.common.unauthenticated", nil))
+			if errors.Is(err, sql.ErrNoRows) || errors.Is(err, errSessionInvalid) || errors.Is(err, errSessionExpired) {
+				httperr.WriteMessage(w, req, messages.New("server.common.unauthenticated", nil))
+			} else {
+				// Store failures deny access but do not imply logout. Never put
+				// database, account, cookie or permission details in the response.
+				httperr.WriteMessage(w, req, messages.New("server.generic.503", nil))
+			}
 			return
 		}
 		if isMutating(req.Method) {
@@ -169,24 +181,24 @@ func (s *SessionStore) lookup(ctx context.Context, token string) (*User, string,
 	if err != nil {
 		return nil, "", err
 	}
-	perms, perr := s.LoadPerms(ctx, u.ID)
-	if perr != nil {
-		return nil, "", perr
-	}
-	u.Perms = perms
 	exp, perr := time.Parse(time.RFC3339, expires)
 	if perr != nil {
 		// A corrupt expires_at row is indistinguishable from an expired
 		// session from the client's perspective — delete it and make the
 		// user log in again. Log so it's visible in ops.
 		slog.Warn("session expires_at parse", "err", perr, "value", expires)
 		_, _ = s.db.DB.ExecContext(ctx, `DELETE FROM sessions WHERE token = ?`, digest)
-		return nil, "", errors.New("session invalid")
+		return nil, "", errSessionInvalid
 	}
 	if time.Now().After(exp) {
 		_, _ = s.db.DB.ExecContext(ctx, `DELETE FROM sessions WHERE token = ?`, digest)
-		return nil, "", errors.New("session expired")
+		return nil, "", errSessionExpired
 	}
+	perms, perr := s.LoadPerms(ctx, u.ID)
+	if perr != nil {
+		return nil, "", perr
+	}
+	u.Perms = perms
 	return &u, csrf, nil
 }
 

api/internal/auth/sessions_availability_test.go

@@ -0,0 +1,87 @@
+package auth
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestAuthenticateStoreFailureDoesNotInvalidateSession(t *testing.T) {
+	s := newAuthDB(t)
+	seedUser(t, s, "alice", "pw", "admin")
+	store := NewSessionStore(s)
+	token, _, err := store.Create(t.Context(), 1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	h := store.Authenticate(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusNoContent)
+	}))
+	request := func(cookie string, want int) {
+		t.Helper()
+		rr := httptest.NewRecorder()
+		req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/protected", nil)
+		if cookie != "" {
+			req.AddCookie(&http.Cookie{Name: sessionCookie, Value: cookie})
+		}
+		h.ServeHTTP(rr, req)
+		if rr.Code != want {
+			t.Fatalf("status=%d body=%q, want %d", rr.Code, rr.Body.String(), want)
+		}
+		if len(rr.Result().Cookies()) != 0 {
+			t.Fatal("authentication failure changed the client's session cookies")
+		}
+		if want == http.StatusServiceUnavailable && rr.Body.String() != "Service Unavailable\n" {
+			t.Fatalf("store error was not generic: %q", rr.Body.String())
+		}
+	}
+
+	request(token, http.StatusNoContent)
+	// A permission-query failure after a successful session lookup must deny
+	// access without being mistaken for an expired/unknown session.
+	if _, err := s.DB.ExecContext(t.Context(), "ALTER TABLE role_permissions RENAME TO unavailable_permissions"); err != nil {
+		t.Fatal(err)
+	}
+	request(token, http.StatusServiceUnavailable)
+	request("", http.StatusUnauthorized)
+	request("unknown-token", http.StatusUnauthorized)
+	if _, err := s.DB.ExecContext(t.Context(), "ALTER TABLE unavailable_permissions RENAME TO role_permissions"); err != nil {
+		t.Fatal(err)
+	}
+	request(token, http.StatusNoContent)
+	if err := s.Close(); err != nil {
+		t.Fatal(err)
+	}
+	request(token, http.StatusServiceUnavailable)
+	request("unknown-token", http.StatusServiceUnavailable)
+	request("", http.StatusUnauthorized)
+}
+
+func TestAuthenticateInvalidSessionRemainsUnauthorized(t *testing.T) {
+	for _, expires := range []string{"invalid-timestamp", time.Now().Add(-time.Hour).UTC().Format(time.RFC3339)} {
+		t.Run(expires, func(t *testing.T) {
+			s := newAuthDB(t)
+			seedUser(t, s, "alice", "pw", "admin")
+			store := NewSessionStore(s)
+			token, _, err := store.Create(t.Context(), 1)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if _, err := s.DB.ExecContext(t.Context(), "UPDATE sessions SET expires_at = ?", expires); err != nil {
+				t.Fatal(err)
+			}
+			h := store.Authenticate(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
+				t.Error("invalid session reached protected handler")
+			}))
+			rr := httptest.NewRecorder()
+			req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/protected", nil)
+			req.AddCookie(&http.Cookie{Name: sessionCookie, Value: token})
+			h.ServeHTTP(rr, req)
+			if rr.Code != http.StatusUnauthorized || strings.TrimSpace(rr.Body.String()) != "unauthenticated" {
+				t.Fatalf("invalid session returned %d %q", rr.Code, rr.Body.String())
+			}
+		})
+	}
+}

api/internal/db/db.go

@@ -84,7 +84,7 @@ func (s *Store) Close() error { return s.DB.Close() }
 // user-management access (role permission edits, primary-role changes,
 // user deletion). Hold it from the last-user-manager check through the
 // commit of the change so two requests can't both pass the check. It
-// covers one API process; the SQLite install runs a single API replica.
+// covers one API process; every database driver requires a single API replica.
 func (s *Store) LockUserManagement() (unlock func()) {
 	s.userMgmtMu.Lock()
 	return s.userMgmtMu.Unlock

api/specs.md

@@ -104,6 +104,7 @@ The HTTP server listens on `:8000` (configurable) with these route groups. Prome
 - `/auth/oidc/start` (legacy) — GET: single helm-provider start
 - `/auth/oidc/callback` (legacy) — GET: single helm-provider callback
 - `/healthz` — GET: liveness probe
+- `/readyz` — GET: database readiness, a `PingContext` with a two-second deadline. At most one ping runs at once; concurrent probes return a generic 503 without queueing more database work. Success returns `ok`; unavailable or busy returns only `not ready`. The chart probes every five seconds with a three-second timeout; three consecutive failed probes remove readiness, tolerating brief SQLite connection contention. `/healthz` remains independent of the database, so outages do not trigger liveness restarts.
 
 **Protected (authenticated + RBAC):**
 - All six `/fleet/*` reads share a per-authenticated-user token bucket: 60 requests/minute sustained, burst 10. Exhausted requests receive 429 with `Retry-After: 1` before cluster discovery or resource listing. Sessions and routes share the account's budget; users behind the same IP do not. This per-process limit bounds request frequency without removing owner/collaborator scans; multiple API replicas have independent budgets.
@@ -537,7 +538,9 @@ Verify from `/api/go.mod`.
 ### Database driver selection
 
 - **Production (default):** `modernc.org/sqlite` — file-based, WAL mode, tested
-- **Experimental:** PostgreSQL via `jackc/pgx/v5` — compile with `-tags=postgres`. Works end to end; `api/internal/db`'s tests run against PostgreSQL in the `api (postgres)` CI job, but there is no Postgres e2e/upgrade coverage yet
+- **Experimental:** PostgreSQL via `jackc/pgx/v5` — compile with `-tags=postgres`. The `api (postgres)` CI job covers the store. The `e2e postgres` job runs API auth/roles/RBAC plus the `postgres-availability` bucket: populated RollingUpdate-to-Recreate migration, singleton Pod observation, audit integrity, and database outage/recovery without restarting the API or replacing the session.
+- Every enabled API install requires exactly one replica, for both drivers and both combined/standalone modes. Helm renders `Recreate` with `rollingUpdate: null`; the combined-install pre-upgrade hook clears an existing rolling-update strategy for either driver. Recreate prevents ordinary rollout overlap, but cannot fence a process on a partitioned node after force deletion. HA requires database-level serialization of audit/user-management transactions and coordinated background work/rate limiting before multiple writers are supported.
+- Protected-route session lookup distinguishes missing, expired and corrupt sessions (generic 401) from database/session-permission lookup failures (generic 503). Both deny access. Operational failures do not delete sessions or clear cookies; access recovers with the same cookie when the store recovers. Missing-cookie responses remain 401 without consulting storage, and login privacy is unchanged.
 - Driver selected at startup via `--db-driver` (sqlite|postgres) + `--db-dsn`
 - SQLite DSNs automatically receive a `busy_timeout(5000)` pragma if one is not already specified, allowing multiple processes (e.g., the API and bootstrap-admin) sharing the same file to wait for locks instead of failing immediately
 - Migrations run automatically on startup (`store.Migrate(ctx)`): the driver's legacy set (`migrations/sqlite/` or `migrations/postgres/`, 001-012, same filenames and resulting schema) then the shared set (`migrations/common/`, 013+, one portable file per migration), in version order; a version present in both sets is a startup error. Versions are recorded in `schema_migrations` by bare filename, so SQLite installs see the same versions as before the split