From b91ecb5ce84503e814a627c26e268e892589a341 Mon Sep 17 00:00:00 2001 From: yash bahuguna Date: Tue, 13 Oct 2026 18:29:42 +0530 Subject: [PATCH] v1.0-hardening: durable redis+postgres journal, gRPC auth, hardened images, prod k8s manifests - Fix reconciler to rebuild all non-terminal tasks (keyset-paginated Active()), removing the redis_pending=false recovery gap; schema index queue_tasks_active_idx; serialize one-time schema init with pg_advisory_xact_lock to stop gateway/worker migration races. - Gate the gRPC surface with the same bearer tokens the HTTP API issues (method->permission map, fail-closed); demoenqueue/loadgen gained a -token flag; OIDC + Redis-backed session state for the web console. - Reproducible multi-stage images (golang:1.26-alpine -> distroless-static; web nitro node-server on :3000) with .dockerignore and a CI images job. - Deploy k8s base + kind/prod overlays: redis StatefulSet with AOF+PVC, Postgres journal StatefulSet with PVC, wait-for-redis/postgres initContainers, NetworkPolicies, HPA, PDB, ingress. --- .dockerignore | 34 ++ .github/workflows/ci.yml | 28 + .gitignore | 8 +- README.md | 152 +++++- cmd/demoenqueue/main.go | 24 +- cmd/loadgen/main.go | 23 +- deploy/k8s/base/apiserver.yaml | 88 ++++ deploy/k8s/base/gateway.yaml | 112 ++++ deploy/k8s/base/hpa.yaml | 41 ++ deploy/k8s/base/ingress.yaml | 29 + deploy/k8s/base/kustomization.yaml | 28 + deploy/k8s/base/namespace.yaml | 6 + deploy/k8s/base/networkpolicy.yaml | 47 ++ deploy/k8s/base/pdb.yaml | 35 ++ deploy/k8s/base/postgres.yaml | 90 ++++ deploy/k8s/base/redis.yaml | 87 +++ deploy/k8s/base/web.yaml | 75 +++ deploy/k8s/base/worker.yaml | 107 ++++ deploy/k8s/kind-config.yaml | 8 + deploy/k8s/observability/kustomization.yaml | 14 + deploy/k8s/observability/servicemonitor.yaml | 18 + deploy/k8s/overlays/kind/kustomization.yaml | 31 ++ deploy/k8s/overlays/prod/kustomization.yaml | 36 ++ deploy/prometheus.yml | 2 +- dockerfiles/apiserver.Dockerfile | 18 + dockerfiles/gateway.Dockerfile | 18 + dockerfiles/web.Dockerfile | 26 + dockerfiles/worker.Dockerfile | 18 + internal/auth/config.go | 113 +++- internal/auth/oidc.go | 415 +++++++++++++++ internal/auth/oidc_test.go | 218 ++++++++ internal/auth/password.go | 6 +- internal/auth/rbac.go | 8 +- internal/auth/redis_state.go | 37 ++ internal/auth/service.go | 32 +- internal/auth/state.go | 59 +++ internal/auth/token.go | 8 +- internal/durable/postgres.go | 106 ++-- internal/durable/postgres_test.go | 64 ++- internal/durable/reconciler.go | 109 ++-- internal/durable/reconciler_test.go | 34 +- internal/durable/store.go | 61 +-- internal/gateway/auth.go | 70 +++ internal/gateway/auth_test.go | 108 ++++ internal/gateway/main.go | 45 +- internal/gateway/router.go | 42 +- internal/handlers/handlers.go | 75 +-- internal/httpapi/auth_test.go | 262 +++++++++ internal/httpapi/handlers.go | 135 ++++- internal/httpapi/server.go | 32 +- internal/httpapi/types.go | 12 + internal/metrics/metrics.go | 104 ++-- internal/model/task.go | 6 +- internal/redisutil/client.go | 2 +- internal/worker/registry.go | 9 +- internal/worker/worker.go | 527 +++++++------------ web/src/api/endpoints.ts | 10 + web/src/components/app-shell.tsx | 7 +- web/src/components/job-tables.tsx | 176 ++++--- web/src/routes/__root.tsx | 3 +- web/src/routes/dlq.tsx | 173 +++--- web/src/routes/jobs.$jobId.tsx | 5 +- web/src/routes/login.tsx | 325 ++++++++++-- web/src/routes/queues.$queueName.tsx | 5 +- web/vite.config.ts | 4 +- 65 files changed, 3648 insertions(+), 962 deletions(-) create mode 100644 .dockerignore create mode 100644 deploy/k8s/base/apiserver.yaml create mode 100644 deploy/k8s/base/gateway.yaml create mode 100644 deploy/k8s/base/hpa.yaml create mode 100644 deploy/k8s/base/ingress.yaml create mode 100644 deploy/k8s/base/kustomization.yaml create mode 100644 deploy/k8s/base/namespace.yaml create mode 100644 deploy/k8s/base/networkpolicy.yaml create mode 100644 deploy/k8s/base/pdb.yaml create mode 100644 deploy/k8s/base/postgres.yaml create mode 100644 deploy/k8s/base/redis.yaml create mode 100644 deploy/k8s/base/web.yaml create mode 100644 deploy/k8s/base/worker.yaml create mode 100644 deploy/k8s/kind-config.yaml create mode 100644 deploy/k8s/observability/kustomization.yaml create mode 100644 deploy/k8s/observability/servicemonitor.yaml create mode 100644 deploy/k8s/overlays/kind/kustomization.yaml create mode 100644 deploy/k8s/overlays/prod/kustomization.yaml create mode 100644 dockerfiles/apiserver.Dockerfile create mode 100644 dockerfiles/gateway.Dockerfile create mode 100644 dockerfiles/web.Dockerfile create mode 100644 dockerfiles/worker.Dockerfile create mode 100644 internal/auth/oidc.go create mode 100644 internal/auth/oidc_test.go create mode 100644 internal/auth/redis_state.go create mode 100644 internal/auth/state.go create mode 100644 internal/gateway/auth.go create mode 100644 internal/gateway/auth_test.go create mode 100644 internal/httpapi/auth_test.go diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..0165b10 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,34 @@ +# VCS + editor noise +.git +.gitignore +.DS_Store + +# docs and local study/benchmark material +docs +bench + +# redis persistence dumps +*.rdb + +# pre-built host binaries (rebuild them from source in the image) +/gateway +/worker +/apiserver +/loadgen +*-binary + +# go test artifacts +*.test +coverage.out + +# web dependencies and build output (installed/rebuilt inside the image) +web/node_modules +web/.output +web/.tanstack +web/.wrangler +web/dist + +# local env files can leak dev-only API bases into the build; use build args +web/.env +web/.env.local +web/.env.*.local diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 99da23e..38e6a9a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,3 +36,31 @@ jobs: # unless FORGEQUEUE_PG_DSN is set. - name: Test run: go test ./... -count=1 + + images: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - name: apiserver + dockerfile: dockerfiles/apiserver.Dockerfile + - name: gateway + dockerfile: dockerfiles/gateway.Dockerfile + - name: worker + dockerfile: dockerfiles/worker.Dockerfile + - name: web + dockerfile: dockerfiles/web.Dockerfile + steps: + - uses: actions/checkout@v4 + + - uses: docker/setup-buildx-action@v3 + + - name: Build ${{ matrix.name }} image + uses: docker/build-push-action@v6 + with: + context: . + file: ${{ matrix.dockerfile }} + push: false + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.gitignore b/.gitignore index 54805e1..f7ce435 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,9 @@ docs/ /apiserver /loadgen +# pre-built binaries copied into docker build context +*-binary + # go test artifacts *.test coverage.out @@ -26,4 +29,7 @@ web/.tanstack/ # local env .env -.env.local \ No newline at end of file +.env.local + +# BullMQ comparison harness: track the source, ignore installed deps +bench/bullmq/node_modules/ diff --git a/README.md b/README.md index 1153c0e..4c1cd38 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,69 @@ if err != nil { fmt.Printf("Enqueued task %s\n", resp.TaskId) ``` +## Run with Docker + +Build every image from source — multi-stage, nothing is copied in pre-built: + +```bash +docker build -f dockerfiles/worker.Dockerfile -t forgequeue/worker:latest . +docker build -f dockerfiles/gateway.Dockerfile -t forgequeue/gateway:latest . +docker build -f dockerfiles/apiserver.Dockerfile -t forgequeue/apiserver:latest . +docker build -f dockerfiles/web.Dockerfile -t forgequeue/web:latest . +``` + +The four Go services are distroless static binaries running as a non-root user; +the console is a Node SSR server (Nitro `node-server` output), not a static +bundle. Bring the stack up on one network: + +```bash +docker network create forgequeue +docker run -d --name redis --network forgequeue redis:7.4-alpine --appendonly yes +docker run -d --name gateway --network forgequeue -e FORGEQUEUE_REDIS_ADDR=redis:6379 forgequeue/gateway:latest +docker run -d --name worker --network forgequeue -e FORGEQUEUE_REDIS_ADDR=redis:6379 \ + -e FORGEQUEUE_WORKER_CONCURRENCY=16 -e FORGEQUEUE_BATCH_SIZE=4 forgequeue/worker:latest +docker run -d --name apiserver --network forgequeue -p 8080:8080 \ + -e FORGEQUEUE_REDIS_ADDR=redis:6379 -e FORGEQUEUE_AUTH_SECRET=devsecret \ + -e FORGEQUEUE_ADMIN_PASSWORD=devpassword forgequeue/apiserver:latest +docker run -d --name web --network forgequeue -p 3000:3000 forgequeue/web:latest +``` + +The console is then on http://localhost:3000 and the API on +http://localhost:8080. The web image bakes `VITE_API_BASE_URL=/api` by default, +which assumes a reverse proxy in front; to hit a browser-reachable API origin +directly, rebuild it with +`--build-arg VITE_API_BASE_URL=http://localhost:8080/api`. + +## Kubernetes + +A Kustomize tree lives under `deploy/k8s`. `base/` holds Redis and the four +services as Deployments (non-root, read-only root filesystem, dropped +capabilities, health probes, resource limits, a Redis-gated init container), +plus an Ingress, HPAs and PodDisruptionBudgets. Two overlays: + +- `overlays/kind` — local kind cluster: `:latest` images loaded from your daemon + and a **dev-only** Secret generated in-cluster. +- `overlays/prod` — `ghcr.io/yashbhu/forgequeue-*:v1.0.0`, scaled replicas, and + no Secret (apply `forgequeue-secrets` out of band, e.g. SOPS + age). + +```bash +kind create cluster --name forgequeue --config deploy/k8s/kind-config.yaml +kind load docker-image forgequeue/worker:latest forgequeue/gateway:latest \ + forgequeue/apiserver:latest forgequeue/web:latest --name forgequeue +kubectl apply -k deploy/k8s/overlays/kind +kubectl -n forgequeue get pods +``` + +Redis runs a single replica on a PersistentVolumeClaim with AOF enabled, so +queued tasks survive a pod restart. Worker and apiserver autoscale on CPU with +the HPAs in `base/` — the cluster needs `metrics-server` for those targets to +resolve. Note that the worker idle-polls, so a **CPU-based** HPA scales on +polling overhead rather than queue depth; a queue-depth scaler (e.g. KEDA) would +be the accurate signal. Metrics are scraped via the `ServiceMonitor` in +`deploy/k8s/observability`, which is applied separately **after** installing +kube-prometheus-stack (the `monitoring.coreos.com` CRDs must exist). The Ingress +targets the `forgequeue.local` host and needs an ingress-nginx controller. + ## Architecture ForgeQueue consists of two main components that communicate through Redis: @@ -118,12 +181,18 @@ service QueueService { ### HTTP / JSON API (console backend) The `apiserver` (`cmd/apiserver`) exposes read and control endpoints on -`:8080` (`FORGEQUEUE_API_ADDR`). Every endpoint except `POST /api/auth/login` -requires a JWT bearer token and a role that carries the relevant permission. +`:8080` (`FORGEQUEUE_API_ADDR`). Every endpoint except the login endpoints +(`POST /api/auth/login`, `GET /api/auth/config`, and the `GET +/api/auth/oidc/*` SSO flow) requires a JWT bearer token and a role that carries +the relevant permission. | Method | Path | Permission | Purpose | | --- | --- | --- | --- | | POST | `/api/auth/login` | — | Exchange username/password for a bearer token | +| GET | `/api/auth/config` | — | Which login methods are enabled (password / SSO) | +| GET | `/api/auth/oidc/login` | — | Start the SSO redirect (auth-code + PKCE) | +| GET | `/api/auth/oidc/callback` | — | SSO provider callback; returns a one-time code | +| POST | `/api/auth/oidc/exchange` | — | Trade the one-time code for a bearer token | | GET | `/api/auth/me` | (any authenticated) | Current identity + roles | | GET | `/api/system/stats` | `system:read` | Totals: enqueued, processing, delayed, completed | | GET | `/api/system/version` | `system:read` | Build/version info | @@ -146,9 +215,21 @@ requires a JWT bearer token and a role that carries the relevant permission. | GET | `/api/metrics/throughput` | `metrics:read` | Time-bucketed throughput series | Auth is configured from the environment (see [Configuration](#configuration)). -If nothing is configured, the server bootstraps a single `admin` user and -**logs the generated password** to stdout — pin it with `FORGEQUEUE_ADMIN_PASSWORD` -or declare real users with `FORGEQUEUE_USERS`. +The recommended setup is **single sign-on**: point the server at any OpenID +Connect provider (Google, Okta, Auth0, Keycloak, Entra ID) with +`FORGEQUEUE_OIDC_ISSUER` and the console offers a "Continue with …" button — the +provider owns accounts, passwords, MFA and revocation, so there is nothing to +create or store here. Roles are assigned by email allowlist +(`FORGEQUEUE_OIDC_ADMIN_EMAILS`, `FORGEQUEUE_OIDC_OPERATOR_EMAILS`) or a default +role for everyone else. + +For a local, provider-less deployment you can instead use username/password +accounts. If nothing is configured, the server bootstraps a single `admin` user +and **logs the generated password** to stdout — pin it with +`FORGEQUEUE_ADMIN_PASSWORD` or declare real users with `FORGEQUEUE_USERS`. When +SSO is enabled and no local accounts are configured, the bootstrap admin is +skipped entirely (set `FORGEQUEUE_ADMIN_PASSWORD` or `FORGEQUEUE_USERS` if you +want a break-glass account). > Because successful ACKs delete `task:`, a **completed** job no longer has a > body and `GET /api/jobs/{id}` returns `404`. Completed jobs are meant to be @@ -210,6 +291,17 @@ The batch's heartbeats are only stopped **after** the ACK/requeue is committed. | API users (JSON) | `FORGEQUEUE_USERS` | bootstrapped admin | | Bootstrap admin user | `FORGEQUEUE_ADMIN_USER` | `admin` | | Bootstrap admin password | `FORGEQUEUE_ADMIN_PASSWORD` | random (logged) | +| SSO issuer (enables OIDC) | `FORGEQUEUE_OIDC_ISSUER` | disabled | +| SSO client id | `FORGEQUEUE_OIDC_CLIENT_ID` | — | +| SSO client secret | `FORGEQUEUE_OIDC_CLIENT_SECRET` | — (PKCE; optional) | +| SSO callback URL | `FORGEQUEUE_OIDC_REDIRECT_URL` | — | +| SSO provider label | `FORGEQUEUE_OIDC_PROVIDER_NAME` | issuer host | +| SSO scopes | `FORGEQUEUE_OIDC_SCOPES` | `openid email profile` | +| SSO admin emails | `FORGEQUEUE_OIDC_ADMIN_EMAILS` | — | +| SSO operator emails | `FORGEQUEUE_OIDC_OPERATOR_EMAILS` | — | +| SSO default role | `FORGEQUEUE_OIDC_DEFAULT_ROLE` | `viewer` (`none` to deny) | +| SSO allowed return origins | `FORGEQUEUE_OIDC_ALLOWED_ORIGINS` | web URL only | +| Console web URL | `FORGEQUEUE_WEB_URL` | `/` | | Lease duration | `wp.leaseDuration` (unexported) | 10s | | Heartbeat interval | Derived | `leaseDuration / 3` | | Recovery interval | `wp.recoveryInterval` | 1s | @@ -238,6 +330,23 @@ FORGEQUEUE_REDIS_ADDR=localhost:6379 \ `FORGEQUEUE_USERS` it bootstraps an `admin` user — set `FORGEQUEUE_ADMIN_PASSWORD` here so you know the password. Then `POST /api/auth/login` to get a token.) +To run behind SSO instead, register a confidential/web client with your provider +and set the callback to `/api/auth/oidc/callback`: + +```bash +FORGEQUEUE_REDIS_ADDR=localhost:6379 \ + FORGEQUEUE_AUTH_SECRET=devsecret \ + FORGEQUEUE_OIDC_ISSUER=https://accounts.google.com \ + FORGEQUEUE_OIDC_CLIENT_ID=xxxx.apps.googleusercontent.com \ + FORGEQUEUE_OIDC_CLIENT_SECRET=xxxx \ + FORGEQUEUE_OIDC_REDIRECT_URL=http://localhost:8080/api/auth/oidc/callback \ + FORGEQUEUE_WEB_URL=http://localhost:3000 \ + FORGEQUEUE_OIDC_ADMIN_EMAILS=you@example.com ./apiserver +``` + +For local testing without a real provider, [Dex](https://dexidp.io) or Keycloak +both work; point `FORGEQUEUE_OIDC_ISSUER` at their discovery URL. + Then load-test with the bundled generator (talks to the gateway over gRPC): ```bash @@ -268,16 +377,34 @@ Journal and reconciler tests exist under `internal/durable/` and require Postgre ## Benchmarks -Measure pure drain throughput with a no-op handler: +Measure pure drain throughput with a no-op handler (worker concurrency 4, 50k +preloaded tasks): ```bash go test ./internal/worker/ -run=^$ -bench BenchmarkWorkerThroughputPreloaded -benchtime=10s -benchmem go test ./internal/worker/ -run=^$ -bench BenchmarkWorkerThroughputBatched -benchtime=10s -benchmem ``` -Typical results on Apple Silicon (M2) with local Redis: ~8,000 to 12,000 tasks/s -single-task, higher with `BATCH_SIZE > 1`. The benchmarks measure queue overhead -only (Redis round trips plus heartbeats); real workloads will differ. +Results on an Apple M2 with a local Redis. Handlers are no-ops, so this is pure +queue overhead (Redis round trips plus heartbeats): + +| Implementation | Claim style | Throughput | +| --- | --- | --- | +| ForgeQueue | one task per claim (`BATCH_SIZE=1`) | ~4,100 tasks/s | +| ForgeQueue | batch claim of 16 (`BATCH_SIZE=16`) | ~14,600 tasks/s | +| BullMQ 5 | one job per claim (concurrency 4) | ~12,600 jobs/s | + +The batch claim is where ForgeQueue pulls ahead: at concurrency 4 a one-task +claim pays a full Redis round trip per task, while a 16-task claim amortizes one +round trip across the batch. Out of the box (`BATCH_SIZE=1`) it is *slower* than +BullMQ — the win only shows up once batching is on. Reproduce the BullMQ side +with the bundled same-machine harness: + +```bash +cd bench/bullmq && npm install && JOBS=50000 CONCURRENCY=4 node bench.js +``` + +Real handlers dominate throughput in practice, so treat these as an upper bound. ## Failure Modes @@ -298,8 +425,8 @@ only (Redis round trips plus heartbeats); real workloads will differ. ## Limitations - **No exactly-once.** Delivery is at-least-once: a lease can expire (worker crash, Redis blip, GC pause) and a task can run twice. Handlers must be idempotent. -- **Single Redis.** No replication, sharding, or failover logic — durability is whatever the Redis deployment provides. -- **Partial auth.** The HTTP API has JWT + RBAC. The **gRPC gateway and Redis have no authentication or TLS**, and the HTTP API itself is plain HTTP. Do not expose any of it to a public network. +- **Single Redis.** No replication, sharding, or failover logic — durability is whatever the Redis deployment provides. The bundled Kubernetes manifest runs one AOF-backed Redis on a PersistentVolumeClaim, but that is still a single point of failure. +- **Partial auth.** The HTTP API has JWT + RBAC, with optional OIDC/SSO login. The **gRPC gateway and Redis have no authentication or TLS**, and the HTTP API itself is plain HTTP. Do not expose any of it to a public network. - **No multi-tenancy / queue isolation.** "Queues" are a single logical namespace, not separate Redis keyspaces or broker connections. - **Completed bodies are deleted**, so a finished job cannot be looked up by ID afterwards (see the HTTP API note above). - **Idle polling** uses a short sleep interval per worker; blocking variants (e.g. `BLMOVE`) are not implemented. @@ -321,12 +448,13 @@ only (Redis round trips plus heartbeats); real workloads will differ. - Prometheus metrics (worker + gateway) and a throughput/stats HTTP API - HTTP/JSON control API (pause/resume/purge, retry/requeue/delete, DLQ ops, delayed ops) - JWT + RBAC auth for the HTTP API +- OIDC / SSO login (authorization code + PKCE) with email-based role mapping - React + TanStack Start web console with login - gRPC load generator **Planned / explicitly out of scope:** - Smarter idle polling (blocking operations) -- Multi-instance deployment hardening +- HA Redis (replication/Sentinel) — the Kubernetes manifest runs a single AOF-backed Redis - AuthN/AuthZ and multi-tenancy - A streaming/Kafka-style log layer (deliberately not pursued — different problem) diff --git a/cmd/demoenqueue/main.go b/cmd/demoenqueue/main.go index 2a252db..74aab25 100644 --- a/cmd/demoenqueue/main.go +++ b/cmd/demoenqueue/main.go @@ -11,6 +11,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" queuev1 "forgequeue/proto/v1" ) @@ -18,9 +19,11 @@ import ( func main() { rounds := flag.Int("n", 1, "how many times to send the batch of tasks") interval := flag.Duration("interval", 0, "wait between rounds") + addr := flag.String("gateway", "localhost:50051", "gateway gRPC address") + token := flag.String("token", "", "bearer token (gateway rejects anonymous calls)") flag.Parse() - conn, err := grpc.NewClient("localhost:50051", + conn, err := grpc.NewClient(*addr, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { log.Fatal(err) @@ -29,9 +32,16 @@ func main() { c := queuev1.NewQueueServiceClient(conn) - // Two 8s sleeps per round. Long enough that the p95 graph has a real - // curve in it, and that a task is still in flight whenever you go to - // screenshot, which is what makes the queue-depth panel look alive. + // Auth header is applied per-call; unauthenticated calls hit the + // interceptor's Unauthenticated error. + authorized := func(ctx context.Context) context.Context { + if *token != "" { + return metadata.AppendToOutgoingContext(ctx, "authorization", "Bearer "+*token) + } + return ctx + } + + // Two 8s sleeps per round so something is in flight when you screenshot. sleep8 := []byte(`{"duration_ms":8000}`) jobs := []struct { @@ -54,10 +64,10 @@ func main() { fmt.Printf("round %d/%d\n", r, *rounds) } for _, j := range jobs { - // Per-task timeout, not one timeout for the whole run: a slow - // round should not abort the rounds after it. + // Time out each task on its own so one slow round does not abort + // the rest. ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - resp, err := c.EnqueueTask(ctx, &queuev1.EnqueueTaskRequest{ + resp, err := c.EnqueueTask(authorized(ctx), &queuev1.EnqueueTaskRequest{ TaskType: j.kind, Payload: j.payload, MaxRetries: 3, diff --git a/cmd/loadgen/main.go b/cmd/loadgen/main.go index a7f8c86..f042869 100644 --- a/cmd/loadgen/main.go +++ b/cmd/loadgen/main.go @@ -1,10 +1,5 @@ -// Command loadgen drives ForgeQueue with a continuous, realistic mix of tasks -// so the dashboard has live traffic to show. Unlike demoenqueue (a fixed -// burst), loadgen runs until stopped and keeps a steady target rate. -// -// It enqueues over the same gRPC API the gateway exposes, choosing each task -// from a weighted mix of echo, sleep, fail and delayed work. Results are -// reported as a running summary once a second. +// Command loadgen sends a steady mix of tasks over the gateway gRPC API so the +// dashboard has live traffic. package main import ( @@ -22,12 +17,14 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" queuev1 "forgequeue/proto/v1" ) func main() { addr := flag.String("gateway", "localhost:50051", "gateway gRPC address") + token := flag.String("token", "", "bearer token (gateway rejects anonymous calls)") rate := flag.Float64("rate", 40, "target enqueue rate in tasks per second") concurrency := flag.Int("concurrency", 8, "parallel enqueue goroutines") duration := flag.Duration("duration", 0, "run time; 0 means run until interrupted") @@ -41,7 +38,7 @@ func main() { seed := flag.Int64("seed", time.Now().UnixNano(), "random seed") flag.Parse() - // Each goroutine owns its own rand.Source, which is not safe to share. + // rand.Rand is not safe to share, so each goroutine seeds its own. rng := rand.New(rand.NewSource(*seed)) conn, err := grpc.NewClient(*addr, @@ -120,6 +117,9 @@ func main() { // Cap each call so one slow enqueue cannot stall the // goroutine forever. callCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + if *token != "" { + callCtx = metadata.AppendToOutgoingContext(callCtx, "authorization", "Bearer "+*token) + } _, err := client.EnqueueTask(callCtx, req) cancel() if err != nil { @@ -161,9 +161,8 @@ func pick(r *rand.Rand, mix []struct { return "echo" } -// buildRequest turns a task kind into a concrete gRPC request. Sleep tasks get -// a random duration and delayed tasks a random delay, so neither the latency -// graph nor the delayed panel is flat. +// buildRequest builds a request for kind, with random durations for sleep and +// delayed tasks. func buildRequest(r *rand.Rand, kind string, maxRetries, sleepMaxMs, delayMaxS int) *queuev1.EnqueueTaskRequest { req := &queuev1.EnqueueTaskRequest{ TaskType: kind, @@ -181,7 +180,7 @@ func buildRequest(r *rand.Rand, kind string, maxRetries, sleepMaxMs, delayMaxS i req.TaskType = "echo" req.Payload = []byte(`{"note":"delayed from loadgen"}`) req.DelaySeconds = int32(1 + r.Intn(delayMaxS)) - default: // echo + default: req.TaskType = "echo" req.Payload = []byte(fmt.Sprintf(`{"note":"loadgen echo","n":%d}`, r.Intn(1_000_000))) } diff --git a/deploy/k8s/base/apiserver.yaml b/deploy/k8s/base/apiserver.yaml new file mode 100644 index 0000000..0c26187 --- /dev/null +++ b/deploy/k8s/base/apiserver.yaml @@ -0,0 +1,88 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: apiserver + labels: + app.kubernetes.io/name: apiserver + app.kubernetes.io/part-of: forgequeue +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: apiserver + template: + metadata: + labels: + app.kubernetes.io/name: apiserver + app.kubernetes.io/part-of: forgequeue + spec: + terminationGracePeriodSeconds: 30 + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + initContainers: + - name: wait-for-redis + image: busybox:1.36 + command: ["sh", "-c", "until nc -z redis 6379; do echo waiting for redis; sleep 1; done"] + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + capabilities: + drop: ["ALL"] + containers: + - name: apiserver + image: forgequeue/apiserver:latest + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + name: forgequeue-config + - secretRef: + name: forgequeue-secrets + env: + - name: FORGEQUEUE_REDIS_ADDR + value: redis:6379 + ports: + - name: http + containerPort: 8080 + readinessProbe: + httpGet: + path: /readyz + port: http + initialDelaySeconds: 3 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 10 + periodSeconds: 20 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] +--- +apiVersion: v1 +kind: Service +metadata: + name: apiserver + labels: + app.kubernetes.io/name: apiserver + app.kubernetes.io/part-of: forgequeue +spec: + selector: + app.kubernetes.io/name: apiserver + ports: + - name: http + port: 8080 + targetPort: http diff --git a/deploy/k8s/base/gateway.yaml b/deploy/k8s/base/gateway.yaml new file mode 100644 index 0000000..cd2b262 --- /dev/null +++ b/deploy/k8s/base/gateway.yaml @@ -0,0 +1,112 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gateway + labels: + app.kubernetes.io/name: gateway + app.kubernetes.io/part-of: forgequeue +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: gateway + template: + metadata: + labels: + app.kubernetes.io/name: gateway + app.kubernetes.io/part-of: forgequeue + spec: + terminationGracePeriodSeconds: 30 + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + initContainers: + - name: wait-for-redis + image: busybox:1.36 + command: ["sh", "-c", "until nc -z redis 6379; do echo waiting for redis; sleep 1; done"] + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + capabilities: + drop: ["ALL"] + - name: wait-for-postgres + image: busybox:1.36 + command: ["sh", "-c", "until nc -z postgres 5432; do echo waiting for postgres; sleep 1; done"] + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + capabilities: + drop: ["ALL"] + containers: + - name: gateway + image: forgequeue/gateway:latest + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + name: forgequeue-config + - secretRef: + name: forgequeue-secrets + env: + - name: FORGEQUEUE_REDIS_ADDR + value: redis:6379 + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: forgequeue-secrets + key: POSTGRES_PASSWORD + # The journal DSN is derived from the shared Postgres password so + # the credential lives in exactly one place. + - name: FORGEQUEUE_PG_DSN + value: "postgres://forgequeue:$(POSTGRES_PASSWORD)@postgres:5432/forgequeue?sslmode=disable" + ports: + - name: grpc + containerPort: 50051 + - name: metrics + containerPort: 9100 + readinessProbe: + httpGet: + path: /readyz + port: metrics + initialDelaySeconds: 3 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /healthz + port: metrics + initialDelaySeconds: 10 + periodSeconds: 20 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] +--- +apiVersion: v1 +kind: Service +metadata: + name: gateway + labels: + app.kubernetes.io/name: gateway + app.kubernetes.io/part-of: forgequeue +spec: + selector: + app.kubernetes.io/name: gateway + ports: + - name: grpc + port: 50051 + targetPort: grpc + - name: metrics + port: 9100 + targetPort: metrics diff --git a/deploy/k8s/base/hpa.yaml b/deploy/k8s/base/hpa.yaml new file mode 100644 index 0000000..dbcf222 --- /dev/null +++ b/deploy/k8s/base/hpa.yaml @@ -0,0 +1,41 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: worker + labels: + app.kubernetes.io/part-of: forgequeue +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: worker + minReplicas: 1 + maxReplicas: 5 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: apiserver + labels: + app.kubernetes.io/part-of: forgequeue +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: apiserver + minReplicas: 1 + maxReplicas: 5 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 diff --git a/deploy/k8s/base/ingress.yaml b/deploy/k8s/base/ingress.yaml new file mode 100644 index 0000000..75a93f2 --- /dev/null +++ b/deploy/k8s/base/ingress.yaml @@ -0,0 +1,29 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: forgequeue + labels: + app.kubernetes.io/part-of: forgequeue + annotations: + nginx.ingress.kubernetes.io/proxy-read-timeout: "3600" + nginx.ingress.kubernetes.io/proxy-send-timeout: "3600" +spec: + ingressClassName: nginx + rules: + - host: forgequeue.local + http: + paths: + - path: /api + pathType: Prefix + backend: + service: + name: apiserver + port: + number: 8080 + - path: / + pathType: Prefix + backend: + service: + name: web + port: + number: 80 diff --git a/deploy/k8s/base/kustomization.yaml b/deploy/k8s/base/kustomization.yaml new file mode 100644 index 0000000..e94c6cc --- /dev/null +++ b/deploy/k8s/base/kustomization.yaml @@ -0,0 +1,28 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: forgequeue + +resources: + - namespace.yaml + - redis.yaml + - postgres.yaml + - worker.yaml + - gateway.yaml + - apiserver.yaml + - web.yaml + - networkpolicy.yaml + - ingress.yaml + - hpa.yaml + - pdb.yaml + +generatorOptions: + disableNameSuffixHash: true + +configMapGenerator: + - name: forgequeue-config + literals: + - FORGEQUEUE_WORKER_CONCURRENCY=4 + - FORGEQUEUE_BATCH_SIZE=1 + - FORGEQUEUE_METRICS_ADDR=:9100 + - FORGEQUEUE_API_ADDR=:8080 diff --git a/deploy/k8s/base/namespace.yaml b/deploy/k8s/base/namespace.yaml new file mode 100644 index 0000000..0aa247d --- /dev/null +++ b/deploy/k8s/base/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: forgequeue + labels: + app.kubernetes.io/part-of: forgequeue diff --git a/deploy/k8s/base/networkpolicy.yaml b/deploy/k8s/base/networkpolicy.yaml new file mode 100644 index 0000000..9ce7160 --- /dev/null +++ b/deploy/k8s/base/networkpolicy.yaml @@ -0,0 +1,47 @@ +# Redis holds live queue state, so only the queue services may reach it. This +# is the network half of the boundary; the gRPC gateway adds token auth on top. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: redis + labels: + app.kubernetes.io/part-of: forgequeue +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: redis + policyTypes: ["Ingress"] + ingress: + - from: + - podSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: ["apiserver", "gateway", "worker"] + ports: + - protocol: TCP + port: 6379 +--- +# The durable journal. Only the gateway (writes) and workers (follow state, +# reconcile) connect to Postgres. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: postgres + labels: + app.kubernetes.io/part-of: forgequeue +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: postgres + policyTypes: ["Ingress"] + ingress: + - from: + - podSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: ["gateway", "worker"] + ports: + - protocol: TCP + port: 5432 diff --git a/deploy/k8s/base/pdb.yaml b/deploy/k8s/base/pdb.yaml new file mode 100644 index 0000000..bd707e9 --- /dev/null +++ b/deploy/k8s/base/pdb.yaml @@ -0,0 +1,35 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: worker + labels: + app.kubernetes.io/part-of: forgequeue +spec: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: worker +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: gateway + labels: + app.kubernetes.io/part-of: forgequeue +spec: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: gateway +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: apiserver + labels: + app.kubernetes.io/part-of: forgequeue +spec: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: apiserver diff --git a/deploy/k8s/base/postgres.yaml b/deploy/k8s/base/postgres.yaml new file mode 100644 index 0000000..96a8951 --- /dev/null +++ b/deploy/k8s/base/postgres.yaml @@ -0,0 +1,90 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: postgres + labels: + app.kubernetes.io/name: postgres + app.kubernetes.io/part-of: forgequeue +spec: + serviceName: postgres + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: postgres + template: + metadata: + labels: + app.kubernetes.io/name: postgres + app.kubernetes.io/part-of: forgequeue + spec: + securityContext: + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + fsGroup: 999 + containers: + - name: postgres + image: postgres:16 + env: + - name: POSTGRES_DB + value: forgequeue + - name: POSTGRES_USER + value: forgequeue + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: forgequeue-secrets + key: POSTGRES_PASSWORD + # Keep the cluster data in a subdir so the volume root can hold the + # lost+found directory some CSI drivers create. + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + ports: + - name: postgres + containerPort: 5432 + readinessProbe: + exec: + command: ["pg_isready", "-U", "forgequeue", "-d", "forgequeue"] + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + exec: + command: ["pg_isready", "-U", "forgequeue", "-d", "forgequeue"] + initialDelaySeconds: 15 + periodSeconds: 20 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + volumeMounts: + - name: data + mountPath: /var/lib/postgresql/data + volumeClaimTemplates: + - metadata: + name: data + labels: + app.kubernetes.io/name: postgres + app.kubernetes.io/part-of: forgequeue + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres + labels: + app.kubernetes.io/name: postgres + app.kubernetes.io/part-of: forgequeue +spec: + selector: + app.kubernetes.io/name: postgres + ports: + - name: postgres + port: 5432 + targetPort: postgres diff --git a/deploy/k8s/base/redis.yaml b/deploy/k8s/base/redis.yaml new file mode 100644 index 0000000..3cc82bf --- /dev/null +++ b/deploy/k8s/base/redis.yaml @@ -0,0 +1,87 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: redis + labels: + app.kubernetes.io/name: redis + app.kubernetes.io/part-of: forgequeue +spec: + serviceName: redis + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: redis + template: + metadata: + labels: + app.kubernetes.io/name: redis + app.kubernetes.io/part-of: forgequeue + spec: + securityContext: + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + fsGroup: 999 + containers: + - name: redis + image: redis:7.4-alpine + # AOF is the durability mechanism: appendonly + everysec fsync, plus + # an RDB snapshot every 60s as a fast restart path. The appendonly + # data lives on a PVC so it outlives the pod. + args: + - "--appendonly" + - "yes" + - "--appendfsync" + - "everysec" + - "--save" + - "60" + - "1000" + ports: + - name: redis + containerPort: 6379 + readinessProbe: + exec: + command: ["redis-cli", "ping"] + initialDelaySeconds: 3 + periodSeconds: 5 + livenessProbe: + exec: + command: ["redis-cli", "ping"] + initialDelaySeconds: 10 + periodSeconds: 10 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 250m + memory: 256Mi + volumeMounts: + - name: data + mountPath: /data + volumeClaimTemplates: + - metadata: + name: data + labels: + app.kubernetes.io/name: redis + app.kubernetes.io/part-of: forgequeue + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: redis + labels: + app.kubernetes.io/name: redis + app.kubernetes.io/part-of: forgequeue +spec: + selector: + app.kubernetes.io/name: redis + ports: + - name: redis + port: 6379 + targetPort: redis diff --git a/deploy/k8s/base/web.yaml b/deploy/k8s/base/web.yaml new file mode 100644 index 0000000..0738755 --- /dev/null +++ b/deploy/k8s/base/web.yaml @@ -0,0 +1,75 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: web + labels: + app.kubernetes.io/name: web + app.kubernetes.io/part-of: forgequeue +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: web + template: + metadata: + labels: + app.kubernetes.io/name: web + app.kubernetes.io/part-of: forgequeue + spec: + terminationGracePeriodSeconds: 15 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + containers: + - name: web + image: forgequeue/web:latest + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 3000 + readinessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 3 + periodSeconds: 10 + livenessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 10 + periodSeconds: 20 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: web + labels: + app.kubernetes.io/name: web + app.kubernetes.io/part-of: forgequeue +spec: + selector: + app.kubernetes.io/name: web + ports: + - name: http + port: 80 + targetPort: http diff --git a/deploy/k8s/base/worker.yaml b/deploy/k8s/base/worker.yaml new file mode 100644 index 0000000..1c11126 --- /dev/null +++ b/deploy/k8s/base/worker.yaml @@ -0,0 +1,107 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: worker + labels: + app.kubernetes.io/name: worker + app.kubernetes.io/part-of: forgequeue +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: worker + template: + metadata: + labels: + app.kubernetes.io/name: worker + app.kubernetes.io/part-of: forgequeue + spec: + terminationGracePeriodSeconds: 30 + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + initContainers: + - name: wait-for-redis + image: busybox:1.36 + command: ["sh", "-c", "until nc -z redis 6379; do echo waiting for redis; sleep 1; done"] + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + capabilities: + drop: ["ALL"] + - name: wait-for-postgres + image: busybox:1.36 + command: ["sh", "-c", "until nc -z postgres 5432; do echo waiting for postgres; sleep 1; done"] + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + capabilities: + drop: ["ALL"] + containers: + - name: worker + image: forgequeue/worker:latest + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + name: forgequeue-config + - secretRef: + name: forgequeue-secrets + env: + - name: FORGEQUEUE_REDIS_ADDR + value: redis:6379 + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: forgequeue-secrets + key: POSTGRES_PASSWORD + # Attaching the journal turns on the worker's reconciler, which + # rebuilds any non-terminal task Redis is missing. + - name: FORGEQUEUE_PG_DSN + value: "postgres://forgequeue:$(POSTGRES_PASSWORD)@postgres:5432/forgequeue?sslmode=disable" + ports: + - name: metrics + containerPort: 9100 + readinessProbe: + httpGet: + path: /readyz + port: metrics + initialDelaySeconds: 3 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /healthz + port: metrics + initialDelaySeconds: 10 + periodSeconds: 20 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] +--- +apiVersion: v1 +kind: Service +metadata: + name: worker + labels: + app.kubernetes.io/name: worker + app.kubernetes.io/part-of: forgequeue +spec: + selector: + app.kubernetes.io/name: worker + ports: + - name: metrics + port: 9100 + targetPort: metrics diff --git a/deploy/k8s/kind-config.yaml b/deploy/k8s/kind-config.yaml new file mode 100644 index 0000000..f68c9b4 --- /dev/null +++ b/deploy/k8s/kind-config.yaml @@ -0,0 +1,8 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane + extraMounts: + - hostPath: /Users/yashbahuguna/forgequeue/deploy/k8s + containerPath: /etc/k8s-manifests +- role: worker diff --git a/deploy/k8s/observability/kustomization.yaml b/deploy/k8s/observability/kustomization.yaml new file mode 100644 index 0000000..fe2f8b2 --- /dev/null +++ b/deploy/k8s/observability/kustomization.yaml @@ -0,0 +1,14 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Opt-in: apply these only after kube-prometheus-stack is installed, because +# ServiceMonitor depends on the monitoring.coreos.com CRDs. +# +# helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +# helm upgrade --install kube-prometheus-stack prometheus-community/kube-prometheus-stack -n monitoring --create-namespace +# kubectl apply -k deploy/k8s/observability + +namespace: forgequeue + +resources: + - servicemonitor.yaml diff --git a/deploy/k8s/observability/servicemonitor.yaml b/deploy/k8s/observability/servicemonitor.yaml new file mode 100644 index 0000000..2d2928f --- /dev/null +++ b/deploy/k8s/observability/servicemonitor.yaml @@ -0,0 +1,18 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: forgequeue + labels: + app.kubernetes.io/part-of: forgequeue + release: kube-prometheus-stack +spec: + namespaceSelector: + matchNames: ["forgequeue"] + selector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: ["worker", "gateway"] + endpoints: + - port: metrics + interval: 15s diff --git a/deploy/k8s/overlays/kind/kustomization.yaml b/deploy/k8s/overlays/kind/kustomization.yaml new file mode 100644 index 0000000..aa443ee --- /dev/null +++ b/deploy/k8s/overlays/kind/kustomization.yaml @@ -0,0 +1,31 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: forgequeue + +resources: + - ../../base + +generatorOptions: + disableNameSuffixHash: true + +# Dev-only credentials for the local kind cluster. Never use these in prod. +secretGenerator: + - name: forgequeue-secrets + literals: + - FORGEQUEUE_AUTH_SECRET=devsecret-change-me + - FORGEQUEUE_ADMIN_USER=admin + - FORGEQUEUE_ADMIN_PASSWORD=devpassword + # Shared by the Postgres StatefulSet (POSTGRES_PASSWORD) and the derived + # FORGEQUEUE_PG_DSN that gateway/worker build in the base manifests. + - POSTGRES_PASSWORD=devpostgres + +images: + - name: forgequeue/worker + newTag: latest + - name: forgequeue/gateway + newTag: latest + - name: forgequeue/apiserver + newTag: latest + - name: forgequeue/web + newTag: latest diff --git a/deploy/k8s/overlays/prod/kustomization.yaml b/deploy/k8s/overlays/prod/kustomization.yaml new file mode 100644 index 0000000..5a70e13 --- /dev/null +++ b/deploy/k8s/overlays/prod/kustomization.yaml @@ -0,0 +1,36 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: forgequeue + +resources: + - ../../base + +# forgequeue-secrets is applied out of band (SOPS + age), not generated here. +# It MUST include FORGEQUEUE_AUTH_SECRET, FORGEQUEUE_ADMIN_USER/PASSWORD, and +# POSTGRES_PASSWORD (the base derives FORGEQUEUE_PG_DSN from it). For an +# external managed Postgres, override FORGEQUEUE_PG_DSN in the secret too. + +images: + - name: forgequeue/worker + newName: ghcr.io/yashbhu/forgequeue-worker + newTag: v1.0.0 + - name: forgequeue/gateway + newName: ghcr.io/yashbhu/forgequeue-gateway + newTag: v1.0.0 + - name: forgequeue/apiserver + newName: ghcr.io/yashbhu/forgequeue-apiserver + newTag: v1.0.0 + - name: forgequeue/web + newName: ghcr.io/yashbhu/forgequeue-web + newTag: v1.0.0 + +replicas: + - name: worker + count: 3 + - name: gateway + count: 2 + - name: apiserver + count: 2 + - name: web + count: 2 diff --git a/deploy/prometheus.yml b/deploy/prometheus.yml index 099e548..05e0558 100644 --- a/deploy/prometheus.yml +++ b/deploy/prometheus.yml @@ -29,4 +29,4 @@ scrape_configs: static_configs: - targets: ["localhost:9091"] labels: - component: worker \ No newline at end of file + component: worker diff --git a/dockerfiles/apiserver.Dockerfile b/dockerfiles/apiserver.Dockerfile new file mode 100644 index 0000000..ffd4125 --- /dev/null +++ b/dockerfiles/apiserver.Dockerfile @@ -0,0 +1,18 @@ +# syntax=docker/dockerfile:1 + +FROM golang:1.26-alpine AS build +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY cmd ./cmd +COPY internal ./internal +COPY proto ./proto +RUN CGO_ENABLED=0 GOOS=linux go build \ + -trimpath -ldflags="-s -w" \ + -o /out/apiserver ./cmd/apiserver + +FROM gcr.io/distroless/static-debian12:nonroot +COPY --from=build /out/apiserver /apiserver +USER nonroot:nonroot +EXPOSE 8080 +ENTRYPOINT ["/apiserver"] diff --git a/dockerfiles/gateway.Dockerfile b/dockerfiles/gateway.Dockerfile new file mode 100644 index 0000000..e5d5c9d --- /dev/null +++ b/dockerfiles/gateway.Dockerfile @@ -0,0 +1,18 @@ +# syntax=docker/dockerfile:1 + +FROM golang:1.26-alpine AS build +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY cmd ./cmd +COPY internal ./internal +COPY proto ./proto +RUN CGO_ENABLED=0 GOOS=linux go build \ + -trimpath -ldflags="-s -w" \ + -o /out/gateway ./cmd/gateway + +FROM gcr.io/distroless/static-debian12:nonroot +COPY --from=build /out/gateway /gateway +USER nonroot:nonroot +EXPOSE 50051 9100 +ENTRYPOINT ["/gateway"] diff --git a/dockerfiles/web.Dockerfile b/dockerfiles/web.Dockerfile new file mode 100644 index 0000000..c0aefb0 --- /dev/null +++ b/dockerfiles/web.Dockerfile @@ -0,0 +1,26 @@ +# syntax=docker/dockerfile:1 + +FROM oven/bun:1 AS build +WORKDIR /src + +COPY web/package.json web/bun.lock web/bunfig.toml ./ +RUN bun install --frozen-lockfile + +COPY web/ ./ + +# The API base is baked at build time. Keep it relative so the browser hits the +# web origin and the ingress can route /api to the apiserver service. +ARG VITE_API_BASE_URL=/api +ENV VITE_API_BASE_URL=$VITE_API_BASE_URL +ENV NITRO_PRESET=node-server +RUN bun run build + +FROM node:22-alpine AS runtime +WORKDIR /app +ENV NODE_ENV=production +ENV HOST=0.0.0.0 +ENV PORT=3000 +COPY --from=build /src/.output ./.output +USER node +EXPOSE 3000 +CMD ["node", ".output/server/index.mjs"] diff --git a/dockerfiles/worker.Dockerfile b/dockerfiles/worker.Dockerfile new file mode 100644 index 0000000..a17fdfa --- /dev/null +++ b/dockerfiles/worker.Dockerfile @@ -0,0 +1,18 @@ +# syntax=docker/dockerfile:1 + +FROM golang:1.26-alpine AS build +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY cmd ./cmd +COPY internal ./internal +COPY proto ./proto +RUN CGO_ENABLED=0 GOOS=linux go build \ + -trimpath -ldflags="-s -w" \ + -o /out/worker ./cmd/worker + +FROM gcr.io/distroless/static-debian12:nonroot +COPY --from=build /out/worker /worker +USER nonroot:nonroot +EXPOSE 9100 +ENTRYPOINT ["/worker"] diff --git a/internal/auth/config.go b/internal/auth/config.go index 763f6ba..d895860 100644 --- a/internal/auth/config.go +++ b/internal/auth/config.go @@ -4,6 +4,7 @@ import ( "crypto/rand" "encoding/base64" "encoding/json" + "errors" "fmt" "log" "os" @@ -18,17 +19,15 @@ type userConfig struct { Roles []string `json:"roles"` } -// NewFromEnv assembles a Service from environment configuration: +// NewFromEnv builds a Service from the environment. // -// FORGEQUEUE_AUTH_SECRET HS256 signing secret (required in production; -// a random one is generated with a warning if unset) -// FORGEQUEUE_AUTH_TTL token lifetime as a Go duration (default 12h) -// FORGEQUEUE_USERS JSON array of {username, password, roles} -// FORGEQUEUE_ADMIN_USER bootstrap admin username (default "admin") +// FORGEQUEUE_AUTH_SECRET HS256 signing secret (random if unset) +// FORGEQUEUE_AUTH_TTL token lifetime (default 12h) +// FORGEQUEUE_USERS JSON array of {username, password, roles} +// FORGEQUEUE_ADMIN_USER bootstrap admin username (default "admin") // FORGEQUEUE_ADMIN_PASSWORD bootstrap admin password (random if unset) // -// When FORGEQUEUE_USERS is empty, a single admin account is bootstrapped so a -// fresh deployment is never left with no way in. +// With no FORGEQUEUE_USERS, a single admin is bootstrapped. func NewFromEnv() (*Service, error) { secret, err := resolveSecret() if err != nil { @@ -45,12 +44,15 @@ func NewFromEnv() (*Service, error) { } var users []User - if raw := strings.TrimSpace(os.Getenv("FORGEQUEUE_USERS")); raw != "" { - users, err = parseUsers(raw) + switch { + case strings.TrimSpace(os.Getenv("FORGEQUEUE_USERS")) != "": + users, err = parseUsers(os.Getenv("FORGEQUEUE_USERS")) if err != nil { return nil, err } - } else { + case oidcConfigured() && strings.TrimSpace(os.Getenv("FORGEQUEUE_ADMIN_PASSWORD")) == "": + log.Printf("auth: OIDC is enabled; no local accounts configured (set FORGEQUEUE_USERS or FORGEQUEUE_ADMIN_PASSWORD for break-glass access)") + default: users, err = bootstrapAdmin() if err != nil { return nil, err @@ -60,6 +62,95 @@ func NewFromEnv() (*Service, error) { return NewService(users, secret, ttl) } +// oidcConfigured reports whether single sign-on is enabled via the environment. +func oidcConfigured() bool { + return strings.TrimSpace(os.Getenv("FORGEQUEUE_OIDC_ISSUER")) != "" +} + +// NewOIDCFromEnv builds the SSO provider from the environment, or returns +// (nil, nil) when FORGEQUEUE_OIDC_ISSUER is unset. +// +// FORGEQUEUE_OIDC_ISSUER provider issuer URL (enables SSO) +// FORGEQUEUE_OIDC_CLIENT_ID OAuth client id +// FORGEQUEUE_OIDC_CLIENT_SECRET OAuth client secret (optional with PKCE) +// FORGEQUEUE_OIDC_REDIRECT_URL our callback URL +// FORGEQUEUE_OIDC_PROVIDER_NAME button label (defaults to issuer host) +// FORGEQUEUE_OIDC_SCOPES space-separated (default "openid email profile") +// FORGEQUEUE_OIDC_ADMIN_EMAILS comma-separated emails mapped to admin +// FORGEQUEUE_OIDC_OPERATOR_EMAILS comma-separated emails mapped to operator +// FORGEQUEUE_OIDC_DEFAULT_ROLE role for others (viewer|operator|admin|none, default viewer) +// FORGEQUEUE_OIDC_ALLOWED_ORIGINS extra origins allowed as post-login redirect targets +// FORGEQUEUE_WEB_URL default post-login redirect (default "/") +func NewOIDCFromEnv(svc *Service, store StateStore) (*OIDC, error) { + if !oidcConfigured() { + return nil, nil + } + + cfg := OIDCConfig{ + Issuer: strings.TrimSpace(os.Getenv("FORGEQUEUE_OIDC_ISSUER")), + ClientID: strings.TrimSpace(os.Getenv("FORGEQUEUE_OIDC_CLIENT_ID")), + ClientSecret: os.Getenv("FORGEQUEUE_OIDC_CLIENT_SECRET"), + RedirectURL: strings.TrimSpace(os.Getenv("FORGEQUEUE_OIDC_REDIRECT_URL")), + WebURL: strings.TrimSpace(os.Getenv("FORGEQUEUE_WEB_URL")), + ProviderName: strings.TrimSpace(os.Getenv("FORGEQUEUE_OIDC_PROVIDER_NAME")), + AdminEmails: splitList(os.Getenv("FORGEQUEUE_OIDC_ADMIN_EMAILS")), + OperatorEmails: splitList(os.Getenv("FORGEQUEUE_OIDC_OPERATOR_EMAILS")), + } + if raw := strings.TrimSpace(os.Getenv("FORGEQUEUE_OIDC_SCOPES")); raw != "" { + cfg.Scopes = strings.Fields(raw) + } + + if raw := strings.TrimSpace(os.Getenv("FORGEQUEUE_OIDC_DEFAULT_ROLE")); raw != "" { + if strings.EqualFold(raw, "none") { + cfg.DefaultRole = "" + } else { + role := Role(raw) + if !ValidRole(role) { + return nil, fmt.Errorf("auth: invalid FORGEQUEUE_OIDC_DEFAULT_ROLE %q", raw) + } + cfg.DefaultRole = role + } + } else { + cfg.DefaultRole = RoleViewer + } + + if cfg.ClientID == "" || cfg.RedirectURL == "" { + return nil, errors.New("auth: FORGEQUEUE_OIDC_CLIENT_ID and FORGEQUEUE_OIDC_REDIRECT_URL are required when FORGEQUEUE_OIDC_ISSUER is set") + } + + oidc, err := NewOIDC(cfg, svc, store) + if err != nil { + return nil, err + } + oidc.SetAllowedOrigins(splitList(os.Getenv("FORGEQUEUE_OIDC_ALLOWED_ORIGINS"))) + + dest := cfg.WebURL + if dest == "" { + dest = "/" + } + log.Printf("auth: OIDC enabled via %s (provider %q, default role %q, redirect %q)", cfg.Issuer, oidc.ProviderName(), roleOrNone(cfg.DefaultRole), dest) + return oidc, nil +} + +func roleOrNone(r Role) string { + if r == "" { + return "none" + } + return string(r) +} + +// splitList parses a comma-separated environment value into trimmed, non-empty +// entries. +func splitList(raw string) []string { + var out []string + for _, part := range strings.Split(raw, ",") { + if p := strings.TrimSpace(part); p != "" { + out = append(out, p) + } + } + return out +} + func parseUsers(raw string) ([]User, error) { var cfgs []userConfig if err := json.Unmarshal([]byte(raw), &cfgs); err != nil { diff --git a/internal/auth/oidc.go b/internal/auth/oidc.go new file mode 100644 index 0000000..852c10c --- /dev/null +++ b/internal/auth/oidc.go @@ -0,0 +1,415 @@ +package auth + +import ( + "context" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "sync" + "time" +) + +// OIDCConfig describes an OpenID Connect provider. Any provider that +// supports discovery works, so only the issuer and client ID are needed. +type OIDCConfig struct { + Issuer string + ClientID string + ClientSecret string // optional: omit for public clients using PKCE + RedirectURL string // our callback, e.g. http://host:8080/api/auth/oidc/callback + WebURL string // where the browser returns after login, e.g. http://localhost:3000 + Scopes []string // defaults to openid email profile + ProviderName string // display name, defaults to the issuer host + DefaultRole Role // role for non-allowlisted users, empty denies + AdminEmails []string // emails granted the admin role + OperatorEmails []string // emails granted the operator role + HTTPClient *http.Client + StateTTL time.Duration // how long a login attempt may take + CodeTTL time.Duration // lifetime of the one-time exchange code +} + +// SSOSession is the result of a completed SSO login. +type SSOSession struct { + Token string + ExpiresAt time.Time + Identity Identity + WebReturn string +} + +// ErrSSOFailed is returned for any failure in the SSO flow. The cause is +// wrapped for logs but kept out of the HTTP response. +var ErrSSOFailed = errors.New("sso authentication failed") + +// OIDC drives the authorization-code + PKCE flow and converts a provider +// identity into a ForgeQueue identity (and JWT). +type OIDC struct { + cfg OIDCConfig + store StateStore + svc *Service + http *http.Client + + admin map[string]bool + op map[string]bool + scopes string + + allowed map[string]bool + allowedDefault string + + mu sync.Mutex + provider *providerMeta +} + +type providerMeta struct { + Issuer string `json:"issuer"` + AuthorizationEndpoint string `json:"authorization_endpoint"` + TokenEndpoint string `json:"token_endpoint"` + UserinfoEndpoint string `json:"userinfo_endpoint"` +} + +type authState struct { + Verifier string `json:"verifier"` + WebReturn string `json:"web_return"` +} + +type userInfo struct { + Subject string `json:"sub"` + Email string `json:"email"` + Name string `json:"name"` + PreferredUsername string `json:"preferred_username"` +} + +func NewOIDC(cfg OIDCConfig, svc *Service, store StateStore) (*OIDC, error) { + if cfg.Issuer == "" || cfg.ClientID == "" || cfg.RedirectURL == "" { + return nil, errors.New("auth: OIDC requires issuer, client id and redirect URL") + } + if store == nil { + return nil, errors.New("auth: OIDC requires a state store") + } + if len(cfg.Scopes) == 0 { + cfg.Scopes = []string{"openid", "email", "profile"} + } + if cfg.StateTTL <= 0 { + cfg.StateTTL = 10 * time.Minute + } + if cfg.CodeTTL <= 0 { + cfg.CodeTTL = 60 * time.Second + } + hc := cfg.HTTPClient + if hc == nil { + hc = &http.Client{Timeout: 10 * time.Second} + } + allowed := map[string]bool{} + for _, raw := range []string{cfg.WebURL, cfg.RedirectURL} { + if origin := originOf(raw); origin != "" { + allowed[origin] = true + } + } + def := cfg.WebURL + if def == "" { + def = "/" + } + return &OIDC{ + cfg: cfg, + store: store, + svc: svc, + http: hc, + admin: emailSet(cfg.AdminEmails), + op: emailSet(cfg.OperatorEmails), + scopes: strings.Join(cfg.Scopes, " "), + allowed: allowed, + allowedDefault: def, + }, nil +} + +// SetAllowedOrigins registers extra post-login redirect origins. Relative +// targets and the configured web/redirect origins are always allowed. +func (o *OIDC) SetAllowedOrigins(origins []string) { + for _, raw := range origins { + if origin := originOf(raw); origin != "" { + o.allowed[origin] = true + } + } +} + +// DefaultReturn is the fallback post-login target used when the requested one +// is unknown (e.g. reporting an error before state is resolved). +func (o *OIDC) DefaultReturn() string { return o.allowedDefault } + +// ResolveReturn validates a requested post-login target against the allowlist, +// falling back to the configured web URL. +func (o *OIDC) ResolveReturn(raw string) string { + raw = strings.TrimSpace(raw) + if raw == "" { + return o.allowedDefault + } + if !strings.Contains(raw, "://") { + if strings.HasPrefix(raw, "/") { + return raw + } + return o.allowedDefault + } + if o.allowed[originOf(raw)] { + return raw + } + return o.allowedDefault +} + +// ProviderName is a human-friendly provider label for the login button. +func (o *OIDC) ProviderName() string { + if o.cfg.ProviderName != "" { + return o.cfg.ProviderName + } + if u, err := url.Parse(o.cfg.Issuer); err == nil && u.Host != "" { + return u.Host + } + return "SSO" +} + +// Begin creates fresh state + a PKCE verifier, stores them, and returns the +// provider authorization URL to redirect the browser to. +func (o *OIDC) Begin(ctx context.Context, webReturn string) (string, error) { + webReturn = o.ResolveReturn(webReturn) + meta, err := o.discover(ctx) + if err != nil { + return "", err + } + state, err := randomToken(24) + if err != nil { + return "", err + } + verifier, err := randomToken(32) + if err != nil { + return "", err + } + stored, _ := json.Marshal(authState{Verifier: verifier, WebReturn: webReturn}) + if err := o.store.Save(ctx, "oidc:state:"+state, stored, o.cfg.StateTTL); err != nil { + return "", fmt.Errorf("store state: %w", err) + } + + params := url.Values{} + params.Set("response_type", "code") + params.Set("client_id", o.cfg.ClientID) + params.Set("redirect_uri", o.cfg.RedirectURL) + params.Set("scope", o.scopes) + params.Set("state", state) + params.Set("code_challenge", pkceChallenge(verifier)) + params.Set("code_challenge_method", "S256") + return meta.AuthorizationEndpoint + "?" + params.Encode(), nil +} + +// Complete validates the callback, exchanges the code, fetches the user, +// maps roles, and mints a JWT. It returns the one-time exchange code. +func (o *OIDC) Complete(ctx context.Context, state, code string) (string, *SSOSession, error) { + raw, ok, err := o.store.Load(ctx, "oidc:state:"+state) + if err != nil { + return "", nil, fmt.Errorf("%w: load state: %v", ErrSSOFailed, err) + } + if !ok { + return "", nil, fmt.Errorf("%w: unknown or expired state", ErrSSOFailed) + } + _ = o.store.Delete(ctx, "oidc:state:"+state) + + var st authState + if err := json.Unmarshal(raw, &st); err != nil { + return "", nil, fmt.Errorf("%w: corrupt state", ErrSSOFailed) + } + + meta, err := o.discover(ctx) + if err != nil { + return "", nil, err + } + + info, err := o.exchange(ctx, meta, code, st.Verifier) + if err != nil { + return "", nil, err + } + + email := strings.TrimSpace(info.Email) + if email == "" { + email = info.PreferredUsername + } + if email == "" { + email = info.Subject + } + roles, err := o.rolesFor(email) + if err != nil { + return "", nil, err + } + + username := info.Name + if username == "" { + username = email + } + identity := o.svc.NewIdentity(username, roles) + token, expires, err := o.svc.IssueToken(identity) + if err != nil { + return "", nil, fmt.Errorf("%w: issue token: %v", ErrSSOFailed, err) + } + + otc, err := randomToken(24) + if err != nil { + return "", nil, err + } + sess := SSOSession{Token: token, ExpiresAt: expires, Identity: identity, WebReturn: st.WebReturn} + payload, _ := json.Marshal(sess) + if err := o.store.Save(ctx, "oidc:code:"+otc, payload, o.cfg.CodeTTL); err != nil { + return "", nil, fmt.Errorf("%w: store code: %v", ErrSSOFailed, err) + } + return otc, &sess, nil +} + +// Exchange resolves a one-time code (returned to the browser) into the session. +func (o *OIDC) Exchange(ctx context.Context, code string) (*SSOSession, error) { + raw, ok, err := o.store.Load(ctx, "oidc:code:"+code) + if err != nil { + return nil, err + } + if !ok { + return nil, errors.New("invalid or expired code") + } + _ = o.store.Delete(ctx, "oidc:code:"+code) + var sess SSOSession + if err := json.Unmarshal(raw, &sess); err != nil { + return nil, errors.New("invalid code payload") + } + return &sess, nil +} + +func (o *OIDC) exchange(ctx context.Context, meta *providerMeta, code, verifier string) (*userInfo, error) { + form := url.Values{} + form.Set("grant_type", "authorization_code") + form.Set("code", code) + form.Set("redirect_uri", o.cfg.RedirectURL) + form.Set("client_id", o.cfg.ClientID) + form.Set("code_verifier", verifier) + if o.cfg.ClientSecret != "" { + form.Set("client_secret", o.cfg.ClientSecret) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, meta.TokenEndpoint, strings.NewReader(form.Encode())) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.Header.Set("Accept", "application/json") + + resp, err := o.http.Do(req) + if err != nil { + return nil, fmt.Errorf("%w: token request: %v", ErrSSOFailed, err) + } + defer resp.Body.Close() + body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%w: token endpoint status %d", ErrSSOFailed, resp.StatusCode) + } + var tok struct { + AccessToken string `json:"access_token"` + } + if err := json.Unmarshal(body, &tok); err != nil || tok.AccessToken == "" { + return nil, fmt.Errorf("%w: malformed token response", ErrSSOFailed) + } + + uiReq, err := http.NewRequestWithContext(ctx, http.MethodGet, meta.UserinfoEndpoint, nil) + if err != nil { + return nil, err + } + uiReq.Header.Set("Authorization", "Bearer "+tok.AccessToken) + uiReq.Header.Set("Accept", "application/json") + uiResp, err := o.http.Do(uiReq) + if err != nil { + return nil, fmt.Errorf("%w: userinfo request: %v", ErrSSOFailed, err) + } + defer uiResp.Body.Close() + uiBody, _ := io.ReadAll(io.LimitReader(uiResp.Body, 1<<20)) + if uiResp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%w: userinfo status %d", ErrSSOFailed, uiResp.StatusCode) + } + var info userInfo + if err := json.Unmarshal(uiBody, &info); err != nil || info.Subject == "" { + return nil, fmt.Errorf("%w: malformed userinfo", ErrSSOFailed) + } + return &info, nil +} + +func (o *OIDC) rolesFor(email string) ([]Role, error) { + e := strings.ToLower(strings.TrimSpace(email)) + if o.admin[e] { + return []Role{RoleAdmin}, nil + } + if o.op[e] { + return []Role{RoleOperator}, nil + } + if o.cfg.DefaultRole != "" { + return []Role{o.cfg.DefaultRole}, nil + } + return nil, fmt.Errorf("%w: %s is not authorized", ErrSSOFailed, email) +} + +func (o *OIDC) discover(ctx context.Context) (*providerMeta, error) { + o.mu.Lock() + if o.provider != nil { + meta := o.provider + o.mu.Unlock() + return meta, nil + } + o.mu.Unlock() + + wellKnown := strings.TrimRight(o.cfg.Issuer, "/") + "/.well-known/openid-configuration" + req, err := http.NewRequestWithContext(ctx, http.MethodGet, wellKnown, nil) + if err != nil { + return nil, err + } + resp, err := o.http.Do(req) + if err != nil { + return nil, fmt.Errorf("%w: discovery: %v", ErrSSOFailed, err) + } + defer resp.Body.Close() + body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%w: discovery status %d", ErrSSOFailed, resp.StatusCode) + } + var meta providerMeta + if err := json.Unmarshal(body, &meta); err != nil { + return nil, fmt.Errorf("%w: malformed discovery document", ErrSSOFailed) + } + if meta.AuthorizationEndpoint == "" || meta.TokenEndpoint == "" { + return nil, fmt.Errorf("%w: discovery missing endpoints", ErrSSOFailed) + } + if meta.UserinfoEndpoint == "" { + meta.UserinfoEndpoint = strings.TrimRight(meta.Issuer, "/") + "/userinfo" + } + + o.mu.Lock() + o.provider = &meta + o.mu.Unlock() + return &meta, nil +} + +func emailSet(emails []string) map[string]bool { + set := make(map[string]bool, len(emails)) + for _, e := range emails { + if e = strings.ToLower(strings.TrimSpace(e)); e != "" { + set[e] = true + } + } + return set +} + +func pkceChallenge(verifier string) string { + sum := sha256.Sum256([]byte(verifier)) + return base64.RawURLEncoding.EncodeToString(sum[:]) +} + +// originOf returns the normalized scheme://host origin of a URL, or "". +func originOf(raw string) string { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || u.Scheme == "" || u.Host == "" { + return "" + } + return strings.ToLower(u.Scheme + "://" + u.Host) +} diff --git a/internal/auth/oidc_test.go b/internal/auth/oidc_test.go new file mode 100644 index 0000000..80804e4 --- /dev/null +++ b/internal/auth/oidc_test.go @@ -0,0 +1,218 @@ +package auth + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" +) + +// fakeIdP stands up a minimal OpenID Connect provider for tests. +func fakeIdP(t *testing.T, email, name string) *httptest.Server { + t.Helper() + mux := http.NewServeMux() + var base string + + mux.HandleFunc("/.well-known/openid-configuration", func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]string{ + "issuer": base, + "authorization_endpoint": base + "/authorize", + "token_endpoint": base + "/token", + "userinfo_endpoint": base + "/userinfo", + }) + }) + mux.HandleFunc("/token", func(w http.ResponseWriter, r *http.Request) { + _ = r.ParseForm() + if r.Form.Get("code_verifier") == "" { + http.Error(w, "missing PKCE verifier", http.StatusBadRequest) + return + } + if r.Form.Get("client_id") != "client" { + http.Error(w, "bad client", http.StatusBadRequest) + return + } + _ = json.NewEncoder(w).Encode(map[string]string{"access_token": "opaque-token"}) + }) + mux.HandleFunc("/userinfo", func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer opaque-token" { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + _ = json.NewEncoder(w).Encode(map[string]string{"sub": "subject-1", "email": email, "name": name}) + }) + + srv := httptest.NewServer(mux) + base = srv.URL + t.Cleanup(srv.Close) + return srv +} + +func newTestOIDC(t *testing.T, cfg OIDCConfig) *OIDC { + t.Helper() + svc, err := NewService(nil, []byte("test-secret"), time.Hour) + if err != nil { + t.Fatalf("NewService: %v", err) + } + oidc, err := NewOIDC(cfg, svc, NewMemoryStore()) + if err != nil { + t.Fatalf("NewOIDC: %v", err) + } + return oidc +} + +func TestOIDCFlow(t *testing.T) { + idp := fakeIdP(t, "ops@example.com", "Ops User") + oidc := newTestOIDC(t, OIDCConfig{ + Issuer: idp.URL, + ClientID: "client", + RedirectURL: "http://console.local/api/auth/oidc/callback", + WebURL: "http://console.local", + OperatorEmails: []string{"ops@example.com"}, + DefaultRole: "", + }) + ctx := context.Background() + + authURL, err := oidc.Begin(ctx, "") + if err != nil { + t.Fatalf("Begin: %v", err) + } + u, err := url.Parse(authURL) + if err != nil { + t.Fatalf("parse auth URL: %v", err) + } + q := u.Query() + if q.Get("code_challenge_method") != "S256" || q.Get("code_challenge") == "" { + t.Fatalf("expected S256 PKCE challenge, got %q", authURL) + } + state := q.Get("state") + if state == "" { + t.Fatal("missing state") + } + + otc, sess, err := oidc.Complete(ctx, state, "auth-code") + if err != nil { + t.Fatalf("Complete: %v", err) + } + if len(sess.Identity.Roles) != 1 || sess.Identity.Roles[0] != RoleOperator { + t.Fatalf("roles = %v, want [operator]", sess.Identity.Roles) + } + if sess.Identity.Username != "Ops User" { + t.Fatalf("username = %q", sess.Identity.Username) + } + if sess.WebReturn != "http://console.local" { + t.Fatalf("webReturn = %q", sess.WebReturn) + } + + exchanged, err := oidc.Exchange(ctx, otc) + if err != nil { + t.Fatalf("Exchange: %v", err) + } + if exchanged.Token != sess.Token { + t.Fatal("exchanged token differs") + } + if _, err := oidc.Exchange(ctx, otc); err == nil { + t.Fatal("expected one-time code to be consumed") + } +} + +func TestOIDCRejectsUnauthorizedEmail(t *testing.T) { + idp := fakeIdP(t, "stranger@example.com", "Stranger") + oidc := newTestOIDC(t, OIDCConfig{ + Issuer: idp.URL, + ClientID: "client", + RedirectURL: "http://console.local/api/auth/oidc/callback", + AdminEmails: []string{"admin@example.com"}, + OperatorEmails: []string{"ops@example.com"}, + DefaultRole: "", + }) + ctx := context.Background() + + authURL, _ := oidc.Begin(ctx, "") + u, _ := url.Parse(authURL) + if _, _, err := oidc.Complete(ctx, u.Query().Get("state"), "auth-code"); err == nil { + t.Fatal("expected unauthorized email to be rejected") + } +} + +func TestOIDCDefaultRole(t *testing.T) { + idp := fakeIdP(t, "anyone@example.com", "Anyone") + oidc := newTestOIDC(t, OIDCConfig{ + Issuer: idp.URL, + ClientID: "client", + RedirectURL: "http://console.local/api/auth/oidc/callback", + DefaultRole: RoleViewer, + }) + ctx := context.Background() + + authURL, _ := oidc.Begin(ctx, "") + u, _ := url.Parse(authURL) + _, sess, err := oidc.Complete(ctx, u.Query().Get("state"), "auth-code") + if err != nil { + t.Fatalf("Complete: %v", err) + } + if len(sess.Identity.Roles) != 1 || sess.Identity.Roles[0] != RoleViewer { + t.Fatalf("roles = %v, want [viewer]", sess.Identity.Roles) + } +} + +func TestOIDCRejectsUnknownState(t *testing.T) { + idp := fakeIdP(t, "ops@example.com", "Ops") + oidc := newTestOIDC(t, OIDCConfig{ + Issuer: idp.URL, + ClientID: "client", + RedirectURL: "http://console.local/api/auth/oidc/callback", + DefaultRole: RoleViewer, + }) + if _, _, err := oidc.Complete(context.Background(), "bogus-state", "code"); err == nil { + t.Fatal("expected unknown state to be rejected") + } +} + +func TestResolveReturnBlocksOpenRedirect(t *testing.T) { + oidc := newTestOIDC(t, OIDCConfig{ + Issuer: "https://idp.example.com", + ClientID: "client", + RedirectURL: "https://console.example.com/api/auth/oidc/callback", + WebURL: "https://console.example.com", + }) + oidc.SetAllowedOrigins([]string{"https://admin.example.com"}) + + if got := oidc.ResolveReturn(""); got != "https://console.example.com" { + t.Fatalf("default return = %q", got) + } + if got := oidc.ResolveReturn("/queues"); got != "/queues" { + t.Fatalf("relative return = %q", got) + } + if got := oidc.ResolveReturn("https://admin.example.com/dlq"); got != "https://admin.example.com/dlq" { + t.Fatalf("allowed origin = %q", got) + } + if got := oidc.ResolveReturn("https://evil.example.com"); got != "https://console.example.com" { + t.Fatalf("open redirect not blocked: %q", got) + } +} + +func TestSSOTokenVerifiesWithoutLocalUser(t *testing.T) { + svc, err := NewService(nil, []byte("secret"), time.Hour) + if err != nil { + t.Fatalf("NewService: %v", err) + } + id := svc.NewIdentity("SSO User", []Role{RoleOperator}) + token, _, err := svc.IssueToken(id) + if err != nil { + t.Fatalf("IssueToken: %v", err) + } + got, err := svc.VerifyToken(token) + if err != nil { + t.Fatalf("VerifyToken: %v", err) + } + if !got.Can(PermQueuesWrite) { + t.Fatal("operator should hold queues:write") + } + if !strings.EqualFold(got.Username, "SSO User") { + t.Fatalf("username = %q", got.Username) + } +} diff --git a/internal/auth/password.go b/internal/auth/password.go index a22b4a3..bc946de 100644 --- a/internal/auth/password.go +++ b/internal/auth/password.go @@ -6,7 +6,6 @@ import ( "crypto/sha256" "crypto/subtle" "encoding/base64" - "errors" "fmt" "strconv" "strings" @@ -16,16 +15,13 @@ import ( // // pbkdf2$sha256$$$ // -// Only the standard library is used so the server has no crypto dependency to -// manage. Verification is constant-time. +// Verification is constant-time. const ( pbkdf2Iterations = 210_000 pbkdf2KeyLength = 32 pbkdf2SaltLength = 16 ) -var errBadHash = errors.New("malformed password hash") - func hashPassword(password string) (string, error) { salt := make([]byte, pbkdf2SaltLength) if _, err := rand.Read(salt); err != nil { diff --git a/internal/auth/rbac.go b/internal/auth/rbac.go index 7b68998..c66b950 100644 --- a/internal/auth/rbac.go +++ b/internal/auth/rbac.go @@ -19,8 +19,8 @@ const ( PermMetricsRead Permission = "metrics:read" ) -// Role groups permissions. Roles are assigned to users; permissions are never -// assigned directly so the surface stays small and auditable. +// Role groups permissions. Roles are assigned to users, permissions are +// never assigned directly. type Role string const ( @@ -29,8 +29,8 @@ const ( RoleViewer Role = "viewer" ) -// rolePermissions is the single source of truth for what each role can do. -// Every permission known to the API must appear in at least one role. +// rolePermissions maps each role to its permissions. Every permission +// must appear in at least one role. var rolePermissions = map[Role][]Permission{ RoleViewer: { PermSystemRead, diff --git a/internal/auth/redis_state.go b/internal/auth/redis_state.go new file mode 100644 index 0000000..58298b7 --- /dev/null +++ b/internal/auth/redis_state.go @@ -0,0 +1,37 @@ +package auth + +import ( + "context" + "time" + + "github.com/redis/go-redis/v9" +) + +// RedisStore is a StateStore backed by Redis so OIDC flow state survives across +// API instances and restarts. +type RedisStore struct { + rdb *redis.Client +} + +func NewRedisStore(rdb *redis.Client) *RedisStore { + return &RedisStore{rdb: rdb} +} + +func (s *RedisStore) Save(ctx context.Context, key string, value []byte, ttl time.Duration) error { + return s.rdb.Set(ctx, key, value, ttl).Err() +} + +func (s *RedisStore) Load(ctx context.Context, key string) ([]byte, bool, error) { + b, err := s.rdb.Get(ctx, key).Bytes() + if err == redis.Nil { + return nil, false, nil + } + if err != nil { + return nil, false, err + } + return b, true, nil +} + +func (s *RedisStore) Delete(ctx context.Context, key string) error { + return s.rdb.Del(ctx, key).Err() +} diff --git a/internal/auth/service.go b/internal/auth/service.go index 3724232..fdbab3c 100644 --- a/internal/auth/service.go +++ b/internal/auth/service.go @@ -6,8 +6,8 @@ import ( "time" ) -// ErrInvalidCredentials is returned when a username is unknown or the password -// does not match. It is deliberately identical in both cases. +// ErrInvalidCredentials is returned for an unknown username or a bad password. +// The same error is used for both. var ErrInvalidCredentials = errors.New("invalid credentials") // Identity is the authenticated principal attached to a request. Permissions @@ -77,6 +77,9 @@ func NewService(users []User, secret []byte, ttl time.Duration) (*Service, error }, nil } +// HasLocalUsers reports whether any username/password accounts are configured. +func (s *Service) HasLocalUsers() bool { return s != nil && len(s.users) > 0 } + // Authenticate verifies credentials and returns the resulting identity. func (s *Service) Authenticate(username, password string) (Identity, error) { u, ok := s.users[username] @@ -91,18 +94,33 @@ func (s *Service) IssueToken(id Identity) (string, time.Time, error) { return s.tokens.issue(id.Username, id.Roles) } -// VerifyToken validates a bearer token and reconstructs the identity. The user -// must still exist, so deleting an account revokes its outstanding tokens. +// VerifyToken validates a bearer token and rebuilds the identity. Local users +// are re-read so a role change takes effect at once. SSO subjects use the +// roles embedded in the token. func (s *Service) VerifyToken(token string) (Identity, error) { c, err := s.tokens.verify(token) if err != nil { return Identity{}, err } - u, ok := s.users[c.Subject] - if !ok { + if u, ok := s.users[c.Subject]; ok { + return s.identityFor(u.Username, u.Roles), nil + } + roles := make([]Role, 0, len(c.Roles)) + for _, r := range c.Roles { + if ValidRole(Role(r)) { + roles = append(roles, Role(r)) + } + } + if len(roles) == 0 { return Identity{}, ErrInvalidToken } - return s.identityFor(u.Username, u.Roles), nil + return s.identityFor(c.Subject, roles), nil +} + +// NewIdentity builds an identity for an externally-authenticated principal +// such as an SSO login. +func (s *Service) NewIdentity(username string, roles []Role) Identity { + return s.identityFor(username, roles) } func (s *Service) identityFor(username string, roles []Role) Identity { diff --git a/internal/auth/state.go b/internal/auth/state.go new file mode 100644 index 0000000..c582dbd --- /dev/null +++ b/internal/auth/state.go @@ -0,0 +1,59 @@ +package auth + +import ( + "context" + "sync" + "time" +) + +// StateStore persists short-lived OIDC flow state (CSRF state, PKCE verifier) +// and one-time login codes across the redirect round trip. +type StateStore interface { + Save(ctx context.Context, key string, value []byte, ttl time.Duration) error + Load(ctx context.Context, key string) ([]byte, bool, error) + Delete(ctx context.Context, key string) error +} + +type memEntry struct { + value []byte + expires time.Time +} + +// MemoryStore is an in-process StateStore for tests and single-instance use. +// It is safe for concurrent use and lazily expires entries. +type MemoryStore struct { + mu sync.Mutex + items map[string]memEntry +} + +func NewMemoryStore() *MemoryStore { + return &MemoryStore{items: make(map[string]memEntry)} +} + +func (m *MemoryStore) Save(_ context.Context, key string, value []byte, ttl time.Duration) error { + m.mu.Lock() + defer m.mu.Unlock() + m.items[key] = memEntry{value: append([]byte(nil), value...), expires: time.Now().Add(ttl)} + return nil +} + +func (m *MemoryStore) Load(_ context.Context, key string) ([]byte, bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + e, ok := m.items[key] + if !ok { + return nil, false, nil + } + if time.Now().After(e.expires) { + delete(m.items, key) + return nil, false, nil + } + return append([]byte(nil), e.value...), true, nil +} + +func (m *MemoryStore) Delete(_ context.Context, key string) error { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.items, key) + return nil +} diff --git a/internal/auth/token.go b/internal/auth/token.go index 25eb590..085d314 100644 --- a/internal/auth/token.go +++ b/internal/auth/token.go @@ -12,13 +12,11 @@ import ( ) // ErrInvalidToken is returned for any token that fails signature or shape -// validation. Callers should not distinguish the reasons to avoid leaking -// information about why a token was rejected. +// validation. Callers should not distinguish the reasons. var ErrInvalidToken = errors.New("invalid token") -// claims is the JWT payload. Roles are embedded so the API can authorize a -// request without a Redis round trip; this means a role change takes effect on -// the next login (until then, the old roles remain valid until expiry). +// claims is the JWT payload. Roles are embedded so the API can authorize +// without a Redis round trip. A role change only takes effect on next login. type claims struct { Subject string `json:"sub"` Roles []string `json:"roles"` diff --git a/internal/durable/postgres.go b/internal/durable/postgres.go index fdc44f5..ac55bf8 100644 --- a/internal/durable/postgres.go +++ b/internal/durable/postgres.go @@ -9,17 +9,12 @@ import ( ) // DefaultDSN points at a local Postgres running the forgequeue database. -// Override it with FORGEQUEUE_PG_DSN when Postgres lives elsewhere -// (docker-compose, CI, a real deployment). +// Override with FORGEQUEUE_PG_DSN. const DefaultDSN = "postgres://localhost:5432/forgequeue" -// The schema is one table. Every enqueued task gets a row; the row follows -// the task through its lifecycle until the task is succeeded or dead. -// -// payload is BYTEA (not JSONB) on purpose: it is the byte-for-byte JSON body -// Redis stores, so the reconciler can rebuild Redis without re-encoding the -// task. JSONB would be nicer for querying, but querying bodies is not what -// this table is for. +// One table, one row per enqueued task, following it until succeeded or dead. +// payload is BYTEA not JSONB: it is the exact body Redis stores, so the +// reconciler can rebuild Redis without re-encoding. const schema = ` CREATE TABLE IF NOT EXISTS queue_tasks ( id TEXT PRIMARY KEY, @@ -34,11 +29,13 @@ CREATE TABLE IF NOT EXISTS queue_tasks ( updated_at TIMESTAMPTZ NOT NULL DEFAULT now() ); --- The reconciler only ever scans the rows that need Redis materialized, so --- the scan should not touch the finished history. -CREATE INDEX IF NOT EXISTS queue_tasks_pending_idx - ON queue_tasks (redis_pending) - WHERE redis_pending = TRUE; +-- The reconciler scans non-terminal tasks in (created_at, id) order, so the +-- index only covers those rows. Terminal history is skipped entirely. +CREATE INDEX IF NOT EXISTS queue_tasks_active_idx + ON queue_tasks (created_at, id) + WHERE state IN ('enqueued', 'processing', 'retrying'); + +DROP INDEX IF EXISTS queue_tasks_pending_idx; ` // Postgres is a Store backed by PostgreSQL. @@ -46,8 +43,15 @@ type Postgres struct { pool *pgxpool.Pool } -// NewPostgres connects to Postgres, fails fast if it is unreachable, and -// makes sure the schema exists. +// NewPostgres connects to Postgres and makes sure the schema exists. Returns +// an error if Postgres is unreachable. +// advisoryLock serializes one-time schema creation across pods. Multiple +// replicas (gateway + workers) all call NewPostgres on boot, and executing the +// same CREATE/INDEX DDL concurrently raises transient catalog races (e.g. +// duplicate key on pg_type_typname_nsp_index). The lock is transaction-scoped, +// so it releases when the commit ends. +const advisoryLock int64 = 884422 + func NewPostgres(ctx context.Context, dsn string) (*Postgres, error) { pool, err := pgxpool.New(ctx, dsn) if err != nil { @@ -58,20 +62,46 @@ func NewPostgres(ctx context.Context, dsn string) (*Postgres, error) { return nil, fmt.Errorf("durable: ping postgres: %w", err) } if _, err := pool.Exec(ctx, schema); err != nil { - pool.Close() - return nil, fmt.Errorf("durable: ensure schema: %w", err) + // A concurrent peer may be mid-migration (gateway and workers all + // initialize on boot). Retry a few times before giving up. + if err := ensureSchema(ctx, pool); err != nil { + pool.Close() + return nil, fmt.Errorf("durable: ensure schema: %w", err) + } } return &Postgres{pool: pool}, nil } +func ensureSchema(ctx context.Context, pool *pgxpool.Pool) error { + var lastErr error + for attempt := 0; attempt < 5; attempt++ { + tx, err := pool.Begin(ctx) + if err != nil { + return err + } + if _, err := tx.Exec(ctx, "SELECT pg_advisory_xact_lock($1)", advisoryLock); err != nil { + tx.Rollback(ctx) + return err + } + if _, err := tx.Exec(ctx, schema); err != nil { + tx.Rollback(ctx) + lastErr = err + time.Sleep(200 * time.Millisecond) + continue + } + return tx.Commit(ctx) + } + return lastErr +} + func (p *Postgres) Close() error { p.pool.Close() return nil } // Enqueue upserts the task and always leaves redis_pending = true. The -// gateway calls this before touching Redis: if the process dies in between, -// the reconciler re-materializes the task and nothing is lost. +// gateway calls this before touching Redis, so a crash in between is healed +// by the reconciler. func (p *Postgres) Enqueue(ctx context.Context, t Task) error { _, err := p.pool.Exec(ctx, ` INSERT INTO queue_tasks @@ -105,14 +135,13 @@ func (p *Postgres) Dead(ctx context.Context, id string, attempts int32) error { } // Retrying records the backoff schedule the worker just wrote into Redis. -// attempts and schedule_at come from the requeue script's own math, so the -// journal mirrors what Redis actually decided - never the other way around. +// attempts and schedule_at come from the requeue script, so the journal +// mirrors Redis rather than the other way around. func (p *Postgres) Retrying(ctx context.Context, id string, attempts int32, scheduleAt time.Time) error { return p.setState(ctx, id, string(StateRetrying), attempts, &scheduleAt) } -// setState is the shared UPDATE. attempts = -1 leaves the counter untouched -// (the worker did not change it). +// setState is the shared UPDATE. attempts = -1 leaves the counter untouched. func (p *Postgres) setState(ctx context.Context, id string, state string, attempts int32, scheduleAt *time.Time) error { _, err := p.pool.Exec(ctx, ` UPDATE queue_tasks @@ -126,18 +155,23 @@ func (p *Postgres) setState(ctx context.Context, id string, state string, attemp return err } -// Pending returns the tasks the journal knows about but Redis has not proven -// it holds. Only active states can need materialization - a succeeded or -// dead task never goes back into a queue. -func (p *Postgres) Pending(ctx context.Context) ([]Task, error) { +// Active returns a page of non-terminal tasks ordered by (created_at, id), +// strictly after the cursor. Every non-terminal task is re-verified against +// Redis on every sweep, so a task that was materialized and then lost when +// Redis restarted is rebuilt. Rows are keyset-paginated so a large active set +// is walked a page at a time instead of starved by an OFFSET/LIMIT window. +func (p *Postgres) Active(ctx context.Context, after Cursor, limit int) ([]Task, error) { + if limit <= 0 { + limit = 1000 + } rows, err := p.pool.Query(ctx, ` SELECT id, task_type, payload, max_retries, attempts, state, schedule_at, created_at FROM queue_tasks - WHERE redis_pending = TRUE - AND state IN ('`+string(StateEnqueued)+`', '`+string(StateProcessing)+`', '`+string(StateRetrying)+`') - ORDER BY created_at - LIMIT 1000 - `) + WHERE state IN ('`+string(StateEnqueued)+`', '`+string(StateProcessing)+`', '`+string(StateRetrying)+`') + AND (created_at, id) > ($1, $2) + ORDER BY created_at, id + LIMIT $3 + `, after.CreatedAt, after.ID, limit) if err != nil { return nil, err } @@ -161,7 +195,7 @@ func (p *Postgres) Pending(ctx context.Context) ([]Task, error) { return tasks, rows.Err() } -// Materialized clears redis_pending now that Redis provably holds the task. +// Materialized clears redis_pending once Redis holds the task. func (p *Postgres) Materialized(ctx context.Context, id string) error { _, err := p.pool.Exec(ctx, ` UPDATE queue_tasks SET redis_pending = FALSE, updated_at = now() WHERE id = $1 @@ -169,8 +203,8 @@ func (p *Postgres) Materialized(ctx context.Context, id string) error { return err } -// nullableTime converts a Go time to the *time.Time Postgres wants for a -// NULL-able column. The zero time means "immediate" and maps to NULL. +// nullableTime maps a Go time to a nullable column. Zero means immediate, so +// it becomes NULL. func nullableTime(t time.Time) *time.Time { if t.IsZero() { return nil diff --git a/internal/durable/postgres_test.go b/internal/durable/postgres_test.go index b3ac404..75a62a5 100644 --- a/internal/durable/postgres_test.go +++ b/internal/durable/postgres_test.go @@ -55,26 +55,28 @@ func TestPostgresLifecycle(t *testing.T) { id := "life-1" body := []byte(`{"id":"life-1","task_type":"noop"}`) - // Enqueue leaves redis_pending = true so the reconciler can prove it. + // Enqueue leaves the task non-terminal, so the reconciler sees it. if err := p.Enqueue(ctx, Task{ ID: id, TaskType: "noop", Payload: body, State: StateEnqueued, }); err != nil { t.Fatalf("enqueue: %v", err) } - pending, err := p.Pending(ctx) + active, err := p.Active(ctx, Cursor{}, 100) if err != nil { - t.Fatalf("pending: %v", err) + t.Fatalf("active: %v", err) } - if len(pending) != 1 || pending[0].ID != id || pending[0].ScheduleAt != (time.Time{}) { - t.Fatalf("pending = %+v, want one immediate task %s", pending, id) + if len(active) != 1 || active[0].ID != id || active[0].ScheduleAt != (time.Time{}) { + t.Fatalf("active = %+v, want one immediate task %s", active, id) } - // Materialized proves Redis has it -> no longer pending. + // Materialized clears the redis_pending hint, but the task is still + // non-terminal and must keep being re-verified: that is what lets the + // reconciler rebuild Redis after a flush. if err := p.Materialized(ctx, id); err != nil { t.Fatalf("materialized: %v", err) } - if pending, _ := p.Pending(ctx); len(pending) != 0 { - t.Fatalf("pending has %d task(s) after materialized, want 0", len(pending)) + if active, _ := p.Active(ctx, Cursor{}, 100); len(active) != 1 { + t.Fatalf("active has %d task(s) after materialized, want 1", len(active)) } // The journal follows the worker's view: processing -> retrying -> done. @@ -99,9 +101,9 @@ func TestPostgresLifecycle(t *testing.T) { t.Fatalf("state = %s, want succeeded", st) } - // A terminal state must never surface as pending. - if pending, _ := p.Pending(ctx); len(pending) != 0 { - t.Fatalf("pending has %d task(s) after terminal state, want 0", len(pending)) + // A terminal state must never surface as active. + if active, _ := p.Active(ctx, Cursor{}, 100); len(active) != 0 { + t.Fatalf("active has %d task(s) after terminal state, want 0", len(active)) } } @@ -130,11 +132,45 @@ func TestPostgresEnqueueIsUpsert(t *testing.T) { t.Fatalf("query: %v", err) } - pending, _ := p.Pending(ctx) - if len(pending) != 1 { - t.Fatalf("pending has %d task(s), want 1", len(pending)) + active, _ := p.Active(ctx, Cursor{}, 100) + if len(active) != 1 { + t.Fatalf("active has %d task(s), want 1", len(active)) } if state != string(StateEnqueued) { t.Fatalf("state = %s, want enqueued", state) } } + +// Active must walk rows in keyset order so a sweep can page through a large +// active set without re-reading or skipping rows. +func TestPostgresActivePaginates(t *testing.T) { + p := newTestDB(t) + truncateTasks(t, p) + ctx := context.Background() + + for _, id := range []string{"page-1", "page-2", "page-3"} { + if err := p.Enqueue(ctx, Task{ + ID: id, TaskType: "noop", Payload: []byte(`{}`), State: StateEnqueued, + }); err != nil { + t.Fatalf("enqueue %s: %v", id, err) + } + } + + var seen []string + cursor := Cursor{} + for { + page, err := p.Active(ctx, cursor, 1) + if err != nil { + t.Fatalf("active: %v", err) + } + if len(page) == 0 { + break + } + seen = append(seen, page[0].ID) + cursor = Cursor{CreatedAt: page[0].CreatedAt, ID: page[0].ID} + } + + if len(seen) != 3 { + t.Fatalf("paged over %v, want all 3 tasks", seen) + } +} diff --git a/internal/durable/reconciler.go b/internal/durable/reconciler.go index a27ebdd..d11ae3a 100644 --- a/internal/durable/reconciler.go +++ b/internal/durable/reconciler.go @@ -9,40 +9,28 @@ import ( "github.com/redis/go-redis/v9" ) -// The reconciler is the answer to the storage split's core problem: you -// cannot atomically write Postgres AND Redis. Every write is ordered, so -// there is always a window in which one store has the task and the other -// does not. The reconciler works through redis_pending rows and makes sure -// the task actually exists in Redis - the hot store converges to what the -// durable store says it should be. -// -// This is the "journal first" half of the design: Postgres decides what -// should exist, Redis executes it, and a crash anywhere in between gets -// repaired on the next sweep. +// Postgres and Redis cannot be written atomically, so there is always a +// window where one has the task and the other doesn't. The reconciler sweeps +// every non-terminal task and makes sure each still exists in Redis. Because +// it re-checks tasks that were already materialized, it also heals a Redis +// restart or flush that dropped them, not just the enqueue crash window. -// materializeScript moves one task into Redis from its journal record. -// -// It is one atomic script so "check presence, then write" has no race: -// -// - LPOS/ZSCORE against the three hot structures. If the task is anywhere -// in Redis already, there is nothing to do and the script returns 0 -// ("already present"). This is what stops a crashed-but-retried worker -// path from ever doubling a task. -// - otherwise the body is re-SET (Redis may have lost everything), and the -// ID is placed on the ready list or the scheduled set depending on the -// journal state. -// -// KEYS[1] --> queue:tasks:immediate -// KEYS[2] --> queue:tasks:inflight -// KEYS[3] --> queue:tasks:scheduled -// KEYS[4] --> task: (the body key) -// ARGV[1] --> task ID -// ARGV[2] --> journal state ("enqueued" | "processing" | "retrying") -// ARGV[3] --> schedule score (unix seconds) when the task is time-bound, -// -// empty otherwise +// reconcilePageSize bounds how many journal rows one sweep verifies, so a +// large active set is walked in pages rather than one unbounded query. +const reconcilePageSize = 1000 + +// materializeScript moves one task from its journal record into Redis. If the +// task is already in one of the hot structures it returns 0, otherwise it +// re-SETs the body and adds the ID to ready or scheduled depending on state. // -// ARGV[4] -> the payload (exact JSON body Redis should hold) +// KEYS[1] queue:tasks:immediate +// KEYS[2] queue:tasks:inflight +// KEYS[3] queue:tasks:scheduled +// KEYS[4] task: +// ARGV[1] task ID +// ARGV[2] journal state +// ARGV[3] schedule score, empty when not time-bound +// ARGV[4] payload JSON const materializeScript = ` local inImmediate = redis.call("LPOS", KEYS[1], ARGV[1]) local inInflight = redis.call("ZSCORE", KEYS[2], ARGV[1]) @@ -55,71 +43,74 @@ const materializeScript = ` redis.call("SET", KEYS[4], ARGV[4]) if ARGV[3] == "" then - -- an immediate enqueue belongs on the ready list. + -- immediate enqueue -> ready list. redis.call("LPUSH", KEYS[1], ARGV[1]) else - -- delayed, in backoff, or a crashed processing task: it waits in - -- the scheduled set until its score is due. + -- delayed, in backoff, or crashed processing -> scheduled set. redis.call("ZADD", KEYS[3], tonumber(ARGV[3]), ARGV[1]) end return 1 ` -// ReconcileOnce does a single sweep: read every pending task from the -// journal, materialize the missing ones into Redis, and clear redis_pending. -// It returns how many tasks it actually repaired. -func ReconcileOnce(ctx context.Context, rdb *redis.Client, store Store) (int, error) { - pending, err := store.Pending(ctx) +// ReconcileOnce verifies one page of non-terminal tasks starting just after +// cursor. It returns how many tasks it had to re-materialize, the cursor to +// resume from on the next call, and whether this page was the last one (fewer +// than a full page of rows came back). +func ReconcileOnce(ctx context.Context, rdb *redis.Client, store Store, cursor Cursor) (repaired int, next Cursor, done bool, err error) { + page, err := store.Active(ctx, cursor, reconcilePageSize) if err != nil { - return 0, err + return 0, cursor, true, err } + next = cursor - repaired := 0 - for _, t := range pending { + for _, t := range page { ok, err := materialize(ctx, rdb, t) if err != nil { log.Printf("reconcile: failed to materialize task %s: %v", t.ID, err) - continue - } - if ok { + } else if ok { repaired++ } if err := store.Materialized(ctx, t.ID); err != nil { log.Printf("reconcile: failed to clear pending for task %s: %v", t.ID, err) } + next = Cursor{CreatedAt: t.CreatedAt, ID: t.ID} } - return repaired, nil + + return repaired, next, len(page) < reconcilePageSize, nil } -// ReconcileLoop runs reconciliations forever until ctx is cancelled. One -// sweep per second is plenty: pending is near-empty in steady state and only -// fills up when something crashed. +// ReconcileLoop reconciles until ctx is cancelled. One page per second is +// plenty: the loop walks the whole active set over successive ticks and wraps +// around once it reaches the end. func ReconcileLoop(ctx context.Context, rdb *redis.Client, store Store) { ticker := time.NewTicker(1 * time.Second) defer ticker.Stop() + var cursor Cursor for { select { case <-ticker.C: - if _, err := ReconcileOnce(ctx, rdb, store); err != nil { + _, next, done, err := ReconcileOnce(ctx, rdb, store, cursor) + if err != nil { log.Printf("reconcile: %v", err) } + if done { + cursor = Cursor{} + } else { + cursor = next + } case <-ctx.Done(): return } } } -// materialize writes one pending task into Redis, or confirms it is already -// there. It returns true when it actually (re)materialized the task. +// materialize writes one pending task into Redis. Returns true if it actually +// wrote it, false if it was already there. func materialize(ctx context.Context, rdb *redis.Client, t Task) (bool, error) { - // Decide where a task whose hot state is gone should land. - // - // - enqueued, no delay: ready list. - // - enqueued, delayed, or retrying: scheduled set, at its due time. - // - processing: the worker claiming it died/was wiped; re-run it like - // recovery would -> scheduled at "now". + // Where a task lands when its hot state is gone. processing means the + // worker died, so re-run it now via scheduled. score := "" switch t.State { case StateProcessing: diff --git a/internal/durable/reconciler_test.go b/internal/durable/reconciler_test.go index bacba2f..0d2c3dc 100644 --- a/internal/durable/reconciler_test.go +++ b/internal/durable/reconciler_test.go @@ -40,7 +40,7 @@ func TestReconcileMaterializesImmediate(t *testing.T) { // Redis is empty (simulates a gateway that journaled a task and then // died before writing to the hot store), so the reconciler must put the // task on the ready list with its body in place. - repaired, err := ReconcileOnce(ctx, rdb, p) + repaired, _, _, err := ReconcileOnce(ctx, rdb, p, Cursor{}) if err != nil { t.Fatalf("reconcile: %v", err) } @@ -59,8 +59,17 @@ func TestReconcileMaterializesImmediate(t *testing.T) { t.Fatalf("body = %q, want %q (err=%v)", got, body, err) } - if pending, _ := p.Pending(ctx); len(pending) != 0 { - t.Fatalf("pending has %d task(s) after reconcile, want 0", len(pending)) + // A second sweep must be a no-op: the task is still non-terminal, but it + // is already in Redis so nothing is repaired and no duplicate appears. + repaired, _, _, err = ReconcileOnce(ctx, rdb, p, Cursor{}) + if err != nil { + t.Fatalf("reconcile again: %v", err) + } + if repaired != 0 { + t.Fatalf("second sweep repaired = %d, want 0", repaired) + } + if n, _ := rdb.LLen(ctx, "queue:tasks:immediate").Result(); n != 1 { + t.Fatalf("immediate queue has %d item(s) after second sweep, want 1", n) } } @@ -79,7 +88,7 @@ func TestReconcileMaterializesDelayed(t *testing.T) { t.Fatalf("enqueue: %v", err) } - if _, err := ReconcileOnce(ctx, rdb, p); err != nil { + if _, _, _, err := ReconcileOnce(ctx, rdb, p, Cursor{}); err != nil { t.Fatalf("reconcile: %v", err) } @@ -117,7 +126,7 @@ func TestReconcileSkipsTaskAlreadyInRedis(t *testing.T) { t.Fatalf("lpush: %v", err) } - repaired, err := ReconcileOnce(ctx, rdb, p) + repaired, _, _, err := ReconcileOnce(ctx, rdb, p, Cursor{}) if err != nil { t.Fatalf("reconcile: %v", err) } @@ -129,9 +138,6 @@ func TestReconcileSkipsTaskAlreadyInRedis(t *testing.T) { if n, _ := rdb.LLen(ctx, "queue:tasks:immediate").Result(); n != 1 { t.Fatalf("immediate queue has %d item(s), want 1 (no dupes)", n) } - if pending, _ := p.Pending(ctx); len(pending) != 0 { - t.Fatalf("pending has %d task(s), want 0 after reconcile", len(pending)) - } } func TestReconcileRebuildsAfterRedisWipe(t *testing.T) { @@ -143,17 +149,19 @@ func TestReconcileRebuildsAfterRedisWipe(t *testing.T) { id := "rec-wipe" body := []byte(`{"id":"rec-wipe","task_type":"noop"}`) - // A worker claimed this task and journaled it as processing, then the - // Redis instance lost everything. The reconciler must bring it back and - // schedule it for a re-run (past-now score), like recovery would. + // A worker claimed and materialized this task (so redis_pending is + // already FALSE), then Redis lost everything. Because the reconciler + // re-verifies every non-terminal task, not just redis_pending rows, it + // must bring the task back and schedule it for a re-run (past-now score), + // like recovery would. if _, err := p.pool.Exec(ctx, ` INSERT INTO queue_tasks (id, task_type, payload, max_retries, attempts, state, redis_pending) - VALUES ($1, 'noop', $2, 3, 1, 'processing', TRUE) + VALUES ($1, 'noop', $2, 3, 1, 'processing', FALSE) `, id, body); err != nil { t.Fatalf("seed processing row: %v", err) } - if _, err := ReconcileOnce(ctx, rdb, p); err != nil { + if _, _, _, err := ReconcileOnce(ctx, rdb, p, Cursor{}); err != nil { t.Fatalf("reconcile: %v", err) } diff --git a/internal/durable/store.go b/internal/durable/store.go index 24cebcc..b24cfa2 100644 --- a/internal/durable/store.go +++ b/internal/durable/store.go @@ -5,12 +5,9 @@ import ( "time" ) -// State is the lifecycle state of a task as the journal records it. -// -// The journal is the durable side of the queue: plain rows in Postgres that -// survive crashes, Redis restarts and even a Redis flush. The hot side of -// the queue (Redis lists and sorted sets) is the part that moves fast; this -// table is the part that remembers. +// State is the lifecycle state of a task as the journal records it. The +// journal is the durable side of the queue, plain Postgres rows that survive +// a crash or a Redis flush. type State string const ( @@ -28,12 +25,9 @@ const ( StateDead State = "dead" ) -// Task is the durable record for one task, as far as the journal cares. -// -// Payload is the exact JSON body Redis stores at task:. Keeping the body -// in Postgres is what lets the reconciler rebuild Redis state from scratch -// after a crash or a Redis flush: it re-SETs the body and re-inserts the ID -// into whichever queue the state says it belongs in. +// Task is the durable record for one task. Payload is the exact JSON body +// Redis stores at task:, which lets the reconciler rebuild Redis from +// scratch after a crash or flush. type Task struct { ID string TaskType string @@ -41,20 +35,25 @@ type Task struct { MaxRetries int32 Attempts int32 State State - // ScheduleAt is when an enqueued/retrying task is due. The zero value - // signals "no delay" - the task belongs in the immediate queue. + // ScheduleAt is when an enqueued/retrying task is due. Zero means no + // delay, so it goes to the immediate queue. ScheduleAt time.Time CreatedAt time.Time } -// Store is the durable half of the queue. The contract with the caller is -// best-effort: queue correctness lives in Redis, this journal exists so Redis -// state can be repaired. Every method is safe to treat as optional - a Nop -// Store absorbs the call and does nothing. +// Cursor is a keyset pagination position over (CreatedAt, ID). The zero value +// means "start from the beginning". +type Cursor struct { + CreatedAt time.Time + ID string +} + +// Store is the durable half of the queue. It is best-effort: queue +// correctness lives in Redis and the journal only exists to repair it. A Nop +// Store absorbs calls and does nothing. type Store interface { - // Enqueue records a newly accepted task. It leaves redis_pending = true, - // meaning "the journal knows about this task but can't yet prove Redis - // does" - the reconciler closes that gap. + // Enqueue records a newly accepted task and leaves redis_pending = true + // until the reconciler confirms Redis holds it. Enqueue(ctx context.Context, t Task) error // Processing marks a task as claimed by a worker. Processing(ctx context.Context, id string) error @@ -63,19 +62,21 @@ type Store interface { // Dead marks a task as dead-lettered. Terminal. Dead(ctx context.Context, id string, attempts int32) error // Retrying records the new attempt count and backoff schedule for a task - // the worker already put back into Redis, so redis_pending is cleared. + // already put back into Redis. Retrying(ctx context.Context, id string, attempts int32, scheduleAt time.Time) error - // Pending returns the tasks redis_pending = true: know by the journal, - // unproven in Redis. The reconciler works through this list. - Pending(ctx context.Context) ([]Task, error) - // Materialized clears redis_pending after the task was (re)written into - // Redis. + // Active returns a page of non-terminal tasks (enqueued, processing, + // retrying) ordered by (created_at, id) after the cursor. The reconciler + // re-verifies each one against Redis, so a Redis restart or flush that + // drops an already-materialized task can still be healed. Terminal tasks + // are never returned. + Active(ctx context.Context, after Cursor, limit int) ([]Task, error) + // Materialized clears redis_pending after the task is in Redis. Materialized(ctx context.Context, id string) error Close() error } -// Nop is a Store that does nothing. It is the default, so a queue without -// Postgres configured behaves exactly as it did before the storage split. +// Nop is a Store that does nothing. It is the default when no Postgres is +// configured. type Nop struct{} func (Nop) Enqueue(context.Context, Task) error { return nil } @@ -83,6 +84,6 @@ func (Nop) Processing(context.Context, string) error { return ni func (Nop) Succeeded(context.Context, string) error { return nil } func (Nop) Dead(context.Context, string, int32) error { return nil } func (Nop) Retrying(context.Context, string, int32, time.Time) error { return nil } -func (Nop) Pending(context.Context) ([]Task, error) { return nil, nil } +func (Nop) Active(context.Context, Cursor, int) ([]Task, error) { return nil, nil } func (Nop) Materialized(context.Context, string) error { return nil } func (Nop) Close() error { return nil } diff --git a/internal/gateway/auth.go b/internal/gateway/auth.go new file mode 100644 index 0000000..f7da731 --- /dev/null +++ b/internal/gateway/auth.go @@ -0,0 +1,70 @@ +package gateway + +import ( + "context" + "strings" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + + "forgequeue/internal/auth" + queuev1 "forgequeue/proto/v1" +) + +// methodPermissions maps each RPC to the permission it requires. A method +// that is not listed is denied, so adding an RPC without granting it here +// fails closed rather than silently exposing it. +var methodPermissions = map[string]auth.Permission{ + queuev1.QueueService_EnqueueTask_FullMethodName: auth.PermQueuesWrite, +} + +// newAuthInterceptor authenticates every unary call with the same bearer +// tokens the HTTP API issues, then authorizes the method's required +// permission. Identity is attached to the handler context for downstream use. +func newAuthInterceptor(svc *auth.Service) grpc.UnaryServerInterceptor { + return func( + ctx context.Context, + req any, + info *grpc.UnaryServerInfo, + handler grpc.UnaryHandler, + ) (any, error) { + token, err := bearerToken(ctx) + if err != nil { + return nil, err + } + + id, err := svc.VerifyToken(token) + if err != nil { + return nil, status.Error(codes.Unauthenticated, "invalid or missing credentials") + } + + required, ok := methodPermissions[info.FullMethod] + if !ok { + return nil, status.Errorf(codes.PermissionDenied, "no policy for method %s", info.FullMethod) + } + if !id.Can(required) { + return nil, status.Errorf(codes.PermissionDenied, "missing permission %s", required) + } + + return handler(auth.WithIdentity(ctx, id), req) + } +} + +// bearerToken extracts the token from the gRPC "authorization" metadata entry. +func bearerToken(ctx context.Context) (string, error) { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return "", status.Error(codes.Unauthenticated, "missing request metadata") + } + vals := md.Get("authorization") + if len(vals) == 0 { + return "", status.Error(codes.Unauthenticated, "missing authorization metadata") + } + fields := strings.Fields(vals[0]) + if len(fields) != 2 || !strings.EqualFold(fields[0], "bearer") { + return "", status.Error(codes.Unauthenticated, "malformed authorization metadata") + } + return fields[1], nil +} diff --git a/internal/gateway/auth_test.go b/internal/gateway/auth_test.go new file mode 100644 index 0000000..c9273ff --- /dev/null +++ b/internal/gateway/auth_test.go @@ -0,0 +1,108 @@ +package gateway + +import ( + "context" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + + "forgequeue/internal/auth" + queuev1 "forgequeue/proto/v1" +) + +// testInterceptor builds an auth Service with a fixed secret and returns the +// interceptor plus a token for the given role. +func testInterceptor(t *testing.T, role auth.Role) (grpc.UnaryServerInterceptor, string) { + t.Helper() + svc, err := auth.NewService(nil, []byte("test-secret"), 0) + if err != nil { + t.Fatalf("new service: %v", err) + } + token, _, err := svc.IssueToken(auth.Identity{Username: "tester", Roles: []auth.Role{role}}) + if err != nil { + t.Fatalf("issue token: %v", err) + } + return newAuthInterceptor(svc), token +} + +func ctxWithToken(token string) context.Context { + md := metadata.Pairs("authorization", "Bearer "+token) + return metadata.NewIncomingContext(context.Background(), md) +} + +// run drives the interceptor with a handler that records whether it was called. +func run(t *testing.T, ic grpc.UnaryServerInterceptor, ctx context.Context) (bool, error) { + t.Helper() + called := false + handler := func(ctx context.Context, req any) (any, error) { + called = true + if _, ok := auth.FromContext(ctx); !ok { + t.Fatal("handler ran without an identity in context") + } + return "ok", nil + } + info := &grpc.UnaryServerInfo{FullMethod: queuev1.QueueService_EnqueueTask_FullMethodName} + _, err := ic(ctx, nil, info, handler) + return called, err +} + +func TestAuthInterceptorRejectsMissingToken(t *testing.T) { + ic, _ := testInterceptor(t, auth.RoleAdmin) + called, err := run(t, ic, context.Background()) + if called { + t.Fatal("handler ran without a token") + } + if status.Code(err) != codes.Unauthenticated { + t.Fatalf("code = %v, want Unauthenticated", status.Code(err)) + } +} + +func TestAuthInterceptorRejectsBadToken(t *testing.T) { + ic, _ := testInterceptor(t, auth.RoleAdmin) + called, err := run(t, ic, ctxWithToken("not-a-jwt")) + if called { + t.Fatal("handler ran with an invalid token") + } + if status.Code(err) != codes.Unauthenticated { + t.Fatalf("code = %v, want Unauthenticated", status.Code(err)) + } +} + +func TestAuthInterceptorRejectsInsufficientRole(t *testing.T) { + // A viewer reaches the auth layer but lacks queues:write. + ic, token := testInterceptor(t, auth.RoleViewer) + called, err := run(t, ic, ctxWithToken(token)) + if called { + t.Fatal("handler ran for an under-privileged identity") + } + if status.Code(err) != codes.PermissionDenied { + t.Fatalf("code = %v, want PermissionDenied", status.Code(err)) + } +} + +func TestAuthInterceptorAllowsOperator(t *testing.T) { + ic, token := testInterceptor(t, auth.RoleOperator) + called, err := run(t, ic, ctxWithToken(token)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !called { + t.Fatal("handler did not run for an authorized identity") + } +} + +func TestAuthInterceptorDeniesUnknownMethod(t *testing.T) { + ic, token := testInterceptor(t, auth.RoleAdmin) + called := false + handler := func(context.Context, any) (any, error) { called = true; return nil, nil } + info := &grpc.UnaryServerInfo{FullMethod: "/queue.v1.QueueService/DeleteEverything"} + if _, err := ic(ctxWithToken(token), nil, info, handler); status.Code(err) != codes.PermissionDenied { + t.Fatalf("code = %v, want PermissionDenied for an unmapped method", status.Code(err)) + } + if called { + t.Fatal("handler ran for an unmapped method") + } +} diff --git a/internal/gateway/main.go b/internal/gateway/main.go index c269308..5ec1f92 100644 --- a/internal/gateway/main.go +++ b/internal/gateway/main.go @@ -14,15 +14,13 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - // Absolute structural paths using your go.mod module name: + "forgequeue/internal/auth" "forgequeue/internal/durable" "forgequeue/internal/metrics" queuev1 "forgequeue/proto/v1" ) // server private struct having pointer to taskrouter to we can transfer downward to redis -// Implementing the service interface generated by the protobuf compiler -// Running a gRPC server to listen for requests from clients and dispatch them to the right service implementation. type server struct { queuev1.UnimplementedQueueServiceServer trouter *TaskRouter @@ -63,12 +61,14 @@ func (s *server) EnqueueTask( // raw netowrk listener socket opening func Start() { // Initialize your custom Redis connection pool pointer - redisAddress := "localhost:6379" + redisAddress := os.Getenv("FORGEQUEUE_REDIS_ADDR") + if redisAddress == "" { + redisAddress = "localhost:6379" + } - // Telemetry is opt-in: build it only when a scrape address is - // configured, otherwise the router and worker keep no-op instruments - // and nothing is exported. FORGEQUEUE_METRICS_ADDR is the scrape - // endpoint (the Prometheus default is :9090). + // Build telemetry only when a scrape address is set, otherwise the + // router keeps no-op instruments. FORGEQUEUE_METRICS_ADDR is the scrape + // endpoint (Prometheus default :9090). metricsAddr := os.Getenv("FORGEQUEUE_METRICS_ADDR") var ( @@ -97,9 +97,8 @@ func Start() { } taskRouter.WithMetrics(telemetry) - // The durable journal is optional and opt-in: set FORGEQUEUE_PG_DSN and - // every enqueue is journaled to Postgres first (survives crashes, heals - // Redis), otherwise the gateway runs Redis-only exactly as before. + // Set FORGEQUEUE_PG_DSN to journal every enqueue to Postgres first + // (survives crashes, heals Redis), otherwise the gateway runs Redis-only. if dsn := os.Getenv("FORGEQUEUE_PG_DSN"); dsn != "" { store, err := durable.NewPostgres(bootctx, dsn) if err != nil { @@ -118,8 +117,18 @@ func Start() { log.Fatalf("failed to listen: %v", err) } + // The gRPC surface is authenticated with the same bearer tokens the HTTP + // API issues, so the gateway shares its signing secret. Every method is + // authorized against methodPermissions; unknown methods fail closed. + authSvc, err := auth.NewFromEnv() + if err != nil { + log.Fatalf("failed to init auth: %v", err) + } + //Initialize the core gRPC microservice server container - grpcServer := grpc.NewServer() + grpcServer := grpc.NewServer( + grpc.UnaryInterceptor(newAuthInterceptor(authSvc)), + ) //Register our server implementation with the gRPC engine queuev1.RegisterQueueServiceServer( @@ -130,17 +139,16 @@ func Start() { log.Println("gateway network online on port :50051") - // Bring the scrape endpoint up alongside the gRPC listener. Prometheus - // then has something to poll for queue counters and latencies. + // Serve /metrics alongside the gRPC listener so Prometheus has + // something to poll. if metricsSrv != nil { if err := metricsSrv.Start(); err != nil { log.Fatalf("failed to start metrics server: %v", err) } log.Printf("metrics endpoint online on %s/metrics", metricsAddr) - // The gateway reports queue depth too, not just enqueues. Without - // this the depth gauges stay absent until a worker happens to - // publish, and an idle gateway would scrape as an empty endpoint. + // The gateway reports queue depth too, otherwise the gauges stay + // absent until a worker happens to publish. go metrics.NewDepthSampler( telemetry, taskRouter.redisClient, @@ -168,8 +176,7 @@ func Start() { log.Printf("received shutdown signal: %v", sh) grpcServer.GracefulStop() - // Stop the scrape endpoint and flush buffered metrics before exiting, - // so a scrape right before shutdown still sees the final counts. + // Stop the scrape endpoint and flush metrics before exiting. if metricsSrv != nil { stopCtx, stopCancel := context.WithTimeout(context.Background(), 3*time.Second) defer stopCancel() diff --git a/internal/gateway/router.go b/internal/gateway/router.go index 0dc574c..63dd29d 100644 --- a/internal/gateway/router.go +++ b/internal/gateway/router.go @@ -16,13 +16,11 @@ import ( // task router should be public // Every TaskRouter object will have a field named redisClient type TaskRouter struct { - redisClient *redis.Client // a pointer to the external redis client assigning to the redisclient field - // durable journal (Postgres). It records every accepted task BEFORE - // Redis is touched, so a crash mid-enqueue never loses the task - the - // reconciler finishes the write. Nop by default (no database configured). + redisClient *redis.Client + // durable journal. Records every accepted task before Redis is touched, + // so a crash mid-enqueue is healed by the reconciler. Nop by default. store durable.Store - // Observability. Never nil: the default router gets no-op instruments so - // instrumentation call sites stay free of nil checks. + // Observability. Never nil, defaults to no-op instruments. metrics *metrics.Metrics } @@ -30,29 +28,25 @@ type TaskRouter struct { // we assign it like router := NewTaskRouter("") func NewTaskRouter(ctx context.Context, addr string) (*TaskRouter, error) { - // fail fast //creating the redis client and pinging the redis server to check if it's available client, err := redisutil.NewClient(addr, 50, ctx) if err != nil { return nil, err } - return &TaskRouter{ // returns a pointer to a new TaskRouter struct with the redis client assigned to the redisClient field + return &TaskRouter{ redisClient: client, store: durable.Nop{}, metrics: metrics.NewNop(), }, nil } -// WithStore attaches the durable journal to the router. Call it before -// serving requests. Tasks are journaled first and materialized into Redis -// afterwards; a crash in between is repaired by the reconciler. +// WithStore attaches the durable journal. Call it before serving requests. func (tr *TaskRouter) WithStore(s durable.Store) *TaskRouter { tr.store = s return tr } -// WithMetrics attaches a telemetry sink to the router. Call it before -// serving requests. Passing nil is a no-op. +// WithMetrics attaches a telemetry sink. Call it before serving requests. func (tr *TaskRouter) WithMetrics(m *metrics.Metrics) *TaskRouter { if m == nil { return tr @@ -79,7 +73,6 @@ func (tr *TaskRouter) RouteTask( } // serialisation or if error return it serializedData, err := json.Marshal(data) - // checking error if the function fails if err != nil { return err } @@ -87,9 +80,8 @@ func (tr *TaskRouter) RouteTask( if delaySeconds > 0 { targetTime := time.Now().Unix() + delaySeconds - // Journal the task first. If we die before the Redis writes below, - // the reconciler sees redis_pending and materializes the task for - // us. This is the whole "journal first, then hot store" order. + // Journal first. If we die before the Redis writes below, the + // reconciler sees redis_pending and materializes the task. if err := tr.store.Enqueue(ctx, durable.Task{ ID: id, TaskType: taskType, @@ -101,10 +93,9 @@ func (tr *TaskRouter) RouteTask( return err } - // Write the body first so it is already in place when the worker's - // promotion loop moves the ID out of the scheduled set. The scheduled - // set only stores the task ID, matching the immediate path: the body - // always lives at task:, no matter which path enqueued it. + // Write the body first so it is in place when the promotion loop + // moves the ID out of the scheduled set. The set only stores the ID, + // the body always lives at task:. err := tr.redisClient.Set( ctx, "task:"+id, @@ -127,16 +118,13 @@ func (tr *TaskRouter) RouteTask( return err } - // Redis now provably holds the task; the journal can stop treating - // it as pending. + // Redis holds the task now, so clear pending. if err := tr.store.Materialized(ctx, id); err != nil { return err } - // Counted only after Redis accepted the task, so the metric - // reflects work the queue will actually run. Queue depth is not - // adjusted here: the depth gauges are sampled from Redis, which - // already counted this push. + // Counted after Redis accepts it. Depth is not adjusted here, the + // gauges sample Redis which already counted the push. tr.metrics.TasksEnqueued.Add(ctx, 1) now := time.Now().Unix() tr.redisClient.ZAdd(ctx, "metrics:enqueued", redis.Z{Score: float64(now), Member: id}) diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index d4f2ebd..65284a5 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -1,20 +1,13 @@ -// Package handlers provides the task handlers the worker binary registers -// out of the box. -// -// The queue is a delivery mechanism: it moves a JSON body from a producer to -// whoever knows what to do with it. That "who" is application code, so a -// real deployment registers its own handlers and this package is only the -// runnable default. +// Package handlers provides the default task handlers the worker binary +// registers. // -// The three handlers here exist so a fresh checkout has something that -// actually processes tasks end to end, and so each of the worker's telemetry -// signals can be provoked deliberately: +// The queue just moves a JSON body to whoever knows what to do with it, so a +// real deployment registers its own handlers. These three are here so a fresh +// checkout runs end to end: // -// "echo" succeeds immediately. Exercises the claim -> handle -> ACK path. -// "sleep" runs for a duration. Exercises heartbeats and lease renewal, and -// shows what happens when a handler is cancelled mid-flight. -// "fail" always fails. Exercises the retry budget, backoff, and the -// dead-letter queue. +// echo succeeds immediately. +// sleep runs for a duration. +// fail always fails. package handlers import ( @@ -40,10 +33,7 @@ type sleepRequest struct { DurationMS int `json:"duration_ms"` } -// Echo completes immediately without doing any work. -// -// Useful as a liveness probe for the queue: enqueue one and confirm -// tasks_processed_total climbs without touching anything else. +// Echo completes immediately. Handy as a liveness probe for the queue. type Echo struct{} // Handle implements worker.TaskHandler. @@ -56,24 +46,20 @@ func (Echo) Handle(ctx context.Context, task *model.TaskMetaData) error { return nil } -// Sleep blocks for a duration and then succeeds. -// -// This is how you exercise the heartbeat path without writing slow real code: -// a task longer than the lease duration proves the lease is being renewed, -// because otherwise recovery would reclaim it mid-flight. +// Sleep blocks for a duration then succeeds. A task longer than the lease +// proves heartbeats are renewing it. // -// The wait is context-aware. When the heartbeat loses the lease the worker -// cancels the task context, and returning promptly on ctx.Done() is what lets -// the worker stop work it no longer owns. A handler that ignores its context -// would keep running against a task another worker is now executing. +// The wait is context-aware: when the heartbeat loses the lease the worker +// cancels the context, and returning on ctx.Done() stops work the worker no +// longer owns. type Sleep struct{} // Handle implements worker.TaskHandler. func (Sleep) Handle(ctx context.Context, task *model.TaskMetaData) error { var req sleepRequest - // An unparseable payload is a permanent error: retrying it would fail - // identically forever and just burn the retry budget. + // A bad payload is a permanent error, retrying would just burn the + // retry budget. if len(task.Payload) > 0 { if err := json.Unmarshal(task.Payload, &req); err != nil { return fmt.Errorf("sleep: bad payload: %w", err) @@ -90,19 +76,13 @@ func (Sleep) Handle(ctx context.Context, task *model.TaskMetaData) error { return nil case <-ctx.Done(): - // Returning the context error marks the attempt as failed, so the - // task is requeued or dead-lettered by the normal path. It is not - // special-cased as a lease loss here; the worker's heartbeat loop - // already recorded that separately. + // Returning the error marks the attempt failed, so it is requeued + // or dead-lettered. The heartbeat loop records the lease loss. return ctx.Err() } } -// Fail always returns an error. -// -// It exists to exercise the failure paths on demand: enqueue it with a -// max_retries budget and watch tasks_retried_total climb, then -// tasks_dead_lettered_total once the budget is spent. +// Fail always returns an error, for exercising the retry and dead-letter paths. type Fail struct{} // Handle implements worker.TaskHandler. @@ -110,16 +90,11 @@ func (Fail) Handle(ctx context.Context, task *model.TaskMetaData) error { return fmt.Errorf("fail: task %s failed on purpose", task.ID) } -// Default returns the task type to handler mapping the worker binary -// registers. -// -// Returning the map rather than registering one by one keeps cmd/worker -// honest about what it supports, and gives a caller a way to add to it -// before handing it over: +// Default returns the task type to handler mapping the worker registers. A +// caller can add to it before handing it over: // // m := handlers.Default() // m["email"] = myEmailHandler{} -// wp.RegisterAll(m) func Default() map[string]WorkerHandler { return map[string]WorkerHandler{ TypeEcho: Echo{}, @@ -128,12 +103,8 @@ func Default() map[string]WorkerHandler { } } -// WorkerHandler mirrors worker.TaskHandler without importing it. -// -// The interface is structural in Go, so any type with this method satisfies -// worker.TaskHandler. Declaring it here keeps this package free of a -// dependency on the worker package, which would be circular once the worker -// binary imports both. +// WorkerHandler mirrors worker.TaskHandler without importing it. Go +// interfaces are structural, so declaring it here avoids a circular import. type WorkerHandler interface { Handle(ctx context.Context, task *model.TaskMetaData) error } diff --git a/internal/httpapi/auth_test.go b/internal/httpapi/auth_test.go new file mode 100644 index 0000000..89702f5 --- /dev/null +++ b/internal/httpapi/auth_test.go @@ -0,0 +1,262 @@ +package httpapi + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "forgequeue/internal/auth" +) + +func newAuthServer(t *testing.T) *Server { + t.Helper() + t.Setenv("FORGEQUEUE_AUTH_SECRET", "test-secret") + t.Setenv("FORGEQUEUE_USERS", `[ + {"username":"admin","password":"admin-pw","roles":["admin"]}, + {"username":"viewer","password":"viewer-pw","roles":["viewer"]} + ]`) + svc, err := auth.NewFromEnv() + if err != nil { + t.Fatalf("auth config: %v", err) + } + return NewServer("", nil, svc) +} + +func login(t *testing.T, s *Server, username, password string) string { + t.Helper() + body := strings.NewReader(`{"username":"` + username + `","password":"` + password + `"}`) + req := httptest.NewRequest(http.MethodPost, "/api/auth/login", body) + rec := httptest.NewRecorder() + s.srv.Handler.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("login status = %d, body = %s", rec.Code, rec.Body.String()) + } + var resp LoginResponse + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode login: %v", err) + } + if resp.Token == "" || resp.User.Username != username { + t.Fatalf("unexpected login response: %+v", resp) + } + return resp.Token +} + +func TestLoginAndMe(t *testing.T) { + s := newAuthServer(t) + token := login(t, s, "admin", "admin-pw") + + req := httptest.NewRequest(http.MethodGet, "/api/auth/me", nil) + req.Header.Set("Authorization", "Bearer "+token) + rec := httptest.NewRecorder() + s.srv.Handler.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("me status = %d, body = %s", rec.Code, rec.Body.String()) + } + var id auth.Identity + if err := json.Unmarshal(rec.Body.Bytes(), &id); err != nil { + t.Fatalf("decode me: %v", err) + } + if id.Username != "admin" || !id.Can(auth.PermJobsWrite) { + t.Fatalf("unexpected identity: %+v", id) + } +} + +func TestRejectsBadCredentials(t *testing.T) { + s := newAuthServer(t) + req := httptest.NewRequest(http.MethodPost, "/api/auth/login", strings.NewReader(`{"username":"admin","password":"wrong"}`)) + rec := httptest.NewRecorder() + s.srv.Handler.ServeHTTP(rec, req) + if rec.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401", rec.Code) + } +} + +func TestMeRequiresBearer(t *testing.T) { + s := newAuthServer(t) + for _, header := range []string{"", "Bearer ", "Bearer not-a-token", "Basic abc"} { + req := httptest.NewRequest(http.MethodGet, "/api/auth/me", nil) + if header != "" { + req.Header.Set("Authorization", header) + } + rec := httptest.NewRecorder() + s.srv.Handler.ServeHTTP(rec, req) + if rec.Code != http.StatusUnauthorized { + t.Fatalf("header %q: status = %d, want 401", header, rec.Code) + } + } +} + +func TestPermissionEnforced(t *testing.T) { + s := newAuthServer(t) + viewer := login(t, s, "viewer", "viewer-pw") + + handler := s.require(auth.PermQueuesWrite, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + // Viewer: authenticated but not authorized for a write. + req := httptest.NewRequest(http.MethodPost, "/api/queues/default/pause", nil) + req.Header.Set("Authorization", "Bearer "+viewer) + rec := httptest.NewRecorder() + handler(rec, req) + if rec.Code != http.StatusForbidden { + t.Fatalf("viewer write status = %d, want 403", rec.Code) + } + + // Same viewer is allowed to read. + read := s.require(auth.PermQueuesRead, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + req = httptest.NewRequest(http.MethodGet, "/api/queues", nil) + req.Header.Set("Authorization", "Bearer "+viewer) + rec = httptest.NewRecorder() + read(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("viewer read status = %d, want 200", rec.Code) + } +} + +func do(s *Server, method, path, token, body string) *httptest.ResponseRecorder { + req := httptest.NewRequest(method, path, strings.NewReader(body)) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + rec := httptest.NewRecorder() + s.srv.Handler.ServeHTTP(rec, req) + return rec +} + +func TestAuthConfigWithoutOIDC(t *testing.T) { + s := newAuthServer(t) + rec := do(s, http.MethodGet, "/api/auth/config", "", "") + if rec.Code != http.StatusOK { + t.Fatalf("status = %d", rec.Code) + } + var cfg AuthConfig + if err := json.Unmarshal(rec.Body.Bytes(), &cfg); err != nil { + t.Fatalf("decode: %v", err) + } + if !cfg.Password || cfg.OIDC { + t.Fatalf("config = %+v, want password-only", cfg) + } + // SSO endpoints are absent when OIDC is disabled. + if rec := do(s, http.MethodGet, "/api/auth/oidc/login", "", ""); rec.Code != http.StatusNotFound { + t.Fatalf("oidc login status = %d, want 404", rec.Code) + } +} + +// fakeOIDCProvider serves discovery, token and userinfo for a browser-less flow. +func fakeOIDCProvider(t *testing.T, email string) *httptest.Server { + t.Helper() + mux := http.NewServeMux() + var base string + mux.HandleFunc("/.well-known/openid-configuration", func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]string{ + "issuer": base, + "authorization_endpoint": base + "/authorize", + "token_endpoint": base + "/token", + "userinfo_endpoint": base + "/userinfo", + }) + }) + mux.HandleFunc("/token", func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]string{"access_token": "tok"}) + }) + mux.HandleFunc("/userinfo", func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]string{"sub": "1", "email": email, "name": "SSO Person"}) + }) + srv := httptest.NewServer(mux) + base = srv.URL + t.Cleanup(srv.Close) + return srv +} + +func TestOIDCLoginFlowEndToEnd(t *testing.T) { + idp := fakeOIDCProvider(t, "ops@example.com") + t.Setenv("FORGEQUEUE_AUTH_SECRET", "test-secret") + t.Setenv("FORGEQUEUE_USERS", `[{"username":"admin","password":"pw","roles":["admin"]}]`) + t.Setenv("FORGEQUEUE_OIDC_ISSUER", idp.URL) + t.Setenv("FORGEQUEUE_OIDC_CLIENT_ID", "client") + t.Setenv("FORGEQUEUE_OIDC_REDIRECT_URL", "http://api.local/api/auth/oidc/callback") + t.Setenv("FORGEQUEUE_WEB_URL", "http://console.local") + t.Setenv("FORGEQUEUE_OIDC_OPERATOR_EMAILS", "ops@example.com") + + svc, err := auth.NewFromEnv() + if err != nil { + t.Fatalf("auth config: %v", err) + } + oidc, err := auth.NewOIDCFromEnv(svc, auth.NewMemoryStore()) + if err != nil { + t.Fatalf("oidc config: %v", err) + } + s := NewServer("", nil, svc) + s.SetOIDC(oidc) + + // Config advertises both methods. + rec := do(s, http.MethodGet, "/api/auth/config", "", "") + var cfg AuthConfig + _ = json.Unmarshal(rec.Body.Bytes(), &cfg) + if !cfg.OIDC || !cfg.Password { + t.Fatalf("config = %+v, want password+oidc", cfg) + } + + // Begin: redirect to the provider with state. + rec = do(s, http.MethodGet, "/api/auth/oidc/login?redirect="+url.QueryEscape("http://console.local/login"), "", "") + if rec.Code != http.StatusFound { + t.Fatalf("login status = %d, body = %s", rec.Code, rec.Body.String()) + } + authURL, err := url.Parse(rec.Header().Get("Location")) + if err != nil { + t.Fatalf("parse location: %v", err) + } + if !strings.HasPrefix(authURL.String(), idp.URL+"/authorize") { + t.Fatalf("redirect target = %q", authURL) + } + state := authURL.Query().Get("state") + if state == "" { + t.Fatal("missing state") + } + + // Callback: exchange the code and bounce back to the console with an otc. + rec = do(s, http.MethodGet, "/api/auth/oidc/callback?state="+state+"&code=abc", "", "") + if rec.Code != http.StatusFound { + t.Fatalf("callback status = %d, body = %s", rec.Code, rec.Body.String()) + } + back, err := url.Parse(rec.Header().Get("Location")) + if err != nil { + t.Fatalf("parse callback location: %v", err) + } + if back.Scheme+"://"+back.Host+back.Path != "http://console.local/login" { + t.Fatalf("return target = %q", back) + } + otc := back.Query().Get("code") + if otc == "" { + t.Fatalf("missing one-time code in %q", back) + } + + // Exchange the otc for a usable bearer token (operator role). + rec = do(s, http.MethodPost, "/api/auth/oidc/exchange", "", `{"code":"`+otc+`"}`) + if rec.Code != http.StatusOK { + t.Fatalf("exchange status = %d, body = %s", rec.Code, rec.Body.String()) + } + var resp LoginResponse + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode exchange: %v", err) + } + if !resp.User.Can(auth.PermQueuesWrite) { + t.Fatalf("sso identity lacks queues:write: %+v", resp.User) + } + + // The token authenticates against protected endpoints. + meRec := do(s, http.MethodGet, "/api/auth/me", resp.Token, "") + if meRec.Code != http.StatusOK { + t.Fatalf("me status = %d", meRec.Code) + } + + // The one-time code cannot be replayed. + if rec := do(s, http.MethodPost, "/api/auth/oidc/exchange", "", `{"code":"`+otc+`"}`); rec.Code != http.StatusUnauthorized { + t.Fatalf("replay status = %d, want 401", rec.Code) + } +} diff --git a/internal/httpapi/handlers.go b/internal/httpapi/handlers.go index 59f46a8..377b8b1 100644 --- a/internal/httpapi/handlers.go +++ b/internal/httpapi/handlers.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "net/http" + "net/url" "time" "github.com/redis/go-redis/v9" @@ -15,12 +16,16 @@ import ( type Handler struct { rdb *redis.Client auth *auth.Service + oidc *auth.OIDC } func NewHandler(rdb *redis.Client, authSvc *auth.Service) *Handler { return &Handler{rdb: rdb, auth: authSvc} } +// setOIDC enables the single sign-on handlers. +func (h *Handler) setOIDC(o *auth.OIDC) { h.oidc = o } + func (h *Handler) writeJSON(w http.ResponseWriter, status int, v interface{}) { writeJSON(w, status, v) } @@ -71,6 +76,88 @@ func (h *Handler) Me(w http.ResponseWriter, r *http.Request) { } h.writeJSON(w, http.StatusOK, id) } + +// AuthConfig advertises which login methods are available. +func (h *Handler) AuthConfig(w http.ResponseWriter, r *http.Request) { + cfg := AuthConfig{Password: h.auth.HasLocalUsers()} + if h.oidc != nil { + cfg.OIDC = true + cfg.Provider = h.oidc.ProviderName() + } + h.writeJSON(w, http.StatusOK, cfg) +} + +// OIDCLogin begins the SSO flow and redirects the browser to the provider. +func (h *Handler) OIDCLogin(w http.ResponseWriter, r *http.Request) { + if h.oidc == nil { + h.writeError(w, http.StatusNotFound, "single sign-on is not enabled") + return + } + target, err := h.oidc.Begin(r.Context(), r.URL.Query().Get("redirect")) + if err != nil { + h.writeError(w, http.StatusBadGateway, "single sign-on is unavailable") + return + } + http.Redirect(w, r, target, http.StatusFound) +} + +// OIDCCallback handles the provider redirect and passes a one-time code +// back to the console. +func (h *Handler) OIDCCallback(w http.ResponseWriter, r *http.Request) { + if h.oidc == nil { + h.writeError(w, http.StatusNotFound, "single sign-on is not enabled") + return + } + q := r.URL.Query() + if msg := q.Get("error"); msg != "" { + redirectTo(w, r, h.oidc.DefaultReturn(), map[string]string{"error": msg}) + return + } + otc, sess, err := h.oidc.Complete(r.Context(), q.Get("state"), q.Get("code")) + if err != nil { + redirectTo(w, r, h.oidc.DefaultReturn(), map[string]string{"error": "sso_failed"}) + return + } + redirectTo(w, r, sess.WebReturn, map[string]string{"code": otc}) +} + +// OIDCExchange exchanges the one-time code for a bearer token. +func (h *Handler) OIDCExchange(w http.ResponseWriter, r *http.Request) { + if h.oidc == nil { + h.writeError(w, http.StatusNotFound, "single sign-on is not enabled") + return + } + var req ExchangeRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.Code == "" { + h.writeError(w, http.StatusBadRequest, "code is required") + return + } + sess, err := h.oidc.Exchange(r.Context(), req.Code) + if err != nil { + h.writeError(w, http.StatusUnauthorized, "invalid or expired code") + return + } + h.writeJSON(w, http.StatusOK, LoginResponse{ + Token: sess.Token, + ExpiresAt: sess.ExpiresAt.UTC().Format(time.RFC3339), + User: sess.Identity, + }) +} + +// redirectTo sends the browser to target, appending the given query params. +func redirectTo(w http.ResponseWriter, r *http.Request, target string, params map[string]string) { + dest, err := url.Parse(target) + if err != nil { + dest = &url.URL{Path: "/"} + } + q := dest.Query() + for k, v := range params { + q.Set(k, v) + } + dest.RawQuery = q.Encode() + http.Redirect(w, r, dest.String(), http.StatusFound) +} + func (h *Handler) Stats(w http.ResponseWriter, r *http.Request) { ctx := r.Context() pending := int(h.rdb.LLen(ctx, "queue:tasks:immediate").Val()) @@ -199,9 +286,8 @@ func (h *Handler) readTaskFull(r *http.Request, id string) (*Job, bool, error) { return job, true, nil } -// isDead reports whether a task body with the given ID is present in the -// dead-letter list. The dead list stores full JSON bodies, so membership is -// checked by parsing each entry rather than by matching the raw list value. +// isDead reports whether a task with the given ID is in the dead list. +// Entries are full JSON bodies, so we parse each one to match the ID. func (h *Handler) isDead(ctx context.Context, id string) bool { bodies, err := h.rdb.LRange(ctx, "queue:tasks:dead", 0, -1).Result() if err != nil { @@ -216,8 +302,8 @@ func (h *Handler) isDead(ctx context.Context, id string) bool { return false } -// pausedKey is set while the queue is paused. The worker's claim script -// checks for its presence and stops handing out work until it is cleared. +// pausedKey is set while the queue is paused. Workers check for it +// before claiming work. const pausedKey = "queue:paused" // queueStatus reports whether the queue is currently accepting work. @@ -228,9 +314,8 @@ func (h *Handler) queueStatus(ctx context.Context) QueueStatus { return QueueHealthy } -// takeDead removes and returns the stored body of a task parked in the -// dead-letter list. The dead list holds full JSON bodies, so the entry is -// located by parsing rather than by matching an ID. +// takeDead removes and returns the body of a task sitting in the dead list. +// Entries are full JSON bodies, so we parse to find the matching ID. func (h *Handler) takeDead(ctx context.Context, id string) ([]byte, bool, error) { bodies, err := h.rdb.LRange(ctx, "queue:tasks:dead", 0, -1).Result() if err != nil { @@ -241,8 +326,8 @@ func (h *Handler) takeDead(ctx context.Context, id string) ([]byte, bool, error) for _, body := range bodies { var tm TaskMetaData if json.Unmarshal([]byte(body), &tm) == nil && tm.ID == id { - // Keep the first body we see, but clear every matching entry: the - // same task can appear more than once in the dead list. + // Keep the first match but remove every one. A task can appear + // more than once in the dead list. if !ok { found = []byte(body) ok = true @@ -255,16 +340,13 @@ func (h *Handler) takeDead(ctx context.Context, id string) ([]byte, bool, error) return found, ok, nil } -// requeue puts a task back on the ready list, clearing any stale membership -// in the scheduled or inflight sets first. resetAttempts restores the retry -// budget, which is what a manual "retry" should do. +// requeue puts a task back on the ready list after clearing it from the +// scheduled and inflight sets. resetAttempts zeroes the attempt count. // -// The body is taken from task: when present, otherwise recovered from the -// dead-letter list. When neither holds it, the task is reported as missing. +// The body comes from task:, or from the dead list if that is gone. func (h *Handler) requeue(ctx context.Context, id string, resetAttempts bool) (bool, error) { - // Always clear the dead-letter list first, even if a body still exists in - // task:: a task can leave stale copies behind there. The recovered - // body is only needed when task: has already been dropped. + // Always clear the dead list first, a task can leave stale copies there. + // The recovered body is only needed when task: is gone. deadBody, wasDead, err := h.takeDead(ctx, id) if err != nil { return false, err @@ -392,9 +474,8 @@ func (h *Handler) Worker(w http.ResponseWriter, r *http.Request, id string) { h.writeError(w, http.StatusNotFound, fmt.Sprintf("worker %s not found", id)) } -// registeredWorker mirrors the document the worker pool publishes at -// worker::meta. It lives here (not in the worker package) so the API -// stays decoupled from worker internals. +// registeredWorker mirrors the doc the worker pool writes to worker::meta. +// It lives here to keep the API decoupled from worker internals. type registeredWorker struct { ID string `json:"id"` Hostname string `json:"hostname"` @@ -405,10 +486,8 @@ type registeredWorker struct { Concurrency int `json:"concurrency"` } -// readWorkers assembles the live worker list from the registry Redis keys. -// Stale registrations (meta key expired, or heartbeat older than the healthy -// window) are reported with status "stale" rather than dropped, so the -// dashboard can see a worker that stopped heartbeating. +// readWorkers builds the worker list from the registry keys. Workers whose +// heartbeat is old are marked "stale" instead of dropped. func (h *Handler) readWorkers(ctx context.Context) []Worker { ids, err := h.rdb.SMembers(ctx, "forgequeue:workers").Result() if err != nil { @@ -455,9 +534,8 @@ func (h *Handler) readWorkers(ctx context.Context) []Worker { return workers } -// readWorkerLeases returns the leases a worker is actively holding. A task is -// only counted if Redis still knows about its lease, which filters out IDs -// left behind after a crash. +// readWorkerLeases returns the leases a worker is holding. A task only +// counts if Redis still has its lease, which drops IDs left over from a crash. func (h *Handler) readWorkerLeases(ctx context.Context, id string) []Lease { ids, err := h.rdb.SMembers(ctx, "worker:"+id+":leases").Result() if err != nil { @@ -665,7 +743,6 @@ func genSeries(rangeQ string, rdb *redis.Client, ctx context.Context) []Throughp bucketEnd := bucketStart.Add(step) start := bucketStart.Unix() end := bucketEnd.Unix() - // Count real events in this bucket enq := int(rdb.ZCount(ctx, "metrics:enqueued", fmt.Sprint(start), fmt.Sprint(end)).Val()) comp := int(rdb.ZCount(ctx, "metrics:completed", fmt.Sprint(start), fmt.Sprint(end)).Val()) fail := int(rdb.ZCount(ctx, "metrics:failed", fmt.Sprint(start), fmt.Sprint(end)).Val()) diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index d952938..6f37aea 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -15,6 +15,8 @@ import ( type Server struct { srv *http.Server auth *auth.Service + oidc *auth.OIDC + h *Handler } func NewServer(addr string, rdb *redis.Client, authSvc *auth.Service) *Server { @@ -26,12 +28,27 @@ func NewServer(addr string, rdb *redis.Client, authSvc *auth.Service) *Server { h := NewHandler(rdb, authSvc) mux := http.NewServeMux() - // Public endpoints. + mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) + }) + mux.HandleFunc("GET /readyz", func(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second) + defer cancel() + if err := rdb.Ping(ctx).Err(); err != nil { + writeError(w, http.StatusServiceUnavailable, "redis unavailable") + return + } + writeJSON(w, http.StatusOK, map[string]string{"status": "ready"}) + }) + mux.HandleFunc("POST /api/auth/login", h.Login) - mux.HandleFunc("GET /api/system/version", h.Version) + mux.HandleFunc("GET /api/auth/config", h.AuthConfig) + mux.HandleFunc("GET /api/auth/oidc/login", h.OIDCLogin) + mux.HandleFunc("GET /api/auth/oidc/callback", h.OIDCCallback) + mux.HandleFunc("POST /api/auth/oidc/exchange", h.OIDCExchange) - // Authenticated endpoints, each gated by a permission. mux.HandleFunc("GET /api/auth/me", s.require("", h.Me)) + mux.HandleFunc("GET /api/system/version", s.require(auth.PermSystemRead, h.Version)) mux.HandleFunc("GET /api/system/stats", s.require(auth.PermSystemRead, h.Stats)) mux.HandleFunc("GET /api/queues", s.require(auth.PermQueuesRead, h.Queues)) mux.HandleFunc("GET /api/queues/{name}", s.require(auth.PermQueuesRead, func(w http.ResponseWriter, r *http.Request) { @@ -88,9 +105,18 @@ func NewServer(addr string, rdb *redis.Client, authSvc *auth.Service) *Server { ReadHeaderTimeout: 5 * time.Second, } s.srv = srv + s.h = h return s } +// SetOIDC enables single sign-on on the server. Call before Start. +func (s *Server) SetOIDC(o *auth.OIDC) { + s.oidc = o + if s.h != nil { + s.h.setOIDC(o) + } +} + // require wraps a handler with bearer-token authentication and an optional // permission check. An empty perm authenticates without authorizing. func (s *Server) require(perm auth.Permission, next http.HandlerFunc) http.HandlerFunc { diff --git a/internal/httpapi/types.go b/internal/httpapi/types.go index 41f2cba..f099257 100644 --- a/internal/httpapi/types.go +++ b/internal/httpapi/types.go @@ -19,6 +19,18 @@ type LoginResponse struct { User auth.Identity `json:"user"` } +// AuthConfig describes the login methods the server offers. +type AuthConfig struct { + Password bool `json:"password"` + OIDC bool `json:"oidc"` + Provider string `json:"provider,omitempty"` +} + +// ExchangeRequest carries the one-time SSO code used to get a session token. +type ExchangeRequest struct { + Code string `json:"code"` +} + type JobStatus string const ( diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 417158d..ccd5125 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -1,8 +1,5 @@ -// Package metrics wires OpenTelemetry instruments to a Prometheus registry -// so the queue's behaviour is observable over HTTP. -// -// Instruments are always non-nil: a disabled Metrics (via NewNop) still -// returns valid no-op instruments, so call sites never need nil checks. +// Package metrics wires OpenTelemetry instruments to a Prometheus registry. +// Instruments are always non-nil, even when telemetry is disabled. package metrics import ( @@ -20,8 +17,8 @@ import ( sdkmetric "go.opentelemetry.io/otel/sdk/metric" ) -// Metrics holds every instrument the queue records into. All fields are -// usable even when telemetry is disabled; they become no-ops then. +// Metrics holds every instrument the queue records into. Fields stay usable +// when telemetry is off, they just become no-ops. type Metrics struct { Ready bool provider *sdkmetric.MeterProvider @@ -35,14 +32,8 @@ type Metrics struct { TasksDLQ metric.Int64Counter LeaseLosses metric.Int64Counter - // Queue depths are observable gauges rather than up/down counters. - // - // An up/down counter needs a matching decrement on every claim, ACK, - // promotion, failure and recovery, spread across the gateway and every - // worker process. Any missed transition leaves the value permanently - // wrong with no way to correct it, and separate processes would each - // export their own partial total. Redis already holds the - // authoritative depth, so these gauges are sampled from it instead. + // Queue depths are gauges sampled from Redis. Up/down counters would + // drift, since a missed decrement cannot be corrected. QueueReady metric.Int64Gauge QueueInflight metric.Int64Gauge QueueScheduled metric.Int64Gauge @@ -54,28 +45,25 @@ type Metrics struct { TaskDuration metric.Float64Histogram } -// DefaultLatencyBuckets covers sub-millisecond Redis calls up to ten seconds, -// which is the range a single claim/ack/heartbeat round trip can occupy. +// DefaultLatencyBuckets covers redis calls from sub-millisecond up to ten +// seconds. var DefaultLatencyBuckets = []float64{ 0.0005, 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, } -// DefaultTaskBuckets spans a no-op handler up to a two minute handler. +// DefaultTaskBuckets spans a no-op handler up to a long-running one. var DefaultTaskBuckets = []float64{ 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10, 30, 60, 120, 300, } -// New builds the instruments and a private Prometheus registry. Call -// Handler to expose it, or pass Registry to NewServer. +// New builds the instruments and a private Prometheus registry. func New() (*Metrics, error) { reg := prometheus.NewRegistry() exporter, err := otelprom.New( otelprom.WithRegisterer(reg), - // Scope and target_info labels are constant for this process and - // carry no information a dashboard can use. Dropping them keeps - // the series names readable instead of drowning every line in - // otel_scope_* boilerplate. + // Scope and target_info labels are constant here, so drop them to + // keep the series names readable. otelprom.WithoutScopeInfo(), otelprom.WithoutTargetInfo(), ) @@ -205,8 +193,7 @@ func New() (*Metrics, error) { return m, nil } -// NewNop returns a Metrics whose instruments record nothing. Useful in tests -// and anywhere telemetry is not wired up. +// NewNop returns a Metrics that records nothing. func NewNop() *Metrics { meter := noop.NewMeterProvider().Meter("forgequeue") m := &Metrics{latencyBs: DefaultLatencyBuckets, taskBs: DefaultTaskBuckets} @@ -230,9 +217,7 @@ func NewNop() *Metrics { return m } -// QueueDepths is a point-in-time snapshot of how many tasks sit in each -// queue. Collected by Redis so it reflects every process, not just the one -// reporting it. +// QueueDepths is a snapshot of how many tasks sit in each queue. type QueueDepths struct { Ready int64 Inflight int64 @@ -240,11 +225,8 @@ type QueueDepths struct { Dead int64 } -// RecordQueueDepths publishes a depth snapshot to the queue gauges. -// -// It records the ready queue even when it is empty: a zero is a meaningful -// value here ("the queue is drained"), and skipping it would leave the last -// non-zero reading on the dashboard forever. +// RecordQueueDepths publishes a depth snapshot to the queue gauges. Zero is +// recorded too, otherwise a drained queue leaves a stale reading behind. func (m *Metrics) RecordQueueDepths(ctx context.Context, d QueueDepths) { m.QueueReady.Record(ctx, d.Ready) m.QueueInflight.Record(ctx, d.Inflight) @@ -252,13 +234,8 @@ func (m *Metrics) RecordQueueDepths(ctx context.Context, d QueueDepths) { m.QueueDLQ.Record(ctx, d.Dead) } -// DepthSampler periodically publishes queue depths read from Redis. -// -// Both the gateway and the worker can run one: they are separate processes -// reporting the same shared queues, and each scrape target simply states -// the current depth. Note that a dashboard should read these with max() or -// last() rather than sum(), since more than one process may report the same -// queue. +// DepthSampler periodically publishes queue depths read from Redis. Both the +// gateway and workers can run one, so read them with max() not sum(). type DepthSampler struct { metrics *Metrics client redis.Cmdable @@ -269,11 +246,8 @@ type DepthSampler struct { // DefaultDepthInterval is how often a DepthSampler refreshes the gauges. const DefaultDepthInterval = 5 * time.Second -// NewDepthSampler returns a sampler reading depths from client. -// -// A nil onFailure swallows errors silently, which suits tests. Production -// callers should log so a broken Redis does not go unnoticed just because -// the queue-depth panel went flat. +// NewDepthSampler returns a sampler reading depths from client. A nil +// onFailure swallows errors, which suits tests. func NewDepthSampler( m *Metrics, client redis.Cmdable, @@ -292,9 +266,8 @@ func NewDepthSampler( } } -// Run publishes depths until ctx is cancelled. It publishes once before -// waiting, so the first scrape does not have to wait a full interval for -// numbers that are already knowable. +// Run publishes depths until ctx is cancelled. Publishes once up front so +// the first scrape does not wait a full interval. func (s *DepthSampler) Run(ctx context.Context) { s.Publish(ctx) @@ -312,11 +285,8 @@ func (s *DepthSampler) Run(ctx context.Context) { } } -// Publish samples the depths once and records them. -// -// On failure the gauges are left untouched. Overwriting them with zero would -// be actively misleading: an unreachable Redis and a genuinely drained queue -// are indistinguishable on a dashboard, and only one of them is healthy. +// Publish samples the depths once and records them. On failure the gauges are +// left untouched, so an unreachable Redis is not shown as a drained queue. func (s *DepthSampler) Publish(ctx context.Context) { depths, err := s.Depths(ctx) if err != nil { @@ -329,14 +299,9 @@ func (s *DepthSampler) Publish(ctx context.Context) { s.metrics.RecordQueueDepths(ctx, depths) } -// Depths reads the current depth of every queue in a single round trip. -// -// The four commands are pipelined, so this costs one network round trip -// rather than four and the numbers come from roughly the same instant. +// Depths reads the depth of every queue in one pipelined round trip. func (s *DepthSampler) Depths(ctx context.Context) (QueueDepths, error) { - // A sampler with no client is a wiring mistake. Return an error so the - // failure is reported rather than panicking inside the caller's - // goroutine, which would take the whole process down. + // No client is a wiring mistake. Return an error rather than panic. if s.client == nil { return QueueDepths{}, errors.New("depth sampler has no redis client") } @@ -360,8 +325,7 @@ func (s *DepthSampler) Depths(ctx context.Context) (QueueDepths, error) { }, nil } -// Redis key names holding each queue. Exported so a dashboard config or a -// test can refer to the same constants the sampler reads. +// Redis key names holding each queue. const ( keyReady = "queue:tasks:immediate" keyInflight = "queue:tasks:inflight" @@ -369,8 +333,7 @@ const ( keyDead = "queue:tasks:dead" ) -// Registry exposes the underlying registry so callers can register extra -// collectors or serve it themselves. +// Registry exposes the underlying registry. func (m *Metrics) Registry() *prometheus.Registry { return m.registry } // Handler serves the Prometheus exposition format. @@ -391,8 +354,7 @@ type Server struct { srv *http.Server } -// NewServer returns a server that serves m on addr. A blank addr serves -// /metrics on :9090. +// NewServer returns a server serving m on addr. A blank addr uses :9090. func NewServer(addr string, m *Metrics) *Server { if addr == "" { addr = ":9090" @@ -400,6 +362,14 @@ func NewServer(addr string, m *Metrics) *Server { mux := http.NewServeMux() mux.Handle("/metrics", m.Handler()) + mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok")) + }) + mux.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ready")) + }) return &Server{ srv: &http.Server{ diff --git a/internal/model/task.go b/internal/model/task.go index 1caabcb..db62451 100644 --- a/internal/model/task.go +++ b/internal/model/task.go @@ -5,9 +5,7 @@ type TaskMetaData struct { TaskType string `json:"task_type"` // 16 byte Payload []byte `json:"payload"` // 24 bytes MaxRetries int32 `json:"max_retries"` // 4 bytes - // Attempts counts how many times this task has been executed and - // failed. It lives in the body so it travels with the task through - // every queue and survives restarts. Defaults to 0 when the task was - // enqueued before this field existed. + // how many times this task ran and failed. lives in the body so it + // travels with the task and survives restarts. defaults to 0. Attempts int32 `json:"attempts"` // 4 bytes } diff --git a/internal/redisutil/client.go b/internal/redisutil/client.go index 78c4f7a..3d05c50 100644 --- a/internal/redisutil/client.go +++ b/internal/redisutil/client.go @@ -14,7 +14,7 @@ func NewClient(addr string, poolSize int, ctx context.Context) (*redis.Client, e Addr: addr, PoolSize: poolSize, }) - // fail fast: ping the redis server to check if it's available + // ping to check the server is up if err := client.Ping(ctx).Err(); err != nil { return nil, err } diff --git a/internal/worker/registry.go b/internal/worker/registry.go index 3dc152f..4596df1 100644 --- a/internal/worker/registry.go +++ b/internal/worker/registry.go @@ -17,10 +17,9 @@ func (wp *WorkerPool) leasesKey() string { return workerMetaPrefix + wp.id + ":leases" } -// publishMeta writes the registration document and refreshes its TTL. It is -// called once on Start and then on every heartbeat tick. Best effort: a -// failure to publish only hides the worker from the dashboard, it never -// stops task processing. +// publishMeta writes the registration document and refreshes its TTL. called +// on Start and every heartbeat tick. best effort: a failed publish only hides +// the worker from the dashboard. func (wp *WorkerPool) publishMeta() { meta := workerMeta{ ID: wp.id, @@ -62,7 +61,7 @@ func (wp *WorkerPool) registrationLoop() { } // deregister removes this worker from the registry so a clean shutdown -// disappears from the dashboard immediately instead of waiting for the TTL. +// disappears from the dashboard right away instead of waiting for the TTL. func (wp *WorkerPool) deregister() { ctx := context.Background() pipe := wp.redisClient.TxPipeline() diff --git a/internal/worker/worker.go b/internal/worker/worker.go index 58c376c..62bb53b 100644 --- a/internal/worker/worker.go +++ b/internal/worker/worker.go @@ -18,19 +18,17 @@ import ( "forgequeue/internal/model" ) -// Worker registry keys. A worker pool publishes its identity here on Start -// and refreshes it on a heartbeat, so the dashboard can show live workers -// without the API and the worker sharing a process. +// worker registry keys. a pool publishes its identity on Start and keeps it +// fresh with a heartbeat so the dashboard can show live workers. const ( - // workerRegistrySet holds the IDs of every worker that has registered. + // ids of every worker that has registered. workerRegistrySet = "forgequeue:workers" - // workerMetaPrefix builds the per-worker keys: :meta and - // :leases. + // builds the per-worker keys: :meta and :leases. workerMetaPrefix = "worker:" - // workerMetaTTL bounds how long a registration outlives its last - // heartbeat. A crashed worker simply disappears once the TTL lapses. + // how long a registration outlives its last heartbeat. a crashed worker + // disappears once the TTL lapses. workerMetaTTL = 30 * time.Second - // workerHeartbeatEvery is how often a live worker refreshes its keys. + // how often a live worker refreshes its keys. workerHeartbeatEvery = 10 * time.Second ) @@ -46,23 +44,22 @@ type workerMeta struct { } type WorkerPool struct { - // The shared Redis client used for all queue operations. + // shared Redis client for all queue operations. redisClient *redis.Client - // Number of worker goroutines that process tasks in parallel. + // number of worker goroutines processing tasks in parallel. concurrencyLimit int // WaitGroup keeps track of how many workers are still running. // Stop() uses it to wait until every worker has finished. wg sync.WaitGroup - // Closing this channel tells all workers that the pool is shutting down. + // closing this channel tells every worker the pool is shutting down. quit chan struct{} - // Shared context for Redis operations. - // Stop() cancels this context so a worker waiting inside Redis - // doesn't have to wait for the full BRPop timeout. + // shared context for Redis operations. Stop() cancels it so a worker + // stuck inside Redis doesn't wait out the full BRPop timeout. ctx context.Context - // Function used to cancel the shared context during shutdown. + // cancels the shared context during shutdown. cancel context.CancelFunc //handler is jsut an arbitary value @@ -71,112 +68,94 @@ type WorkerPool struct { // time to wait for a lease before giving up on a task. leaseDuration time.Duration - // starting delay before the first retry after a failure. - // every retry doubles it (1s, 2s, 4s, ...) until backoffCap. + // delay before the first retry. doubles every attempt (1s, 2s, 4s...) + // up to backoffCap. backoffBase time.Duration - // ceiling for the exponential backoff delay so a flaky task - // can't schedule itself minutes into the future. + // ceiling on the backoff delay so a flaky task can't retry minutes out. backoffCap time.Duration - // how many tasks one worker claims per round trip. 1 keeps the old - // single-task behaviour; larger batches trade round trips for latency - // (tasks claimed together wait for each other). Opt-in via WithBatchSize. + // tasks one worker claims per round trip. 1 is the old single-task + // behaviour; bigger batches cut round trips but tasks wait on each other. batchSize int - // The durable journal (Postgres). It mirrors task lifecycles so Redis - // state can be rebuilt after a crash. Nop by default: without a store - // the pool behaves exactly as it always did. + // durable journal (Postgres). mirrors task lifecycles so Redis state can + // be rebuilt after a crash. Nop by default. store durable.Store - // Observability. Never nil: the default pool gets no-op instruments so - // instrumentation call sites stay free of nil checks. + // observability. never nil, the default pool gets no-op instruments so + // call sites skip nil checks. metrics *metrics.Metrics - // Identity of this pool instance. Published to Redis on Start so the - // dashboard can attribute leases and heartbeats to a specific worker. + // this pool's identity. published to Redis on Start so the dashboard can + // attribute leases and heartbeats to a worker. id string hostname string startedAt time.Time version string - // bgCtx/bgCancel govern the background infrastructure loops (queue-depth - // sampling, the durable reconciler) that stop only on context - // cancellation. They are deliberately separate from ctx so Stop can drain - // in-flight handlers (which run under ctx) before tearing the background - // down. + // govern the background loops (depth sampling, the reconciler) that stop + // only on context cancellation. separate from ctx so Stop can drain + // in-flight handlers before tearing these down. bgCtx context.Context bgCancel context.CancelFunc - // drainTimeout bounds how long Stop waits for in-flight handlers to - // finish before cancelling them. + // how long Stop waits for in-flight handlers before cancelling them. drainTimeout time.Duration - // recoveryInterval is how often the recovery loop looks for expired - // leases. Defaults to one second; tests shrink it to exercise expiry - // without waiting. + // how often the recovery loop looks for expired leases. tests shrink it + // to exercise expiry without waiting. recoveryInterval time.Duration } -// taskRuntime holds the per-task lifecycle of one claimed task: the context -// its handler runs under and the cancellation hooks for the handler and its -// heartbeat. +// taskRuntime holds the lifecycle of one claimed task: the context its +// handler runs under, plus the cancels for the handler and its heartbeat. // -// A batch is handled sequentially, so the runtime for every task in it is -// created and its heartbeat started the moment the batch is claimed. Without -// that, tasks waiting behind the one being executed would have no heartbeat -// and their leases would expire mid-batch, letting recovery hand them to -// another worker while this one was still running them. +// a batch runs sequentially, so every task starts its heartbeat the moment +// the batch is claimed. a task waiting behind the running one would otherwise +// have no heartbeat, its lease would expire, and recovery would hand it to +// another worker while this one still had it. type taskRuntime struct { ctx context.Context cancel context.CancelFunc stopBeat context.CancelFunc } -// TaskLease is the proof of ownership a worker holds over a claimed task. +// TaskLease is a worker's proof of ownership over a claimed task. // -// TaskID identifies the task. LeaseID is a unique token generated at claim -// time; Redis stores the pair in the leases hash so later operations -// (heartbeat, ACK) can verify this worker still owns the task. +// LeaseID is a token made at claim time, stored with the taskID in the leases +// hash so heartbeat and ACK can check this worker still owns it. type TaskLease struct { TaskID string LeaseID string } -// TaskClaim is what a worker receives from claimBatch(): the ownership lease -// plus the task's JSON body. The claim Lua script reads the body in the same -// atomic call, so fetching it does not cost a second round trip to Redis. +// TaskClaim is what claimBatch() returns: the lease plus the task's JSON +// body, read in the same atomic script call so there is no second round trip. type TaskClaim struct { TaskLease Body []byte } func NewWorkerPool(addr string, concurrencyLimit int) (*WorkerPool, error) { - // Create the Redis client used by all workers. client := redis.NewClient(&redis.Options{ Addr: addr, - // Give Redis enough connections for the workers - // and their concurrent Redis operations. + // enough connections for the workers and their concurrent Redis ops. PoolSize: concurrencyLimit * 2, }) - // Create one context for the whole worker pool. - // - // This context will be passed to Redis operations. - // When Stop() calls cancel(), operations using this context - // can be interrupted. + // one context for the whole pool, passed to every Redis operation so + // Stop() can interrupt them. ctx, cancel := context.WithCancel(context.Background()) - // Separate context for the background infrastructure loops, so they can - // be stopped at the end of a graceful drain without cancelling the - // handlers still finishing under ctx. + // separate context for the background loops, so a graceful drain can stop + // them without cancelling handlers still finishing under ctx. bgCtx, bgCancel := context.WithCancel(context.Background()) - // Check that Redis is reachable before returning the pool. + // make sure Redis is reachable before returning the pool. if err := client.Ping(context.Background()).Err(); err != nil { - // If Redis isn't reachable, cancel the contexts we just created - // because the worker pool won't be started. + // pool won't start, so drop the contexts we just made. cancel() bgCancel() return nil, err @@ -193,14 +172,9 @@ func NewWorkerPool(addr string, concurrencyLimit int) (*WorkerPool, error) { startedAt: time.Now(), version: "v0.9.1", - // sync.WaitGroup starts with a counter of 0. - // Start() will add one for every worker it creates. wg: sync.WaitGroup{}, - // Channel used as a shutdown signal. - quit: make(chan struct{}), - - // Shared context and its cancellation function. + quit: make(chan struct{}), ctx: ctx, cancel: cancel, bgCtx: bgCtx, @@ -216,9 +190,8 @@ func NewWorkerPool(addr string, concurrencyLimit int) (*WorkerPool, error) { }, nil } -// WithMetrics attaches a telemetry sink to the pool. Call it before Start(). -// Passing nil is a no-op, so callers can forward an optional *metrics.Metrics -// without branching. +// WithMetrics attaches a telemetry sink. call before Start(). passing nil is +// a no-op. func (wp *WorkerPool) WithMetrics(m *metrics.Metrics) *WorkerPool { if m == nil { return wp @@ -227,27 +200,22 @@ func (wp *WorkerPool) WithMetrics(m *metrics.Metrics) *WorkerPool { return wp } -// workerLoop is the work performed by one worker. +// workerLoop runs one worker. // -// workerID identifies which worker is running this loop. -// For example, Start() can create worker 0, worker 1, worker 2, etc. +// workerID just labels which worker this is (0, 1, 2...). // -// One iteration handles a whole batch: claim up to batchSize tasks in a -// single round trip, run each task's handler sequentially, then ACK all the -// successes in a single round trip. With batchSize = 1 this degenerates -// exactly to the original claim-one/ack-one loop. +// one iteration handles a whole batch: claim up to batchSize tasks, run +// their handlers one at a time, then ACK the successes together. func (wp *WorkerPool) workerLoop(workerID int) { for { select { - // If the quit channel is closed, the worker stops. case <-wp.quit: return default: claims, err := wp.claimBatch() - // claimBatch returns nothing when the queue is empty. - // Sleep briefly so workers don't spin hot on an idle queue. + // empty queue: sleep a little so idle workers don't spin. if len(claims) == 0 { time.Sleep(1 * time.Millisecond) continue @@ -264,8 +232,8 @@ func (wp *WorkerPool) workerLoop(workerID int) { var ackIDs []string var ackLeases []string - // Record every claimed task against this worker so the - // dashboard can show what it is processing. + // record every claimed task so the dashboard can show what this + // worker is processing. claimedIDs := make([]string, 0, len(claims)) for _, claim := range claims { if claim.TaskID != "" { @@ -274,11 +242,10 @@ func (wp *WorkerPool) workerLoop(workerID int) { } wp.markLeases(claimedIDs) - // Start a heartbeat for every claimed task up front. The batch - // is handled sequentially, so a task waiting behind the one - // being executed would otherwise have no heartbeat and its lease - // would expire mid-batch - exactly the window in which recovery - // hands the task to a second worker and it runs twice. + // start a heartbeat for every claimed task up front. the batch + // runs one at a time, so a task waiting behind the running one + // would otherwise have no heartbeat and its lease would expire, + // letting recovery hand it to a second worker. runtimes := make(map[string]*taskRuntime, len(claims)) for _, claim := range claims { if claim.TaskID == "" { @@ -295,8 +262,7 @@ func (wp *WorkerPool) workerLoop(workerID int) { } // stopTask ends the heartbeat and execution context for one task. - // Idempotent, so it is safe on every exit path and again in the - // cleanup sweep below. + // idempotent, so it is safe on every exit path. stopTask := func(id string) { if rt, ok := runtimes[id]; ok { rt.stopBeat() @@ -306,14 +272,14 @@ func (wp *WorkerPool) workerLoop(workerID int) { for _, claim := range claims { - // Defensive guard: a claimed task must always have an ID. + // a claimed task should always have an ID. if claim.TaskID == "" { time.Sleep(100 * time.Millisecond) continue } - // The task body was already read by the claim script. - // An empty body means the task data was missing. + // the claim script already read the body. empty means the + // task data was missing. if len(claim.Body) == 0 { log.Printf( "worker %d: failed to get task %s: no task body", @@ -326,10 +292,7 @@ func (wp *WorkerPool) workerLoop(workerID int) { var task model.TaskMetaData - // Convert the JSON stored in Redis into TaskMetaData. - // - // &task gives Unmarshal the address of task so that - // it can fill the struct. + // decode the JSON from Redis into task. if err := json.Unmarshal(claim.Body, &task); err != nil { log.Printf( "worker %d: failed to parse task %s: %v", @@ -341,11 +304,8 @@ func (wp *WorkerPool) workerLoop(workerID int) { continue } - // Look up the handler for this task's type. - // - // ok is true when a handler was registered for task.TaskType - // (e.g. "email"). If none is registered, the task can't be - // processed, so skip it and let it expire/requeue. + // look up the handler for this task type. nothing registered + // means it can't run, so skip and let the lease expire. handler, ok := wp.handlers[task.TaskType] if !ok { log.Printf( @@ -357,9 +317,8 @@ func (wp *WorkerPool) workerLoop(workerID int) { continue } - // The task is now owned by this worker, so the journal should - // know it is being executed. Best-effort: if Postgres is down - // the task still runs, the record is just stale. + // this worker owns the task now, so tell the journal it is + // running. best effort: if Postgres is down the task still runs. if err := wp.store.Processing(wp.ctx, claim.TaskID); err != nil { log.Printf( "journal: failed to mark task %s as processing: %v", @@ -370,18 +329,16 @@ func (wp *WorkerPool) workerLoop(workerID int) { handlerStart := time.Now() - // The handler runs under the task's pre-created context, whose - // heartbeat is already renewing the lease in the background. - // The heartbeat is deliberately left running after the handler - // returns: the ACK is deferred to the single batched call at - // the end of the loop, so stopping it here would let an - // early-finished task's lease expire while it waits its turn. + // the handler runs under the task's own context, whose + // heartbeat is already renewing the lease. the heartbeat keeps + // running after the handler returns, because the ACK waits for + // the batched call at the end of the loop. stopping it here + // would expire an early-finished task while it waits its turn. rt := runtimes[claim.TaskID] handlerErr := handler.Handle(rt.ctx, &task) - // How long the handler actually ran. Recorded for both - // outcomes: a slow success and a slow failure are very - // different signals when you are looking at p99. + // how long the handler ran. recorded for successes too, since + // a slow success and a slow failure look very different at p99. wp.metrics.TaskDuration.Record( wp.ctx, time.Since(handlerStart).Seconds(), @@ -400,10 +357,9 @@ func (wp *WorkerPool) workerLoop(workerID int) { handlerErr, ) - // A handler error counts as a failed attempt. Release the - // lease right away (fenced, like an ACK) instead of waiting - // for it to expire, so the retry happens on the backoff - // delay instead of whenever the lease runs out. + // a handler error is a failed attempt. release the lease now + // (fenced, like an ACK) instead of letting it expire, so the + // retry lands on the backoff delay. status, rerr := wp.requeueOrDead(task.ID, claim.LeaseID) if rerr != nil { log.Printf( @@ -432,22 +388,16 @@ func (wp *WorkerPool) workerLoop(workerID int) { wp.redisClient.Expire(wp.ctx, "metrics:failed", 24*time.Hour) } - // The lease was released by requeueOrDead, so there is - // nothing left for the heartbeat to keep alive. + // requeueOrDead released the lease, nothing left to keep alive. stopTask(task.ID) } - // Everything is decided for this batch now, so ACK the successes - // in one fenced script. Only after the ACK lands do we stop the - // remaining heartbeats: an early-finished success has to keep - // heartbeating until its ACK is committed, otherwise its lease - // expires while it waits for the rest of the batch and recovery - // hands the task to a second worker. + // ACK the successes in one fenced script. only after it lands do + // we stop the other heartbeats, otherwise an early-finished task + // would expire while it waits for the rest of the batch. // - // The script fences per task, so a batch can be partially ACKed: - // if one task's lease really did expire, recovery may already - // have handed it to another worker and rotated its lease. That - // task is skipped, the rest are ACKed, and the returned ids are + // the script fences per task, so a batch can be partially ACKed: + // tasks that lost their lease are skipped and the returned ids are // the ones that actually completed. if len(ackIDs) > 0 { ackedIDs, ackErr := wp.ackBatch(ackIDs, ackLeases) @@ -460,9 +410,8 @@ func (wp *WorkerPool) workerLoop(workerID int) { ) } else { if len(ackedIDs) < len(ackIDs) { - // Worth a warning: this is the at-least-once duplicate - // path. A silent overcount here would look like healthy - // throughput while tasks were actually running twice. + // worth a warning: this is the at-least-once duplicate + // path and would otherwise look like healthy throughput. log.Printf( "worker %d: only %d of %d task(s) ACKed, %d lost their lease "+ "and may run again elsewhere", @@ -470,15 +419,11 @@ func (wp *WorkerPool) workerLoop(workerID int) { ) } - // ACKed means done. Counted from the script's return value - // rather than len(ackIDs), so a fenced task is not reported - // as processed while it is still outstanding elsewhere. + // counted from the script's return value, not len(ackIDs), + // so a fenced task isn't reported as processed. wp.metrics.TasksProcessed.Add(wp.ctx, int64(len(ackedIDs))) - // ACKed means done: record it so the journal stops tracking - // it. Only the tasks the script actually ACKed are counted, - // so a task that lost its lease is not double-counted as - // completed. + // record the ACKed tasks so the journal stops tracking them. now := time.Now().Unix() for _, id := range ackedIDs { if err := wp.store.Succeeded(wp.ctx, id); err != nil { @@ -494,9 +439,8 @@ func (wp *WorkerPool) workerLoop(workerID int) { } } - // Every task in this batch has reached an outcome by now (ACKed, - // retried or dead-lettered), so stop any heartbeat still running - // and drop them from this worker's lease set. + // every task has an outcome now (ACKed, retried or dead), so stop + // any heartbeat left and drop them from this worker's lease set. for id := range runtimes { stopTask(id) } @@ -505,17 +449,14 @@ func (wp *WorkerPool) workerLoop(workerID int) { } } -// WithStore attaches the durable journal to the pool and makes Start() run -// the reconciler alongside the workers. Call it before Start(). +// WithStore attaches the durable journal. call before Start(). func (wp *WorkerPool) WithStore(s durable.Store) *WorkerPool { wp.store = s return wp } -// WithBatchSize configures how many tasks each worker claims per round trip. -// 1 (the default) preserves the old single-task behaviour. Values above 1 -// claim a whole batch atomically and collect ACKs, cutting Redis round trips -// well below one per task. +// WithBatchSize sets how many tasks a worker claims per round trip. 1 is the +// default single-task behaviour. func (wp *WorkerPool) WithBatchSize(n int) *WorkerPool { if n < 1 { n = 1 @@ -524,8 +465,8 @@ func (wp *WorkerPool) WithBatchSize(n int) *WorkerPool { return wp } -// WithDrainTimeout bounds how long Stop waits for in-flight handlers to finish -// before cancelling them. Call it before Start(). +// WithDrainTimeout bounds how long Stop waits for in-flight handlers before +// cancelling them. call before Start(). func (wp *WorkerPool) WithDrainTimeout(d time.Duration) *WorkerPool { if d > 0 { wp.drainTimeout = d @@ -533,12 +474,10 @@ func (wp *WorkerPool) WithDrainTimeout(d time.Duration) *WorkerPool { return wp } -// Start creates and starts the workers. -// -// The number of workers is controlled by concurrencyLimit. +// Start creates and starts the workers, concurrencyLimit of them. func (wp *WorkerPool) Start() { - // Publish this pool's identity and keep it fresh with a heartbeat, so - // the dashboard can see the worker without sharing its process. + // publish this pool's identity and keep it fresh with a heartbeat so the + // dashboard can see the worker. wp.wg.Add(1) go func() { defer wp.wg.Done() @@ -547,45 +486,37 @@ func (wp *WorkerPool) Start() { for i := 0; i < wp.concurrencyLimit; i++ { - // Tell the WaitGroup that another worker is about to start. wp.wg.Add(1) - // Start the worker as a goroutine. - // - // i is passed into the anonymous function so each worker - // receives its own worker ID. + // i is captured so each worker gets its own ID. go func(i int) { - // Done() is called automatically when this worker exits, - // even if workerLoop returns from the quit signal. + // Done() runs even if workerLoop returns via the quit signal. defer wp.wg.Done() - // Run the actual worker loop. wp.workerLoop(i) }(i) } - // The recovery loop runs separately from the workers: it has its own - // goroutine so it keeps requeueing expired tasks even while every - // worker is busy executing. + // recovery runs in its own goroutine so expired tasks keep getting + // requeued even while every worker is busy. wp.wg.Add(1) go func() { defer wp.wg.Done() wp.recoveryLoop() }() - // The promotion loop is separate too: it watches the scheduled set and - // moves tasks whose delay has passed into the ready queue. + // promotion is separate too: it moves due tasks from the scheduled set + // into the ready queue. wp.wg.Add(1) go func() { defer wp.wg.Done() wp.promotionLoop() }() - // The reconciler heals the journal's write gap: it re-materializes into - // Redis any task the journal knows about that Redis is missing. It only - // exists when a durable store is attached, otherwise it would be a - // pointless wire to an empty table every second. + // the reconciler heals the journal's write gap: it re-materializes into + // Redis any task the journal knows about but Redis doesn't. only runs + // when a durable store is attached. if _, isNop := wp.store.(durable.Nop); !isNop { wp.wg.Add(1) go func() { @@ -594,9 +525,9 @@ func (wp *WorkerPool) Start() { }() } - // Queue depth is sampled from Redis on a timer. Doing it in-process - // would need a matching increment and decrement for every transition, - // and one missed transition would leave the gauge permanently wrong. + // sample queue depth from Redis on a timer. doing it in-process would + // need an inc/dec for every transition, and one miss leaves the gauge + // wrong for good. if wp.metrics.Ready { wp.wg.Add(1) go func() { @@ -613,29 +544,25 @@ func (wp *WorkerPool) Start() { } } -// Stop gracefully shuts down the worker pool. -// -// It works in three phases so in-flight work is drained rather than aborted: +// Stop gracefully shuts down the worker pool in three phases so in-flight +// work drains instead of being aborted. // -// 1. close quit and stop the background infrastructure. Workers stop claiming -// new tasks at the top of their loop, but a worker already inside a batch -// keeps running it. -// 2. wait for every goroutine to return. Handlers run under ctx, which is -// still alive, so they finish normally instead of being cancelled. -// 3. cancel ctx and deregister. By now no handler is running, so cancelling -// is safe. +// 1. close quit and stop the background loops. workers stop claiming new +// tasks but finish a batch already running. +// 2. wait for every goroutine. handlers run under ctx, still alive, so they +// finish normally. +// 3. cancel ctx and deregister. // -// A slow or stuck handler must not hold shutdown open forever, so phase 2 is -// bounded by drainTimeout; past it the context is cancelled to force the -// handlers to unwind. +// phase 2 is bounded by drainTimeout so a stuck handler can't hold shutdown +// open forever. func (wp *WorkerPool) Stop() { - // Phase 1: stop accepting new claims and stop the loops that would - // otherwise run for the lifetime of the pool. + // phase 1: stop new claims and the loops that would otherwise run for the + // life of the pool. close(wp.quit) wp.bgCancel() - // Phase 2: wait for workers to finish the batch they are inside, while - // their contexts are still valid. + // phase 2: let workers finish the batch they are in while their contexts + // are still valid. done := make(chan struct{}) go func() { wp.wg.Wait() @@ -651,9 +578,8 @@ func (wp *WorkerPool) Stop() { ) } - // Phase 3: nothing should be executing now, so tear the context down and - // remove the registration so the dashboard reflects the shutdown. In the - // normal path the wait above already returned, so the pool is fully idle. + // phase 3: nothing should be running now, so cancel the context and + // deregister so the dashboard sees the shutdown. wp.cancel() wp.deregister() } @@ -667,31 +593,23 @@ func (wp *WorkerPool) RegisterHandler( wp.handlers[taskType] = handler } -// TaskHandler defines the interface for task handlers. type TaskHandler interface { //create a taskhandler type which needs to handle and have these Handle(ctx context.Context, task *model.TaskMetaData) error } -// claimBatch pops up to batchSize tasks off the immediate queue, claims -// ownership of them all, and reads their bodies — in one atomic Lua script. +// claimBatch pops up to batchSize tasks off the immediate queue, claims them +// all, and reads their bodies in one atomic Lua script, so a batch costs +// about two round trips instead of one per task. // -// Every task still costs a claim and an ACK, but they no longer cost one -// round trip each: a whole batch is claimed together and ACKed together, so -// the cost collapses to ~2 round trips per batch. The script returns the -// lease (ID + ownership token sent in ARGV) together with each stored JSON -// body, so the worker never sends a separate GET. -// -// Lease IDs are generated here in Go and passed in ARGV; the script assigns -// them to whatever it pops (ARGV[3+i] goes to the i-th popped task), and the -// caller pairs the returned bodies back to those IDs by index. +// lease IDs are made here in Go and passed in ARGV; the script hands them to +// whatever it pops and the caller pairs the returned bodies back by index. func (wp *WorkerPool) claimBatch() ([]TaskClaim, error) { claimStart := time.Now() - // The expiry is the deadline for the lease. If the worker doesn't - // ACK or heartbeat before this, recovery will requeue the task. - // Keep fractional seconds: truncating to whole seconds would silently - // shave up to a second off short leases, expiring them early. + // lease deadline. no ACK or heartbeat before this and recovery requeues + // the task. keep fractional seconds: truncating would shave up to a + // second off short leases and expire them early. leaseExpiry := float64(time.Now().UnixNano())/1e9 + wp.leaseDuration.Seconds() batch := wp.batchSize @@ -762,18 +680,15 @@ func (wp *WorkerPool) claimBatch() ([]TaskClaim, error) { return nil, err } - // The script returns an empty string when the queue was empty. + // empty string means the queue was empty. if result == "" { - // Deliberately not recorded. An idle worker polls this script every - // millisecond, so those round trips outnumber real claims by orders - // of magnitude and complete in microseconds. Averaging them into the - // same histogram drags every percentile toward zero and hides the - // claim latency that actually matters. The poll rate is still - // visible as Redis command traffic in Prometheus. + // not timed on purpose. an idle worker polls this every millisecond, + // so averaging those in would drag every percentile toward zero and + // hide the claim latency that matters. return nil, nil } - // A claim that returned work is worth timing: it is the queue's floor on + // a claim that returned work is worth timing: it is the queue's floor on // per-task overhead. wp.metrics.ClaimLatency.Record( wp.ctx, @@ -810,9 +725,8 @@ func (wp *WorkerPool) claimBatch() ([]TaskClaim, error) { // findExpiredTasks returns the task IDs whose lease has expired. // -// The inflight sorted set stores taskID -> lease expiry. Asking for -// everything between "-inf" and the current Unix time ("now") returns -// exactly the tasks whose deadline has passed and are ready for requeue. +// the inflight set maps taskID -> lease expiry, so everything scored up to +// now is expired and ready for requeue. func (wp *WorkerPool) findExpiredTasks() ([]string, error) { now := float64(time.Now().UnixNano()) / 1e9 return wp.redisClient.ZRangeArgs( @@ -829,32 +743,17 @@ func (wp *WorkerPool) findExpiredTasks() ([]string, error) { // requeueOrDead decides what happens to a failed or expired task. // -// Two different callers share this one script: -// -// - the worker, right after a handler returns an error. It passes the -// leaseID so the script fences like an ACK: if the worker no longer owns -// the task, the call is rejected and the task is left alone. -// - the recovery loop, for a task whose lease expired while nobody was -// processing it. It passes "" and instead relies on re-checking the -// inflight score against Redis's own clock, exactly like the old -// removeExpiredTasks did. +// two callers share this script: the worker after a handler error passes the +// leaseID so it fences like an ACK (a task it no longer owns is rejected); +// the recovery loop passes "" and lets the script re-check the inflight score +// against Redis's clock. // -// Whichever caller it is, the script then: +// either way the script loads the body, bumps "attempts", rewrites the body, +// then schedules a retry with a backoff score (now + base * 2^(attempts-1), +// capped) or pushes it to the dead-letter queue once the budget is gone. +// max_retries = 0 means retry forever. // -// 1. loads the task body, increments the "attempts" counter, and rewrites -// the body with the new counter either way. -// 2. decides between two outcomes: -// - "scheduled": still below its retry budget. The task goes into the -// scheduled set with a backoff score (now + base * 2^(attempts-1), -// capped). -// - "dead": the budget is exhausted. The full body (with attempts) is -// pushed onto the dead-letter queue so a human can inspect it. -// -// A task with max_retries = 0 has an unlimited budget and always retries. -// -// Return values: "fenced" (worker lost the lease), "gone" (nothing in -// inflight), "renewed" (lease extended before us), "no-body" (dropped), -// "dead" (moved to the dead-letter queue), "scheduled" (retry queued). +// returns: "fenced", "gone", "renewed", "no-body", "dead", "scheduled". // // KEYS[1] -> queue:tasks:inflight // KEYS[2] -> queue:tasks:leases @@ -960,9 +859,9 @@ func (wp *WorkerPool) requeueOrDead(taskID string, leaseID string) (string, erro return result.(string), nil } -// backoffDelay mirrors the delay computed inside the requeueOrDead script: -// base * 2^(attempts-1), capped. The two halves must keep the same math, or -// the journal would record a different schedule than Redis actually applied. +// backoffDelay mirrors the delay the requeueOrDead script computes: +// base * 2^(attempts-1), capped. keep the two in sync or the journal records +// a different schedule than Redis applied. func backoffDelay(attempts int32, base, cap time.Duration) time.Duration { delay := base for i := int32(2); i <= attempts; i++ { @@ -974,10 +873,9 @@ func backoffDelay(attempts int32, base, cap time.Duration) time.Duration { return delay } -// reportRetry tells the journal a task was put into backoff. Redis already -// holds the task (the requeue script scheduled it), so the journal only -// records what Redis decided - the counter and the new schedule. Reading the -// body back is one extra round trip, but only on the failure path. +// reportRetry tells the journal a task went into backoff. Redis already +// scheduled it, so the journal only records the counter and the new due time. +// reading the body back is one extra round trip, only on the failure path. func (wp *WorkerPool) reportRetry(taskID string) { t, ok := wp.readTaskForJournal(taskID) if !ok { @@ -989,8 +887,8 @@ func (wp *WorkerPool) reportRetry(taskID string) { } } -// reportDead tells the journal a task exhausted its retry budget. The body -// (with its final attempt count) is already in the dead-letter queue. +// reportDead tells the journal a task ran out of retries. the body with its +// final attempt count is already in the dead-letter queue. func (wp *WorkerPool) reportDead(taskID string) { t, ok := wp.readTaskForJournal(taskID) if !ok { @@ -1001,9 +899,8 @@ func (wp *WorkerPool) reportDead(taskID string) { } } -// readTaskForJournal fetches the current body of a task so its post-script -// attempt count can be reported. The body is undefined when the queue has no -// record of it, which is fine - there is nothing useful to report then. +// readTaskForJournal fetches a task body so its post-script attempt count can +// be reported. missing means nothing to report. func (wp *WorkerPool) readTaskForJournal(taskID string) (model.TaskMetaData, bool) { body, err := wp.redisClient.Get(wp.ctx, "task:"+taskID).Result() if err != nil { @@ -1018,15 +915,9 @@ func (wp *WorkerPool) readTaskForJournal(taskID string) (model.TaskMetaData, boo return t, true } -// recoveryLoop watches for expired leases and handles them. -// -// Every second it asks Redis which in-flight tasks have expired, then hands -// each one to requeueOrDead, which either schedules a backoff retry or dead- -// letters it once the retry budget is gone. It runs in its own goroutine and -// stops when the pool shuts down. +// recoveryLoop watches for expired leases. on every tick it asks Redis which +// in-flight tasks expired and hands each to requeueOrDead. func (wp *WorkerPool) recoveryLoop() { - // Re-check for expired tasks on the configured interval (one second by - // default). ticker := time.NewTicker(wp.recoveryInterval) defer ticker.Stop() @@ -1059,9 +950,8 @@ func (wp *WorkerPool) recoveryLoop() { case "no-body": log.Printf("recovery: task %s has no body, dropped", taskID) } - // "renewed", "gone" and "scheduled" are the quiet cases: - // the lease was extended, someone else handled it, or it - // was scheduled for another attempt. + // "renewed", "gone" and "scheduled" are quiet: the lease was + // extended, someone else handled it, or a retry was queued. } case <-wp.quit: @@ -1070,10 +960,10 @@ func (wp *WorkerPool) recoveryLoop() { } } -// findDueTasks returns task IDs in the scheduled set whose delay has passed. +// findDueTasks returns scheduled task IDs whose delay has passed. // -// Being "due" as reported here is only a candidate: promoteTask re-checks the -// score against Redis's own clock atomically, just like recovery does. +// "due" here is only a candidate: promoteTask re-checks the score against +// Redis's own clock. func (wp *WorkerPool) findDueTasks() ([]string, error) { now := float64(time.Now().UnixNano()) / 1e9 return wp.redisClient.ZRangeArgs( @@ -1089,10 +979,9 @@ func (wp *WorkerPool) findDueTasks() ([]string, error) { // promoteTask moves one task from the scheduled set to the immediate queue. // -// The task body lives at task: (both the gateway's delayed path and -// requeueOrDead write it there), so promotion is just a list move. The score -// is re-checked against TIME inside the script so a task is never promoted -// early and never promoted twice. +// the body already lives at task:, so this is just a list move. the score +// is re-checked against TIME in the script so a task is never promoted early +// or twice. // // KEYS[1] -> queue:tasks:scheduled // KEYS[2] -> queue:tasks:immediate @@ -1129,11 +1018,9 @@ func (wp *WorkerPool) promoteTask(taskID string) (bool, error) { return result.(int64) == 1, nil } -// promotionLoop watches the scheduled set and moves due tasks to the ready -// queue. Delayed enqueues and backoff retries both land here. It runs in its -// own goroutine and stops when the pool shuts down. +// promotionLoop moves due tasks from the scheduled set to the ready queue. +// delayed enqueues and backoff retries both land here. func (wp *WorkerPool) promotionLoop() { - // Check for due tasks once per second. ticker := time.NewTicker(1 * time.Second) defer ticker.Stop() @@ -1159,15 +1046,13 @@ func (wp *WorkerPool) promotionLoop() { // ackBatch tells Redis that a list of tasks finished successfully. // -// Every task is fenced individually: it is only removed if the presented -// leaseID still matches the owner in the leases hash, so an ACK from a worker -// whose lease was overwritten (e.g. the task was requeued after expiry) is -// rejected without touching anything else. One round trip for the whole list. +// each task is fenced on its own: it is only removed if the presented leaseID +// still matches the leases hash, so a stale ACK is rejected. one round trip +// for the whole list. // -// A successfully ACKed task has no further use for its body, so the script -// deletes task: in the same atomic call. That is what keeps the task -// store from growing without bound. The returned slice holds exactly the IDs -// that were ACKed, so callers never count a fenced task as completed. +// an ACKed task has no use for its body, so task: is deleted in the same +// atomic call, keeping the task store from growing without bound. the returned +// slice is exactly the IDs that were ACKed. // // KEYS[1] -> queue:tasks:inflight (sorted set: taskID -> expiry) // KEYS[2] -> queue:tasks:leases (hash: taskID -> leaseID) @@ -1229,18 +1114,15 @@ func (wp *WorkerPool) ackBatch(taskIDs []string, leaseIDs []string) ([]string, e return acked, nil } -// errLeaseLost is returned by heartbeat when Redis answered but the lease -// token no longer matches. A distinct sentinel (rather than a generic -// error) so the caller can tell "somebody else owns this task now" apart -// from "Redis was unreachable", which is a transport problem rather than a -// lost lease and should not be counted as one. +// errLeaseLost means Redis answered but the lease token no longer matches. a +// sentinel so the caller can tell "someone else owns this task now" apart +// from "Redis was unreachable", which shouldn't count as a lost lease. var errLeaseLost = errors.New("heartbeat rejected: lease no longer held") -// heartbeat renews the lease for a task that is still being processed. +// heartbeat renews the lease for a task still being processed. // -// It checks ownership first (HGET) and, if the worker still owns the task, -// pushes the expiry forward using Redis's own clock rather than the -// worker's clock. +// it checks ownership first (HGET) and, if we still own the task, pushes the +// expiry forward using Redis's clock, not the worker's. // // KEYS[1] -> queue:tasks:leases (hash: taskID -> leaseID) // KEYS[2] -> queue:tasks:inflight (sorted set: taskID -> expiry) @@ -1277,8 +1159,8 @@ func (wp *WorkerPool) heartbeat(taskID string, leaseID string) error { strconv.FormatFloat(wp.leaseDuration.Seconds(), 'f', -1, 64), ).Result() - // Recorded on both paths: a slow heartbeat is what causes lease - // expiry in the first place, so the failures matter too. + // recorded on both paths: a slow heartbeat is what causes lease expiry, + // so the failures matter too. wp.metrics.HeartbeatLatency.Record( wp.ctx, time.Since(beatStart).Seconds(), @@ -1288,8 +1170,7 @@ func (wp *WorkerPool) heartbeat(taskID string, leaseID string) error { return err } - // return 0 means the worker no longer holds the lease, - // so the heartbeat is rejected. + // 0 means we no longer hold the lease, so the heartbeat is rejected. if result.(int64) == 0 { return errLeaseLost } @@ -1297,16 +1178,12 @@ func (wp *WorkerPool) heartbeat(taskID string, leaseID string) error { return nil } -// heartbeatLoop keeps a task's lease alive while it is being handled. -// -// heartbeat() extends the lease once; this loop calls it repeatedly at a -// fraction of the lease duration (leaseDuration / 3) so a long-running task -// never expires while it's still making progress. +// heartbeatLoop keeps a task's lease alive while it is handled. // -// It stops either when the per-task context is cancelled (the handler -// finished) or when a heartbeat is rejected (the lease was lost). When the -// lease is lost, it also cancels the task context so the running handler -// is told to stop early. +// it calls heartbeat() every leaseDuration/3 so a long task never expires. +// it stops when the task context is cancelled (the handler finished) or a +// heartbeat is rejected. on a rejected lease it also cancels the task +// context so the handler stops early. func (wp *WorkerPool) heartbeatLoop( ctx context.Context, taskID string, @@ -1320,10 +1197,9 @@ func (wp *WorkerPool) heartbeatLoop( select { case <-ticker.C: if err := wp.heartbeat(taskID, leaseID); err != nil { - // A heartbeat that raced with the batch's own ACK (the lease - // was removed the instant we stopped) is not a real loss: the - // stop already happened. Only report it when the loop is still - // meant to be running. + // a heartbeat that raced with the batch's own ACK (lease gone + // the instant we stopped) is not a real loss. only report when + // the loop is still meant to be running. if ctx.Err() != nil { return } @@ -1334,10 +1210,9 @@ func (wp *WorkerPool) heartbeatLoop( err, ) - // Only a rejected lease counts as a lease loss. A - // transport error or a shutdown-induced cancellation - // is a different failure and would otherwise inflate - // the metric during a Redis blip or a rolling restart. + // only a rejected lease is a lease loss. a transport error or + // shutdown cancellation is different and shouldn't inflate the + // metric during a Redis blip. if errors.Is(err, errLeaseLost) { wp.metrics.LeaseLosses.Add(wp.ctx, 1) } diff --git a/web/src/api/endpoints.ts b/web/src/api/endpoints.ts index db71fad..24f3e39 100644 --- a/web/src/api/endpoints.ts +++ b/web/src/api/endpoints.ts @@ -12,8 +12,16 @@ import { request } from "./client"; export type MetricsRange = "5m" | "15m" | "1h" | "6h" | "24h" | "7d"; +export interface AuthConfig { + password: boolean; + oidc: boolean; + provider?: string; +} + export interface Api { + authConfig(): Promise; login(username: string, password: string): Promise; + exchangeOidc(code: string): Promise; me(): Promise; stats(): Promise; queues(): Promise; @@ -41,7 +49,9 @@ export interface Api { const e = encodeURIComponent; const httpApi: Api = { + authConfig: () => request("GET", "/auth/config"), login: (username, password) => request("POST", "/auth/login", { username, password }), + exchangeOidc: (code) => request("POST", "/auth/oidc/exchange", { code }), me: () => request("GET", "/auth/me"), stats: () => request("GET", "/system/stats"), queues: () => request("GET", "/queues"), diff --git a/web/src/components/app-shell.tsx b/web/src/components/app-shell.tsx index 1f75a81..6587255 100644 --- a/web/src/components/app-shell.tsx +++ b/web/src/components/app-shell.tsx @@ -176,7 +176,7 @@ function UserFooter({ collapsed }: { collapsed: boolean }) {
{username}
-
+
{roles.join(", ")}
@@ -388,6 +388,9 @@ function Topbar({ onToggle, onSearch }: { onToggle: () => void; onSearch: () => function CommandPalette({ open, setOpen }: { open: boolean; setOpen: (o: boolean) => void }) { const navigate = useNavigate(); + const session = useSession(); + const perms = session?.user.permissions ?? []; + const nav = visibleNav(NAV, perms); const queues = useQueues().data ?? []; const workers = useWorkers().data ?? []; const dlq = useDlq().data ?? []; @@ -414,7 +417,7 @@ function CommandPalette({ open, setOpen }: { open: boolean; setOpen: (o: boolean No results - {[...NAV, ...SYS].map((n) => ( + {[...nav, ...SYS].map((n) => ( >(new Set()); const retry = useBulk(api.retryJob, "Retried"); const del = useBulk(api.deleteJob, "Deleted"); + const canWrite = useHasPermission("jobs:write"); useNow(5000); const ids = [...sel]; return ( <> setSel(new Set())}> - - - Delete - - } - destructive - title={`Delete ${sel.size} jobs?`} - description={`This permanently removes ${sel.size} jobs from Redis and PostgreSQL. This cannot be undone.`} - confirmLabel={`Delete ${sel.size} jobs`} - onConfirm={() => (del.mutate(ids), setSel(new Set()))} - /> + + + + Delete + + } + destructive + title={`Delete ${sel.size} jobs?`} + description={`This permanently removes ${sel.size} jobs from Redis and PostgreSQL. This cannot be undone.`} + confirmLabel={`Delete ${sel.size} jobs`} + onConfirm={() => (del.mutate(ids), setSel(new Set()))} + /> + — ), }, - { - key: "a", - header: "", - align: "right", - cell: (j) => - ["failed", "retrying", "cancelled"].includes(j.status) ? ( - - ) : null, - }, + ...(canWrite + ? [ + { + key: "a", + header: "", + align: "right" as const, + cell: (j: Job) => + ["failed", "retrying", "cancelled"].includes(j.status) ? ( + + ) : null, + }, + ] + : []), ]} /> @@ -160,6 +169,7 @@ export function DelayedTable({ showQueue?: boolean; }) { const now = useNow(1000); + const canWrite = useHasPermission("delayed:write"); const promote = useAction(api.promoteDelayed, (id) => `Promoted ${id.slice(0, 8)} to pending`); const cancel = useAction(api.cancelDelayed, (id) => `Cancelled ${id.slice(0, 8)}`); return ( @@ -203,49 +213,57 @@ export function DelayedTable({ }, }, { key: "st", header: "Status", cell: (d) => }, - { - key: "a", - header: "Actions", - align: "right", - cell: (d) => ( - - - Promote - - } - title="Promote job now?" - description={ - <> - Job {d.jobId} will be moved to{" "} - {d.queue} pending immediately, skipping its - scheduled delay. - - } - confirmLabel="Promote now" - onConfirm={() => promote.mutate(d.jobId)} - /> - - Cancel - - } - destructive - title="Cancel scheduled job?" - description={ - <> - Job {d.jobId} will be marked cancelled and - never run. - - } - confirmLabel="Cancel job" - onConfirm={() => cancel.mutate(d.jobId)} - /> - - ), - }, + ...(canWrite + ? [ + { + key: "a", + header: "Actions", + align: "right" as const, + cell: (d: DelayedJob) => ( + + + Promote + + } + title="Promote job now?" + description={ + <> + Job {d.jobId} will be moved to{" "} + {d.queue} pending immediately, skipping + its scheduled delay. + + } + confirmLabel="Promote now" + onConfirm={() => promote.mutate(d.jobId)} + /> + + Cancel + + } + destructive + title="Cancel scheduled job?" + description={ + <> + Job {d.jobId} will be marked cancelled + and never run. + + } + confirmLabel="Cancel job" + onConfirm={() => cancel.mutate(d.jobId)} + /> + + ), + }, + ] + : []), ]} /> ); diff --git a/web/src/routes/__root.tsx b/web/src/routes/__root.tsx index 820f33d..7eb79f2 100644 --- a/web/src/routes/__root.tsx +++ b/web/src/routes/__root.tsx @@ -128,7 +128,8 @@ function RedirectToLogin() { navigate({ to: "/login", replace: true }); }, [navigate]); return ( -
+
+ Redirecting to sign in…
); diff --git a/web/src/routes/dlq.tsx b/web/src/routes/dlq.tsx index 1e47258..dbde6b4 100644 --- a/web/src/routes/dlq.tsx +++ b/web/src/routes/dlq.tsx @@ -5,6 +5,7 @@ import { api } from "@/api/endpoints"; import { useAction, useBulk, useDlq } from "@/api/queries"; import { DataTable } from "@/components/data-table"; import { ConfirmDialog } from "@/components/confirm-dialog"; +import { Can } from "@/components/can"; import { BulkBar } from "@/components/job-tables"; import { JobId, Mono, PageHeader, Section, StatCard, TimeAgo } from "@/components/shared"; import { FilterSelect, SearchInput, Toolbar } from "@/components/toolbar"; @@ -58,24 +59,26 @@ function DlqPage() { title="Dead Letter Queue" description="Jobs that exhausted max attempts. Retrying moves them back to their original queue." actions={ - - Purge DLQ - - } - destructive - typeToConfirm="purge dlq" - title="Purge entire DLQ?" - description={`This action cannot be undone. Affected: ${data?.length ?? 0} jobs across ${queues.length} queues.`} - confirmLabel="Purge DLQ" - onConfirm={() => purge.mutate(undefined)} - /> + + + Purge DLQ + + } + destructive + typeToConfirm="purge dlq" + title="Purge entire DLQ?" + description={`This action cannot be undone. Affected: ${data?.length ?? 0} jobs across ${queues.length} queues.`} + confirmLabel="Purge DLQ" + onConfirm={() => purge.mutate(undefined)} + /> + } />
@@ -93,26 +96,30 @@ function DlqPage() { setSel(new Set())}> - - - Delete selected - - } - destructive - title={`Delete ${sel.size} DLQ jobs?`} - description={`Permanently discards ${sel.size} jobs. They will not be retried.`} - confirmLabel={`Delete ${sel.size}`} - onConfirm={() => (del.mutate(ids), setSel(new Set()))} - /> + + + + + + Delete selected + + } + destructive + title={`Delete ${sel.size} DLQ jobs?`} + description={`Permanently discards ${sel.size} jobs. They will not be retried.`} + confirmLabel={`Delete ${sel.size}`} + onConfirm={() => (del.mutate(ids), setSel(new Set()))} + /> + ( - - - - - - - } - destructive - title="Delete DLQ job?" - description={ - <> - Permanently discards {d.jobId}. - - } - confirmLabel="Delete" - onConfirm={() => del.mutate([d.jobId])} - /> - + + + + + + + + } + destructive + title="Delete DLQ job?" + description={ + <> + Permanently discards {d.jobId}. + + } + confirmLabel="Delete" + onConfirm={() => del.mutate([d.jobId])} + /> + + ), }, ]} diff --git a/web/src/routes/jobs.$jobId.tsx b/web/src/routes/jobs.$jobId.tsx index 581f271..4e70364 100644 --- a/web/src/routes/jobs.$jobId.tsx +++ b/web/src/routes/jobs.$jobId.tsx @@ -3,6 +3,7 @@ import { RotateCcw, Trash2, Undo2 } from "lucide-react"; import { api } from "@/api/endpoints"; import { useAction, useJob } from "@/api/queries"; import { ConfirmDialog } from "@/components/confirm-dialog"; +import { Can } from "@/components/can"; import { JsonViewer } from "@/components/json-viewer"; import { DataTable } from "@/components/data-table"; import { @@ -150,7 +151,7 @@ function JobDetail() { } actions={ - <> + @@ -195,7 +196,7 @@ function JobDetail() { }) } /> - + } />
diff --git a/web/src/routes/login.tsx b/web/src/routes/login.tsx index 1f580e0..773f065 100644 --- a/web/src/routes/login.tsx +++ b/web/src/routes/login.tsx @@ -1,11 +1,12 @@ import { useEffect, useState, type FormEvent } from "react"; import { createFileRoute, useNavigate } from "@tanstack/react-router"; -import { api } from "@/api/endpoints"; -import { ApiError } from "@/api/client"; +import { AlertCircle, ArrowRight, Eye, EyeOff, KeyRound, Loader2, Lock, User } from "lucide-react"; +import { api, type AuthConfig } from "@/api/endpoints"; +import { ApiError, getApiBase } from "@/api/client"; import { setSession, useSession } from "@/lib/auth"; +import { getPrefs } from "@/lib/prefs"; +import { cn } from "@/lib/utils"; import { Button } from "@/components/ui/button"; -import { Input } from "@/components/ui/input"; -import { Label } from "@/components/ui/label"; export const Route = createFileRoute("/login")({ ssr: false, @@ -18,20 +19,142 @@ export const Route = createFileRoute("/login")({ component: LoginPage, }); +function LogoMark({ className }: { className?: string }) { + return ( + + + + + + ); +} + +function Field({ + id, + label, + icon: Icon, + children, +}: { + id: string; + label: string; + icon: typeof User; + children: React.ReactNode; +}) { + return ( +
+ +
+ + {children} +
+
+ ); +} + +function ErrorNote({ message }: { message: string }) { + return ( +
+ + {message} +
+ ); +} + +const inputClass = + "h-9 w-full rounded-sm border border-input bg-background/60 pl-8 pr-3 text-sm text-foreground shadow-none outline-none transition-colors placeholder:text-muted-foreground/60 focus-visible:border-ring focus-visible:ring-1 focus-visible:ring-ring/40 disabled:opacity-60"; + +function loginReturnTarget() { + return `${window.location.origin}/login`; +} + function LoginPage() { const navigate = useNavigate(); const session = useSession(); + const [config, setConfig] = useState(null); const [username, setUsername] = useState(""); const [password, setPassword] = useState(""); + const [showPassword, setShowPassword] = useState(false); const [error, setError] = useState(null); const [busy, setBusy] = useState(false); + const [exchanging, setExchanging] = useState(false); + + const apiBase = getPrefs().apiBase; + const env = apiBase.startsWith("/") ? "local-dev" : apiBase.replace(/^https?:\/\//, ""); + const version = import.meta.env["VITE_APP_VERSION"] || "v0.9.1"; useEffect(() => { if (session) navigate({ to: "/", replace: true }); }, [session, navigate]); + useEffect(() => { + let active = true; + api + .authConfig() + .then((c) => active && setConfig(c)) + .catch(() => active && setConfig({ password: true, oidc: false })); + return () => { + active = false; + }; + }, []); + + // Complete an SSO redirect: exchange the one-time code for a session. + useEffect(() => { + const params = new URLSearchParams(window.location.search); + const code = params.get("code"); + const ssoError = params.get("error"); + if (!code && !ssoError) return; + + const clean = () => { + const url = new URL(window.location.href); + url.searchParams.delete("code"); + url.searchParams.delete("error"); + window.history.replaceState({}, "", url.toString()); + }; + + if (ssoError) { + setError( + ssoError === "access_denied" + ? "Sign-in was cancelled." + : "Sign-in failed. Please try again.", + ); + clean(); + return; + } + + setExchanging(true); + api + .exchangeOidc(code as string) + .then((next) => { + setSession(next); + clean(); + navigate({ to: "/", replace: true }); + }) + .catch((err) => { + setError(err instanceof ApiError ? err.message : "Sign-in failed. Please try again."); + clean(); + setExchanging(false); + }); + }, [navigate]); + + function startSSO() { + const target = `${getApiBase()}/auth/oidc/login?redirect=${encodeURIComponent(loginReturnTarget())}`; + window.location.href = target; + } + async function onSubmit(e: FormEvent) { e.preventDefault(); + if (busy) return; setBusy(true); setError(null); try { @@ -45,48 +168,164 @@ function LoginPage() { } } + const showOIDC = !!config?.oidc; + const passwordEnabled = config ? config.password : true; + const provider = config?.provider || "single sign-on"; + return ( -
-
-
-

ForgeQueue

-

Sign in to the operational console.

-
-
- - setUsername(e.target.value)} - className="h-8 text-xs" - /> -
-
- - setPassword(e.target.value)} - className="h-8 text-xs" - /> +
+ {/* Ambient background: faint grid + radial glow behind the card. */} +
+
+ +
+
+
+ +
+
+
ForgeQueue
+
+ Operational Console +
+
- {error && ( -
- {error} + +
+
+ + {exchanging ? ( +
+ +

Completing sign in…

+
+ ) : ( +
+
+

Sign in

+

+ {showOIDC && !passwordEnabled + ? `Continue with ${provider} to access the console.` + : "Authenticate to manage queues, workers and jobs."} +

+
+ + {error && } + + {showOIDC && ( + + )} + + {showOIDC && passwordEnabled && ( +
+ + + or + + +
+ )} + + {passwordEnabled && ( + + + setUsername(e.target.value)} + placeholder="admin" + className={inputClass} + /> + + + + setPassword(e.target.value)} + placeholder="••••••••••••" + className={cn(inputClass, "pr-9")} + /> + + + + + + )} +
+ )} + +
+ + {env} + + {version}
- )} - - +
+ +

+ {showOIDC + ? `You'll be redirected to ${provider} to sign in. Access is scoped by role.` + : "Credentials are configured on the API server. Access is scoped by role."} +

+
); } diff --git a/web/src/routes/queues.$queueName.tsx b/web/src/routes/queues.$queueName.tsx index 46c599a..68d579d 100644 --- a/web/src/routes/queues.$queueName.tsx +++ b/web/src/routes/queues.$queueName.tsx @@ -3,6 +3,7 @@ import { Pause, Play, Trash2 } from "lucide-react"; import { api } from "@/api/endpoints"; import { useAction, useDelayed, useQueue, useQueueJobs, useThroughput } from "@/api/queries"; import { ConfirmDialog } from "@/components/confirm-dialog"; +import { Can } from "@/components/can"; import { DelayedTable, JobsTable } from "@/components/job-tables"; import { Legend, THROUGHPUT, TimeChart } from "@/components/charts"; import { EmptyState, Mono, PageHeader, Section, StatCard, StatusBadge } from "@/components/shared"; @@ -59,7 +60,7 @@ function QueueDetail() { } actions={ q && ( - <> + {q.status === "paused" ? (