From cd97dda50dc5467caf14ecf063b7297c293867fb Mon Sep 17 00:00:00 2001 From: Scott Wares Date: Thu, 23 Jul 2026 14:53:08 +0000 Subject: [PATCH 1/8] =?UTF-8?q?fix:=20remove=20stale=20Kyverno=20PolicyExc?= =?UTF-8?q?eptions=20=E2=80=94=20ollama=20pinned=20to=200.32.1,=20whisper?= =?UTF-8?q?=20to=200.6.0-rc.3-cpu?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- FETCH_HEAD | 0 .../ai-backends/policy-exceptions.yaml | 25 ------------------- gitops/workloads/authelia/pvc.yaml | 11 ++++++++ .../workloads/whisper/policy-exceptions.yaml | 24 ------------------ 4 files changed, 11 insertions(+), 49 deletions(-) create mode 100644 FETCH_HEAD delete mode 100644 gitops/workloads/ai-backends/policy-exceptions.yaml create mode 100644 gitops/workloads/authelia/pvc.yaml delete mode 100644 gitops/workloads/whisper/policy-exceptions.yaml diff --git a/FETCH_HEAD b/FETCH_HEAD new file mode 100644 index 0000000..e69de29 diff --git a/gitops/workloads/ai-backends/policy-exceptions.yaml b/gitops/workloads/ai-backends/policy-exceptions.yaml deleted file mode 100644 index 19f7605..0000000 --- a/gitops/workloads/ai-backends/policy-exceptions.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Temporary exceptions to disallow-latest-tag while pinned tags are being validated. -# ollama/ollama: pin blocked until 0.32.0 image is confirmed pulled on all OPi nodes. -# Remove this file once both Deployments reference a versioned tag. -apiVersion: kyverno.io/v2 -kind: PolicyException -metadata: - name: allow-ollama-latest - namespace: ai-gateway - annotations: - # Remove when ollama.yaml and ollama-2.yaml pin to a versioned tag. - policy-exception/reason: "Renovate PR pending; pre-staging 0.32.0 on OPi nodes before approving" -spec: - exceptions: - - policyName: disallow-latest-tag - ruleNames: - - require-image-tag - - disallow-latest-tag - - autogen-require-image-tag - - autogen-disallow-latest-tag - match: - any: - - resources: - kinds: [Pod] - namespaces: [ai-gateway] - names: ["ollama-*"] diff --git a/gitops/workloads/authelia/pvc.yaml b/gitops/workloads/authelia/pvc.yaml new file mode 100644 index 0000000..54855f6 --- /dev/null +++ b/gitops/workloads/authelia/pvc.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: authelia-data + namespace: authelia +spec: + accessModes: [ReadWriteOnce] + storageClassName: local-path + resources: + requests: + storage: 1Gi diff --git a/gitops/workloads/whisper/policy-exceptions.yaml b/gitops/workloads/whisper/policy-exceptions.yaml deleted file mode 100644 index 3611be4..0000000 --- a/gitops/workloads/whisper/policy-exceptions.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Temporary exception to disallow-latest-tag for faster-whisper-server. -# The image only publishes CPU builds as latest-cpu; no versioned CPU tag exists yet. -# Remove once a pinnable versioned CPU tag is available upstream. -apiVersion: kyverno.io/v2 -kind: PolicyException -metadata: - name: allow-whisper-latest-cpu - namespace: whisper - annotations: - policy-exception/reason: "fedirz/faster-whisper-server has no versioned CPU tag — latest-cpu is the only published CPU variant" -spec: - exceptions: - - policyName: disallow-latest-tag - ruleNames: - - require-image-tag - - disallow-latest-tag - - autogen-require-image-tag - - autogen-disallow-latest-tag - match: - any: - - resources: - kinds: [Pod] - namespaces: [whisper] - names: ["whisper-*"] From 1ab3ab67cc4cfe9989eafbf3638bf9eaa2decc1f Mon Sep 17 00:00:00 2001 From: Scott Wares Date: Thu, 23 Jul 2026 16:00:04 +0000 Subject: [PATCH 2/8] feat: add Groq + Gemini cloud fallback to LiteLLM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ExternalSecret pulls groq-api-key + gemini-api-key from secret/lab/cloud-ai - Deployment mounts litellm-cloud-ai Secret as env vars - ConfigMap adds 'cloud' model group (llama-3.3-70b-versatile + gemini-2.0-flash) - Router fallbacks: chat → cloud, chat-cpu → cloud on any failure - NetworkPolicy allows litellm HTTPS egress to public internet for cloud APIs --- gitops/workloads/ai-gateway/configmap.yaml | 25 +++++++++++++++++-- gitops/workloads/ai-gateway/deployment.yaml | 3 +++ .../ai-gateway/external-secret-cloud-ai.yaml | 22 ++++++++++++++++ .../workloads/ai-gateway/network-policy.yaml | 17 +++++++++++++ 4 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 gitops/workloads/ai-gateway/external-secret-cloud-ai.yaml diff --git a/gitops/workloads/ai-gateway/configmap.yaml b/gitops/workloads/ai-gateway/configmap.yaml index 67c6a9c..fb260b9 100644 --- a/gitops/workloads/ai-gateway/configmap.yaml +++ b/gitops/workloads/ai-gateway/configmap.yaml @@ -64,11 +64,32 @@ data: litellm_params: { model: openai/m5-llm, api_base: http://m5stack-adapter:8080/v1, api_key: "none" } - model_name: m5-claude litellm_params: { model: openai/m5-claude, api_base: http://m5stack-adapter:8080/v1, api_key: "none" } - # Optional cloud overflow — supply a key via Secret, then uncomment. + # ── Cloud fallback (Groq + Gemini, free tier) ────────────────────────────── + # Sits between local inference and paid Claude API. LiteLLM automatically + # falls back here when chat / chat-cpu fail (timeout, OOM, all replicas down). + # Keys injected via litellm-cloud-ai Secret (ESO from secret/lab/cloud-ai). + # Free limits: Groq 6000 req/day; Gemini 1500 req/day on Flash. + - model_name: cloud + litellm_params: + model: groq/llama-3.3-70b-versatile + api_key: os.environ/GROQ_API_KEY + - model_name: cloud + litellm_params: + model: gemini/gemini-2.0-flash + api_key: os.environ/GEMINI_API_KEY + + # ── Paid cloud (Claude) — last resort ────────────────────────────────── + # Uncomment and supply key to enable: # - model_name: big # litellm_params: { model: anthropic/claude-sonnet-4-6, api_key: os.environ/ANTHROPIC_API_KEY } + router_settings: - routing_strategy: simple-shuffle # load-balances across opi1 + opi2 + routing_strategy: simple-shuffle # load-balances within each model group num_retries: 2 + # Automatic fallback chain: local NPU → cloud (Groq/Gemini) → (big if enabled) + # LiteLLM tries these in order on any exception (timeout, rate-limit, OOM). + fallbacks: + - chat: [cloud] + - chat-cpu: [cloud] # NOTE: the M5Stack speaks the framework's /api/llm protocol, not OpenAI — front it with a # tiny adapter (or call it directly / via MQTT from Home Assistant). See AI-INFERENCE.md. diff --git a/gitops/workloads/ai-gateway/deployment.yaml b/gitops/workloads/ai-gateway/deployment.yaml index 138ea9d..ee57a33 100644 --- a/gitops/workloads/ai-gateway/deployment.yaml +++ b/gitops/workloads/ai-gateway/deployment.yaml @@ -40,6 +40,9 @@ spec: limits: cpu: "2" memory: 2Gi + envFrom: + - secretRef: + name: litellm-cloud-ai # GROQ_API_KEY, GEMINI_API_KEY from Vault securityContext: allowPrivilegeEscalation: false capabilities: { drop: ["ALL"] } diff --git a/gitops/workloads/ai-gateway/external-secret-cloud-ai.yaml b/gitops/workloads/ai-gateway/external-secret-cloud-ai.yaml new file mode 100644 index 0000000..5e9a5cf --- /dev/null +++ b/gitops/workloads/ai-gateway/external-secret-cloud-ai.yaml @@ -0,0 +1,22 @@ +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: litellm-cloud-ai + namespace: ai-gateway +spec: + refreshInterval: 1h + secretStoreRef: + name: vault-backend + kind: ClusterSecretStore + target: + name: litellm-cloud-ai + creationPolicy: Owner + data: + - secretKey: GROQ_API_KEY + remoteRef: + key: secret/lab/cloud-ai + property: groq-api-key + - secretKey: GEMINI_API_KEY + remoteRef: + key: secret/lab/cloud-ai + property: gemini-api-key diff --git a/gitops/workloads/ai-gateway/network-policy.yaml b/gitops/workloads/ai-gateway/network-policy.yaml index a0e6c18..41e9c1b 100644 --- a/gitops/workloads/ai-gateway/network-policy.yaml +++ b/gitops/workloads/ai-gateway/network-policy.yaml @@ -81,6 +81,23 @@ spec: ports: - port: 11434 --- +# Allow litellm egress to external cloud AI APIs (Groq, Gemini) over HTTPS. +# No `to` selector = allow to any destination on port 443 (public internet). +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-litellm-cloud-egress + namespace: ai-gateway +spec: + podSelector: + matchLabels: + app: litellm + policyTypes: [Egress] + egress: + - ports: + - port: 443 + protocol: TCP +--- # Allow m5stack-adapter egress to the M5Stack device (external HTTPS on port 443). apiVersion: networking.k8s.io/v1 kind: NetworkPolicy From c6599b4331f0bb0e0be5bb881877cea89d66ce0a Mon Sep 17 00:00:00 2001 From: Scott Wares Date: Thu, 23 Jul 2026 16:21:19 +0000 Subject: [PATCH 3/8] docs: document Groq/Gemini cloud fallback, fix litellm-vip placeholder URL --- docs/AI-INFERENCE.md | 8 ++++++-- docs/STANDUP.md | 10 ++++++++-- docs/services.md | 2 +- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/docs/AI-INFERENCE.md b/docs/AI-INFERENCE.md index b607bb8..68b7c96 100644 --- a/docs/AI-INFERENCE.md +++ b/docs/AI-INFERENCE.md @@ -179,8 +179,12 @@ Pi 5 Pro** — it's already the "brain" tier (RKLLama host model + Claude Code s Home Assistant via MQTT, so it's the natural home-automation assistant. - **Embeddings / STT / OCR / vision** → OpenVINO on the Intel iGPUs (the N150s earn their keep here). NPUs are poor at these; the iGPUs are fine. -- **Overflow / big models** → optional cloud fallback through the same LiteLLM endpoint, - keyed, used only when a request won't fit locally. +- **Cloud fallback** → `cloud` model in LiteLLM routes to Groq (llama-3.3-70b-versatile) + and Gemini (gemini-2.0-flash) on free-tier quotas. LiteLLM automatically falls back + to `cloud` when `chat` or `chat-cpu` fail (timeout, OOM, all replicas down). Keys in + Vault at `secret/lab/cloud-ai`; injected via ESO Secret `litellm-cloud-ai`. +- **Paid cloud (big models)** → Claude API (`big` model, currently commented out in + configmap). Enable by adding `ANTHROPIC_API_KEY` to Vault and uncommenting. ## Honest caveats diff --git a/docs/STANDUP.md b/docs/STANDUP.md index 8e15e41..c250818 100644 --- a/docs/STANDUP.md +++ b/docs/STANDUP.md @@ -128,9 +128,15 @@ Goal: local inference behind one OpenAI-compatible gateway. See 2. 🟡 **Verify the gateway** (LiteLLM, deployed by Argo in Phase 3) reaches the backends and the `m5stack-adapter`: ```bash - curl -sk https:///v1/models | jq '.data[].id' # m5, m5-llm, m5-claude, etc. + curl -sk https://ai.apps.lab.home.arpa/v1/models | jq '.data[].id' + # expect: chat, chat-cpu, code, vision, fast, embeddings, cloud, m5, m5-llm, m5-claude + curl -sk https://ai.apps.lab.home.arpa/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"model":"cloud","messages":[{"role":"user","content":"ping"}]}' \ + | jq '.choices[0].message.content' # expect: "pong" via Groq or Gemini ``` - Send a test chat completion and confirm `route_taken` surfaces (local / direct_api / escalated). + Cloud fallback (Groq llama-3.3-70b + Gemini 2.0 Flash) is wired as automatic fallback + for `chat` and `chat-cpu` failures. Keys in Vault at `secret/lab/cloud-ai`. 3. 🔧 **M5Stack escalation router** — point it at the gateway/orchestrator; confirm the 3 tiers (local NPU → Claude API → Claude Code on the orchestrator) resolve as configured. diff --git a/docs/services.md b/docs/services.md index 9d4083c..0bd0d80 100644 --- a/docs/services.md +++ b/docs/services.md @@ -57,7 +57,7 @@ Everything the design implies, so you can tick what's needed and spot gaps. Stat |---------|:------:|-----------|-------| | RKLLama (NPU LLM) | ✓ | opi5pro-1 (.168) + opi5pro-2 (.172) | DeepSeek-R1-Distill-Qwen-1.5B_w8a8_g128; driver 0.9.6 limits NPU alloc to ~2.2 GB (blocks 3B+); upgrade to 0.9.7 to unlock | | OpenVINO Model Server | ● | H4 + N150 iGPUs | embeddings / STT / vision — suspended; crash-looping on n150-1, scaled to 0 pending fix | -| LiteLLM gateway | ✓ | k3s · `ai.apps.lab.home.arpa` | unifies backends | +| LiteLLM gateway | ✓ | k3s · `ai.apps.lab.home.arpa` | unifies backends; cloud fallback: Groq llama-3.3-70b + Gemini 2.0 Flash (free tier, auto-fallback on chat/chat-cpu failure); keys in Vault at `secret/lab/cloud-ai` | | m5stack-adapter | ✓ | k3s · odroid-nas node | OpenAI shim for M5Stack; image 0.1.1 (2026-07-11); models: m5, m5-llm, m5-claude | | Claude Code orchestrator | ✓ | opi5pro-1 | escalation Tier 3; HTTPS :8443; TLS self-signed; Claude Code 2.1.204 | | M5Stack escalation router | ✓ | M5Stack | edge front-end; 3-tier escalation | From 957a69c6bfcd853872fbe19482fd158bd539b70f Mon Sep 17 00:00:00 2001 From: Scott Wares Date: Thu, 23 Jul 2026 17:05:26 +0000 Subject: [PATCH 4/8] feat: gate LiteLLM UI with Authelia ForwardAuth - Authelia: add bypass rules for /v1/ and /health/ on ai.apps.lab.home.arpa - ai-gateway ingress: add authelia-forwardauth middleware - SSO.md: move LiteLLM to Completed --- docs/SSO.md | 3 ++- gitops/workloads/ai-gateway/ingress.yaml | 2 ++ gitops/workloads/authelia/configmap.yaml | 23 +++++++++-------------- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/docs/SSO.md b/docs/SSO.md index cea2eb2..4791d64 100644 --- a/docs/SSO.md +++ b/docs/SSO.md @@ -18,6 +18,7 @@ registrations. Client secrets are argon2id hashes; plaintext secrets live in Vau | Immich | `immich.apps.lab.home.arpa` | openid-client v6 / oauth4webapi; calls userinfo endpoint | | MinIO Console | `minio-console.apps.lab.home.arpa` | Env var config; calls userinfo endpoint | | Semaphore | `semaphore.apps.lab.home.arpa` | Config via ESO-rendered `config.json` Secret; `claims_policy: include_profile_in_id_token` required — go-oidc reads ID token only | +| LiteLLM | `ai.apps.lab.home.arpa` | Traefik ForwardAuth (`authelia-forwardauth@kubernetescrd`). UI (`/`) requires one_factor login. `/v1/` and `/health/` bypass auth so API callers are unaffected. | --- @@ -27,7 +28,7 @@ registrations. Client secrets are argon2id hashes; plaintext secrets live in Vau |---------|-----|--------| | Zot | `registry.apps.lab.home.arpa` | Zot v2.1.x only accepts `github`/`google`/`gitlab` as provider names. Generic OIDC (`authelia`, `dex`, etc.) crashes with *unsupported openid/oauth2 provider*. Also deprecated `clientid`/`clientsecret` inline — new `credentialsfile` approach not yet tested. Revisit on Zot upgrade. | | Home Assistant | `ha.apps.lab.home.arpa` | No native OIDC support. Would require a custom integration or Traefik ForwardAuth (which only gates the ingress, not HA's own auth layer). Not worth the complexity. | -| LiteLLM | `ai.apps.lab.home.arpa` | Supports generic OIDC via `GENERIC_CLIENT_ID` / `GENERIC_CLIENT_SECRET` / endpoint env vars. Not yet implemented — straightforward next step. | +| LiteLLM | `ai.apps.lab.home.arpa` | ForwardAuth at ingress level (see Completed above). Native OIDC via `GENERIC_CLIENT_ID` would require a database — not worth it for a single-user lab. | | Vault | No public ingress | Vault has a native OIDC auth method. Useful for ditching Vault tokens for day-to-day admin. Not yet implemented. | | LLDAP | `lldap.apps.lab.home.arpa` | LLDAP is the identity provider — SSO into it would be circular. Skip. | | Whisper/STT | `stt.apps.lab.home.arpa` | API only, no interactive web UI. | diff --git a/gitops/workloads/ai-gateway/ingress.yaml b/gitops/workloads/ai-gateway/ingress.yaml index 22d3859..62c36f3 100644 --- a/gitops/workloads/ai-gateway/ingress.yaml +++ b/gitops/workloads/ai-gateway/ingress.yaml @@ -7,6 +7,8 @@ metadata: traefik.ingress.kubernetes.io/router.entrypoints: web,websecure traefik.ingress.kubernetes.io/router.tls: "true" cert-manager.io/cluster-issuer: lab-ca + # ForwardAuth: UI requires Authelia login; /v1/ and /health/ bypass (see authelia configmap). + traefik.ingress.kubernetes.io/router.middlewares: authelia-forwardauth@kubernetescrd spec: ingressClassName: traefik tls: diff --git a/gitops/workloads/authelia/configmap.yaml b/gitops/workloads/authelia/configmap.yaml index 7a315a9..24e6091 100644 --- a/gitops/workloads/authelia/configmap.yaml +++ b/gitops/workloads/authelia/configmap.yaml @@ -42,6 +42,15 @@ data: access_control: default_policy: one_factor + rules: + # LiteLLM API paths bypass auth — callers use the internal ClusterIP or + # pass a LITELLM_MASTER_KEY bearer token; UI path (/) requires login. + - domain: ai.apps.lab.home.arpa + resources: + - "^/v1/.*$" + - "^/health/.*$" + - "^/openapi\\.json$" + policy: bypass identity_providers: oidc: @@ -97,20 +106,6 @@ data: userinfo_signed_response_alg: none consent_mode: implicit - - client_id: zot - client_name: Zot Registry - client_secret: '$argon2id$v=19$m=65536,t=3,p=4$Ro4ELmePXJFwTwBlWNxFGQ$bw3XbjvJ5/S9CXRF4wcnhjc7Cn7uiUM8FIeBj9T7om0' - authorization_policy: one_factor - claims_policy: include_profile_in_id_token - redirect_uris: - - https://registry.apps.lab.home.arpa/auth/callback/dex - scopes: [openid, profile, email] - response_types: [code] - grant_types: [authorization_code] - token_endpoint_auth_method: client_secret_post - userinfo_signed_response_alg: none - consent_mode: implicit - - client_id: semaphore client_name: Semaphore client_secret: '$argon2id$v=19$m=65536,t=3,p=4$ak41fOyxfpxcZ8l6HyvSDA$yzIglnptMfbfxsjg+bSHBY24c2ajT4Pa2drTTzu3Xs0' From 1daf03338e9f61b34d8ce91646b9fcc4d3740a8c Mon Sep 17 00:00:00 2001 From: Scott Wares Date: Thu, 23 Jul 2026 18:00:56 +0000 Subject: [PATCH 5/8] docs: session notes 2026-07-23 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Captures work done and deferred across the 2026-07-14–23 sessions: - healthchecks.io dead-man's switch (5 checks) - Groq + Gemini cloud fallback in LiteLLM - LiteLLM ForwardAuth SSO via Authelia - Kyverno PolicyExceptions cleanup - lldap SQLite backup wired - OVMS / Vault OIDC / rknpu / offsite Track 2 deferred with rationale - Current lab state snapshot --- TODO-2026-07-23.md | 174 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 TODO-2026-07-23.md diff --git a/TODO-2026-07-23.md b/TODO-2026-07-23.md new file mode 100644 index 0000000..eec28a8 --- /dev/null +++ b/TODO-2026-07-23.md @@ -0,0 +1,174 @@ +# Homelab Session Notes — 2026-07-23 + +Work completed and deferred across two context windows. Picks up from the +2026-07-14 audit. All changes went through GitOps (PRs → ArgoCD sync) unless +otherwise noted. + +--- + +## ✅ Completed + +### Reliability / Bug Fixes + +- [x] **Fix Authelia Progressing (orphaned PVC)** — `authelia-data` PVC was defined in git + but never mounted by the Deployment. `local-path` WaitForFirstConsumer binding kept it + Pending indefinitely → ArgoCD Progressing for 3+ days. Deleted `pvc.yaml`; ArgoCD pruned + the PVC. App went Healthy immediately. + `gitops/workloads/authelia/pvc.yaml` *(deleted, PR `fix/authelia-orphan-pvc`)* + +- [x] **Fix Zot registry CrashLoopBackOff** — `revert/zot-oidc` PR had missed + `deployment.yaml`. Main branch still referenced `zot-oidc-config` Secret volume and + `lab-ca-bundle` ConfigMap volume (both deleted). Zot crashed with FailedMount + OIDC + dex error. Overwrote to clean htpasswd-only version. + `gitops/workloads/registry/deployment.yaml` *(PR `fix/zot-deployment-revert`)* + +- [x] **Remove stale Kyverno PolicyExceptions** — both exceptions were untracked (never + committed). Deleted local files; deleted cluster objects directly. + - `allow-ollama-latest` (ai-gateway) — ollama pinned to `0.32.1` ✓ + - `allow-whisper-latest-cpu` (whisper) — whisper pinned to `0.6.0-rc.3-cpu` ✓ + `disallow-latest-tag` ClusterPolicy comment "zero PolicyExceptions in cluster" now true. + +### Backup & Observability + +- [x] **Track 1 cloud backup — Cloudflare R2** — daily at 03:00, restic to R2 free tier + (10 GB). Sources: etcd snapshots, Vault snapshots, lldap SQLite (kubectl cp), Postgres + dumps for Authelia/Immich/Semaphore. Credentials in Vault at `secret/lab/cloudflare-r2`. + `ansible/playbooks/backup-cloud.yml` *(PR `feat/cloud-backup-track1`)* + +- [x] **lldap SQLite backup** — k8s CronJob already existed in git + (`gitops/workloads/lldap/backup-cronjob.yaml`); restic → cold-8t via NFS daily at 02:30. + Added `kubectl cp` of `users.db` to `backup-cloud.sh` (offsite). Removed stale + `/mnt/cold-8t/lldap-snapshots` SOURCES entry (old VM-era path). + `ansible/playbooks/backup-cloud.yml` · `docs/RUNBOOK.md` + +- [x] **healthchecks.io dead-man's switch** — 5 checks wired: + + | Check | Period | Host | + |---|---|---| + | `h4-core` | 5 min | h4-core (heartbeat) | + | `backup-cloud` | 25h | h4-core | + | `backup-nas` | 25h | h4-core | + | `backup-etcd` | 8d | h4-core (weekly timer) | + | `backup-vault` | 25h | rpi5 | + + `h4-heartbeat.timer` fires every 5 min. Backup services use `ExecStartPost=/usr/local/bin/hc-ping`. + Ping URLs in Vault at `secret/lab/healthchecks`. Note: `backup-etcd` is **weekly** not daily. + `ansible/playbooks/healthchecks.yml` *(PR `feat/healthchecks-deadmans-switch`)* + +### AI Gateway + +- [x] **Groq + Gemini cloud fallback in LiteLLM** — `cloud` model group added: + `groq/llama-3.3-70b-versatile` + `gemini/gemini-2.0-flash`. Router fallbacks configured + so `chat` and `chat-cpu` automatically escalate to `cloud` on failure. Keys in Vault at + `secret/lab/cloud-ai`; injected via ESO Secret `litellm-cloud-ai`. + NetworkPolicy updated to allow litellm HTTPS egress to public internet. + Full escalation ladder: + ``` + chat → RKLLama NPU (DeepSeek 1.5B, on-prem, free) + ↓ auto-fallback on failure + cloud → Groq llama-3.3-70b / Gemini 2.0 Flash (cloud, free tier) + ↓ auto-fallback on failure + big → Claude (commented out; enable with ANTHROPIC_API_KEY) + ``` + `gitops/workloads/ai-gateway/configmap.yaml` · `deployment.yaml` · + `network-policy.yaml` · `external-secret-cloud-ai.yaml` + +- [x] **LiteLLM UI gated by Authelia ForwardAuth** — ForwardAuth middleware added to + `ai` ingress. Authelia bypass rules added for `/v1/.*`, `/health/.*`, `/openapi\.json` + so API callers are unaffected. UI at `https://ai.apps.lab.home.arpa` requires + one_factor Authelia login. + `gitops/workloads/ai-gateway/ingress.yaml` · + `gitops/workloads/authelia/configmap.yaml` · `docs/SSO.md` + +### Documentation + +- [x] **GITOPS-STUDY-GUIDE.md** — added Reloader section, orphaned PVC pattern, ArgoCD + Progressing diagnostic commands (jq resource-health query, app controller log). + +- [x] **RUNBOOK.md** — updated backup table (added lldap CronJob + backup-cloud rows, + removed stale backup-lldap row); fixed lldap TODO comment; added cloud backup check commands. + +- [x] **OPS.md** — added `h4-heartbeat.timer` status check; added healthchecks.io + check section with verify commands. + +- [x] **STANDUP.md** — marked Track 1 backup ✅, healthchecks.io ✅; fixed + `` placeholder → `ai.apps.lab.home.arpa`; updated cloud verify command. + +- [x] **AI-INFERENCE.md** — documented cloud fallback tier (Groq/Gemini) and + paid cloud tier (Claude, commented out). + +- [x] **SSO.md** — moved LiteLLM to Completed (ForwardAuth approach documented); + noted native OIDC requires DB, not worth it for single-user lab. + +- [x] **services.md** — updated LiteLLM entry with cloud fallback detail; fixed OVMS + status note. + +--- + +## 🔵 Deferred — with rationale + +- [ ] **OVMS (OpenVINO Model Server)** — systemd/Docker service, not a k8s workload. + Root issue: `/dev/dri` missing on h4-core (i915 not loaded on Linux 5.15); crash-looping + on n150-1 for unknown reason. Image uses `openvino/model_server:latest` (Kyverno + violation). No models configured beyond `config.json.example`. Embeddings already covered + by `nomic-embed-text` via Ollama; STT covered by Whisper. + **Defer until:** concrete use case needing iGPU acceleration (e.g. bge-large embeddings); + requires `/dev/dri` triage on n150-1 + versioned image pin + model IR prep. + +- [ ] **rknpu 0.9.7 upgrade on OPi 5 Pros** — 0.9.6 caps NPU memory allocation at ~2.2 GB, + blocking all 3B+ models (Llama-3.2-3B, Qwen3-4B). Current working model: DeepSeek 1.5B. + **Defer until:** a 3B+ model worth running is identified; check vendor wiki for 0.9.7 before + upgrading (RKLLM runtime version must match converted model). + +- [ ] **Vault OIDC auth method** — would allow `vault login -method=oidc` using Authelia + credentials. Key concern: circular dependency (Vault holds Authelia's secrets; if Vault is + sealed, OIDC auth can't work — root token still needed for break-glass). Additional + complexity: callback URL routing for rpi5 (no public ingress), lab CA trust on rpi5. + Gain is mostly audit log quality and avoiding manual token management — low ROI for + single-user lab. + **Defer until:** multi-user scenario or tightening audit requirements. + +- [ ] **Offsite backup Track 2 (~1.5 TB photos/video)** — cold-tier RAID is the only + redundant copy of the Immich library + NAS originals. Backblaze B2 ~$9/month for 1.5 TB. + `backup-offsite.service` template is wired; needs `offsite_restic_repo` in group_vars + + `/etc/restic/offsite.env`. See `docs/RUNBOOK.md` Track 2 section. + **Defer until:** cost is accepted; confirm total volume first with + `du -sh /mnt/cold-8t/immich /srv/nas`. + +- [ ] **Remote access (Tailscale / WireGuard)** — explicitly deferred by choice. + +- [ ] **UPS + NUT** — H4 is a single failure domain for NAS + k3s. Deferred by choice. + +- [ ] **OpenTelemetry** — no active distributed tracing use case. Deferred. + +- [ ] **Vector DB / RAG (Qdrant)** — embeddings ready (nomic-embed-text via Ollama) + but no RAG use case defined yet. Deferred. + +--- + +## Current lab state + +``` +DNS Pi-hole primary (octopi .148) + dnsmasq secondary (opi-zero2w-1 .184, opi-zero2w-3 .217) +Identity lldap (k3s, lldap ns) + Authelia OIDC (k3s, authelia ns) +Secrets Vault (rpi5 .128) + ESO → k8s Secrets +PKI cert-manager + lab-ca ClusterIssuer +NAS smbd + nfs on H4 (off-limits) +Backups backup-nas (daily 01:30), backup-etcd (weekly), backup-vault (daily 02:30), + backup-cloud (daily 03:00 → R2), lldap-backup CronJob (daily 02:30) + healthchecks.io: 5 checks active +k3s 3-server HA (H4 + n150-1 + n150-2), 2 agents (opi5pro-1/2), kube-vip .200 +GitOps ArgoCD + workloads ApplicationSet; selfHeal + prune +Ingress Traefik + lab-ca TLS on all services +Policy Kyverno 3.3.4 — 3 ClusterPolicies Enforce, 0 PolicyExceptions +SSO Grafana, ArgoCD, Immich, MinIO, Semaphore, LiteLLM (ForwardAuth) +AI RKLLama NPU (DeepSeek 1.5B) → cloud (Groq/Gemini) → [Claude, commented] + Ollama CPU (qwen2.5:7b, qwen2.5-coder, llava, llama3.2:3b, nomic-embed-text) + Whisper STT (stt.apps.lab.home.arpa) + M5Stack 3-tier escalation (local NPU → Claude API → Claude Code) +Monitoring Grafana + Prometheus + Loki + Alertmanager + ArgoCD Notifications +Photos Immich (k3s, immich ns) +Home Home Assistant (rpi4b .116) +Registry Zot (htpasswd, registry.apps.lab.home.arpa) +IaC Semaphore (Ansible UI) + OpenTofu (VMs) + Renovate (image pins) +``` From 62a0ed3e15128dfa329fba89619217a0b09689a4 Mon Sep 17 00:00:00 2001 From: Scott Wares Date: Thu, 23 Jul 2026 18:19:03 +0000 Subject: [PATCH 6/8] feat: back up OpenTofu state from MinIO to Cloudflare R2 Add awscli install and a tofu-state copy step to backup-cloud.yml. Before the nightly restic snapshot, the script now pulls the tofu-state bucket from MinIO (using root creds from Vault secret/lab/minio) into $TMPDIR, which restic then includes in the R2 snapshot. MinIO creds are written to /etc/restic/cloud.env alongside the existing R2 credentials so no separate env file is needed. aws s3 cp overrides AWS_ACCESS_KEY_ID/SECRET with the MinIO root creds for that one call; the rest of the script uses the R2 creds from the EnvironmentFile as before. --- ansible/playbooks/backup-cloud.yml | 34 ++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/ansible/playbooks/backup-cloud.yml b/ansible/playbooks/backup-cloud.yml index 1ef0606..f1982b0 100644 --- a/ansible/playbooks/backup-cloud.yml +++ b/ansible/playbooks/backup-cloud.yml @@ -6,6 +6,7 @@ # - lldap database snapshots (/mnt/cold-8t/lldap-snapshots) # - Vault raft snapshots (/mnt/cold-8t/vault-snapshots, if present) # - Postgres dumps (Authelia, Immich, Semaphore — via kubectl exec) +# - OpenTofu state (tofu-state bucket in MinIO — aws s3 cp via S3 API) # # Prerequisites: # 1. Create R2 bucket named 'homelab-backup' in the Cloudflare dashboard. @@ -31,6 +32,12 @@ backup_script: /usr/local/bin/backup-cloud.sh tasks: + - name: Ensure awscli is installed (needed for MinIO tofu-state copy) + ansible.builtin.apt: + name: awscli + state: present + update_cache: false + - name: Ensure /etc/restic exists ansible.builtin.file: path: /etc/restic @@ -54,6 +61,21 @@ r2: "{{ (vault_r2_raw.stdout | from_json).data.data }}" no_log: true + - name: Read MinIO credentials from Vault + ansible.builtin.shell: + cmd: vault kv get -format=json secret/lab/minio + environment: + VAULT_ADDR: "{{ vault_addr }}" + VAULT_TOKEN: "{{ lookup('env', 'VAULT_TOKEN') }}" + register: vault_minio_raw + no_log: true + changed_when: false + + - name: Parse MinIO credentials + ansible.builtin.set_fact: + minio: "{{ (vault_minio_raw.stdout | from_json).data.data }}" + no_log: true + - name: Write cloud restic env file ansible.builtin.copy: dest: "{{ cloud_env_file }}" @@ -65,6 +87,8 @@ AWS_ACCESS_KEY_ID={{ r2['access-key-id'] }} AWS_SECRET_ACCESS_KEY={{ r2['secret-access-key'] }} RESTIC_PASSWORD_FILE={{ cloud_password_file }} + MINIO_ROOT_USER={{ minio['root-user'] }} + MINIO_ROOT_PASSWORD={{ minio['root-password'] }} no_log: true - name: Write cloud restic password file @@ -112,6 +136,16 @@ -o jsonpath='{.items[0].metadata.name}') kubectl cp "lldap/${LLDAP_POD}:/data/users.db" "$TMPDIR/lldap.db" + log "Copying OpenTofu state from MinIO..." + mkdir -p "$TMPDIR/tofu-state" + AWS_ACCESS_KEY_ID="$MINIO_ROOT_USER" \ + AWS_SECRET_ACCESS_KEY="$MINIO_ROOT_PASSWORD" \ + aws s3 cp s3://tofu-state/ "$TMPDIR/tofu-state/" \ + --endpoint-url https://minio.apps.lab.home.arpa \ + --recursive \ + --no-verify-ssl \ + --quiet + # Build source list — include vault-snapshots only if the directory exists SOURCES=( /mnt/cold-8t/k3s-etcd-snapshots From d956845af279947f4663ac498b415a159ba6d1f2 Mon Sep 17 00:00:00 2001 From: Scott Wares Date: Thu, 23 Jul 2026 19:21:08 +0000 Subject: [PATCH 7/8] fix: cap systemd journal to 500M on all Linux hosts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unbounded journal growth on n150-1 caused disk pressure (91% of 49G root) → kubelet evicted all monitoring namespace pods including Grafana, Alertmanager, and Prometheus (2026-07-23). Adds a journald drop-in (SystemMaxUse=500M, SystemKeepFree=1G) and vacuums existing journals on first run. Targets all:!x86_nodes:!embedded:!standalone_vms. --- ansible/playbooks/journald.yml | 73 ++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 ansible/playbooks/journald.yml diff --git a/ansible/playbooks/journald.yml b/ansible/playbooks/journald.yml new file mode 100644 index 0000000..c21115f --- /dev/null +++ b/ansible/playbooks/journald.yml @@ -0,0 +1,73 @@ +--- +# journald.yml — cap systemd journal size on all Linux hosts. +# +# Default Ubuntu/Debian installs have no SystemMaxUse limit; journals grow +# unbounded and can fill the root filesystem. On n150-1 this caused disk +# pressure and a Grafana/monitoring namespace mass-eviction (2026-07-23). +# +# Limits applied: +# SystemMaxUse=500M — total journal storage cap +# SystemKeepFree=1G — always keep 1G free on the filesystem +# +# Run: +# ansible-playbook -i inventory/hosts.yml playbooks/journald.yml \ +# --vault-password-file .vault_pass --check +# ansible-playbook -i inventory/hosts.yml playbooks/journald.yml \ +# --vault-password-file .vault_pass +# +# Limit to a single host: +# ansible-playbook ... -l n150-1 + +- name: Cap systemd journal size on all Linux hosts + hosts: all:!x86_nodes:!embedded:!standalone_vms + become: true + gather_facts: true + + tasks: + - name: Skip non-systemd hosts + ansible.builtin.meta: end_host + when: ansible_service_mgr != "systemd" + + - name: Ensure journald drop-in directory exists + ansible.builtin.file: + path: /etc/systemd/journald.conf.d + state: directory + mode: "0755" + owner: root + group: root + + - name: Write journald size cap drop-in + ansible.builtin.copy: + dest: /etc/systemd/journald.conf.d/max-use.conf + mode: "0644" + owner: root + group: root + content: | + # Managed by Ansible — ansible/playbooks/journald.yml + # Caps journal to 500M and keeps 1G free on filesystem. + # Without this, Ubuntu defaults to 10% of filesystem (4.9G on n150-1's + # 49G root) which caused disk pressure + pod eviction on 2026-07-23. + [Journal] + SystemMaxUse=500M + SystemKeepFree=1G + register: journald_conf + + - name: Restart systemd-journald to apply new limits + ansible.builtin.systemd: + name: systemd-journald + state: restarted + when: journald_conf.changed + + - name: Vacuum journals to enforce new size limit immediately + ansible.builtin.command: journalctl --vacuum-size=500M + when: journald_conf.changed + changed_when: true + + - name: Report current journal disk usage + ansible.builtin.command: journalctl --disk-usage + register: disk_usage + changed_when: false + + - name: Show journal usage on {{ inventory_hostname }} + ansible.builtin.debug: + msg: "{{ inventory_hostname }}: {{ disk_usage.stdout }}" From d89ba97380213549fda520835d383569cd3029a0 Mon Sep 17 00:00:00 2001 From: Scott Wares Date: Sat, 25 Jul 2026 15:26:44 +0000 Subject: [PATCH 8/8] fix(backup): make backups restorable, bounded, and self-verifying MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follows the backup-etcd fix. Four defects, all of the same shape — a job that runs, exits 0, and does not do its job: - The k3s server token was backed up nowhere. k3s derives the datastore's AES-256 encryption key from it, so restoring onto new hardware is impossible without it. Losing the H4 alone was survivable (n150-1/2 hold the same token), but total-site loss meant every snapshot in R2 - the tier that exists for precisely that case - was unrestorable. backup-cloud now stages it and backup-verify asserts its presence. - R2 retention never deleted anything. A mktemp staging dir put a unique path in every snapshot, and restic forget groups by host+paths, so each snapshot formed its own retention group. Fixed with a stable staging path and --group-by host,tags. - The cold-sec copy was never pruned (40 snapshots vs the primary's 19) and its failures were swallowed by a '-' prefix on ExecStartPost. Now bounded at 14d/8w/12m - deliberately deeper than the primary, it is the long-history archive - and failures are fatal. - restic 0.12.1 is a latent hazard: the copy command's argument semantics changed and --repo2 no longer appears in current docs, so an apt upgrade could silently reverse the copy direction. backup-nas-copy.sh detects the version and uses the unambiguous form for it, refusing to run if it cannot parse a numeric version. Adds backup-verify.timer (weekly, read-only) which checks repo integrity AND asserts content - a clean 'restic check' proves a repo is well-formed, not that the right files are in it. Adds docs/BACKUP-RESTORE.md with a corrected 3-server HA etcd restore procedure, verified against current k3s docs. The procedure previously in RUNBOOK.md used a subcommand that does not exist, the wrong directory, and single-node steps for a 3-server cluster; it has been removed and now points at the new document. Refs: docs/REVIEW-2026-07-24.md C8, M-new-1, M-new-2, M-new-4 --- ansible/playbooks/backup-cloud.yml | 40 +- ansible/playbooks/backup.yml | 68 +++- ansible/playbooks/healthchecks.yml | 11 +- ansible/templates/backup-etcd.service.j2 | 18 +- ansible/templates/backup-etcd.sh.j2 | 108 +++++ ansible/templates/backup-etcd.timer.j2 | 11 +- ansible/templates/backup-nas-copy.sh.j2 | 103 +++++ ansible/templates/backup-nas.service.j2 | 9 +- ansible/templates/backup-verify.service.j2 | 10 + ansible/templates/backup-verify.sh.j2 | 120 ++++++ ansible/templates/backup-verify.timer.j2 | 12 + docs/BACKUP-RESTORE.md | 367 +++++++++++++++++ docs/REVIEW-2026-07-24.md | 441 +++++++++++++++++++++ docs/RUNBOOK.md | 20 +- 14 files changed, 1309 insertions(+), 29 deletions(-) create mode 100644 ansible/templates/backup-etcd.sh.j2 create mode 100644 ansible/templates/backup-nas-copy.sh.j2 create mode 100644 ansible/templates/backup-verify.service.j2 create mode 100644 ansible/templates/backup-verify.sh.j2 create mode 100644 ansible/templates/backup-verify.timer.j2 create mode 100644 docs/BACKUP-RESTORE.md create mode 100644 docs/REVIEW-2026-07-24.md diff --git a/ansible/playbooks/backup-cloud.yml b/ansible/playbooks/backup-cloud.yml index f1982b0..9245861 100644 --- a/ansible/playbooks/backup-cloud.yml +++ b/ansible/playbooks/backup-cloud.yml @@ -111,7 +111,21 @@ set -euo pipefail export KUBECONFIG=/etc/rancher/k3s/k3s.yaml - TMPDIR=$(mktemp -d -t backup-cloud-XXXXXX) + # STABLE staging path — deliberately NOT mktemp. + # + # This directory is passed to `restic backup` as a source, so its name + # becomes part of the snapshot's path list. With mktemp the name was + # different every run (/tmp/backup-cloud-PX4ZQS, -pOvw3A, ...), and + # since `restic forget` groups by host+paths by default, every snapshot + # formed its own retention group. --keep-daily 7 --keep-weekly 4 + # --keep-monthly 3 then dutifully kept the single member of each group, + # so R2 retention deleted nothing, ever, on a 10 GB free tier. + # A fixed path also means every snapshot has the same tree layout, so + # restore procedures can hardcode paths. See docs/BACKUP-RESTORE.md. + TMPDIR=/var/tmp/backup-cloud-stage + rm -rf "$TMPDIR" + mkdir -p "$TMPDIR" + chmod 700 "$TMPDIR" trap "rm -rf $TMPDIR" EXIT log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*"; } @@ -146,6 +160,24 @@ --no-verify-ssl \ --quiet + # The k3s server token is REQUIRED to restore any etcd snapshot onto new + # hardware: k3s derives an AES-256 key from it (PBKDF2) to encrypt + # confidential data — CA private keys, bootstrap data — inside the + # datastore itself. Without it the snapshots in this very backup are + # unusable. It was previously backed up nowhere at all; it existed only on + # the three server nodes' local disks, so a total-site loss meant every + # etcd snapshot was decorative. + # + # SECURITY TRADEOFF, read before changing: snapshot + token together is + # full cluster compromise (see docs.k3s.io/cli/etcd-snapshot#security). + # They are co-located here only because this restic repo is encrypted as a + # whole, and its password lives in Vault and on the H4 — not in R2. Anyone + # able to decrypt this repo already holds the keys to everything in it. + # The offline break-glass envelope described in docs/BACKUP-RESTORE.md is + # still the primary recovery path; this is the convenience copy. + log "Copying k3s server token (required to restore etcd snapshots)..." + install -m 0600 /var/lib/rancher/k3s/server/token "$TMPDIR/k3s-server-token" + # Build source list — include vault-snapshots only if the directory exists SOURCES=( /mnt/cold-8t/k3s-etcd-snapshots @@ -157,7 +189,11 @@ restic backup "${SOURCES[@]}" --tag cloud --host h4-core log "Pruning old cloud snapshots..." - restic forget --prune \ + # --group-by host,tags (not the default host,paths): all cloud snapshots + # share host+tag, so they form ONE retention series. Belt-and-braces + # alongside the stable staging path above — if a future source path ever + # varies again, retention still works. + restic forget --prune --group-by host,tags \ --keep-daily 7 --keep-weekly 4 --keep-monthly 3 log "Cloud backup complete." diff --git a/ansible/playbooks/backup.yml b/ansible/playbooks/backup.yml index 09f450d..094a84c 100644 --- a/ansible/playbooks/backup.yml +++ b/ansible/playbooks/backup.yml @@ -1,9 +1,14 @@ --- # Two backup streams onto the cold tier (8TB primary + 6TB copy), via host systemd timers: -# - daily : restic snapshot of live NAS data (online, no downtime) -# - weekly: MicroShift data/etcd snapshot (brief downtime in a 03:00 window) +# - daily 01:30: restic snapshot of live NAS data (online, no downtime) +# - daily 00:30: copy of k3s' own etcd snapshots to cold (online, no downtime) # # Cluster *config* is already safe in git (Argo); these cover *data* and *state*. +# +# Note on the etcd stream: k3s takes embedded-etcd snapshots itself and keeps them +# on the NVMe. We do not snapshot — we replicate k3s' output onto the cold mirror +# and verify it. See ansible/templates/backup-etcd.sh.j2 for the full history of +# why this used to be a sqlite3 call and why it silently produced nothing. - name: Configure backups to the cold tier hosts: microshift become: true @@ -11,8 +16,26 @@ restic_repo: "{{ cold_primary_mount }}/restic" # 8TB primary restic_password_file: /etc/restic/password # vaulted; created out of band restic_copy_repo: "{{ cold_secondary_mount }}/restic" # 6TB redundant copy - etcd_backup_dir: "{{ cold_primary_mount }}/microshift-backups" restic_offsite_repo: "{{ offsite_restic_repo | default('') }}" # e.g. b2:bucket:lab — set via host/group vars or Vault; creds in /etc/restic/offsite.env + + # --- k3s etcd snapshot replication --- + k3s_snapshot_dir: /var/lib/rancher/k3s/server/db/snapshots # k3s writes here (NVMe) + etcd_snapshot_dest: "{{ cold_primary_mount }}/k3s-etcd-snapshots" + etcd_snapshot_keep: 30 # ~15 days at k3s' 12h cadence; ~1 GB at 32 MB each + etcd_snapshot_min_bytes: 1048576 # 1 MiB — a real snapshot is ~30 MB; 4 KB was the bug + etcd_snapshot_max_age_seconds: 90000 # 25h, matching the LabBackupEtcdSilent alert + + # --- Retention for the cold-sec copy (M-new-1) --- + # Deliberately DEEPER than the primary's 7/4/6: cold-sec is the long-history + # archive, not a mirror. Until 2026-07-25 it was never pruned at all. + copy_keep_daily: 14 + copy_keep_weekly: 8 + copy_keep_monthly: 12 + + # --- Weekly verification --- + cloud_env_file: /etc/restic/cloud.env # written by backup-cloud.yml + cloud_stage_dir: /var/tmp/backup-cloud-stage # must match backup-cloud.yml's TMPDIR + verify_max_snapshot_age_seconds: 172800 # 48h — weekly check, daily backups tasks: - name: Install restic ansible.builtin.package: @@ -26,6 +49,14 @@ mode: "0700" # --- NAS data backup (restic, daily) --- + - name: Render restic copy-to-secondary script + ansible.builtin.template: + src: ../templates/backup-nas-copy.sh.j2 + dest: /usr/local/sbin/backup-nas-copy.sh + owner: root + group: root + mode: "0750" + - name: Render restic backup service + timer ansible.builtin.template: src: "../templates/{{ item }}.j2" @@ -35,8 +66,16 @@ - backup-nas.service - backup-nas.timer - # --- MicroShift state backup (weekly, maintenance window) --- - - name: Render etcd/data backup service + timer + # --- k3s etcd snapshot replication (daily) --- + - name: Render etcd snapshot copy script + ansible.builtin.template: + src: ../templates/backup-etcd.sh.j2 + dest: /usr/local/sbin/backup-etcd.sh + owner: root + group: root + mode: "0750" + + - name: Render etcd backup service + timer ansible.builtin.template: src: "../templates/{{ item }}.j2" dest: "/etc/systemd/system/{{ item }}" @@ -55,6 +94,24 @@ - backup-offsite.service - backup-offsite.timer + # --- Weekly verification that the backups are real (not merely running) --- + - name: Render backup verification script + ansible.builtin.template: + src: ../templates/backup-verify.sh.j2 + dest: /usr/local/sbin/backup-verify.sh + owner: root + group: root + mode: "0750" + + - name: Render backup verification service + timer + ansible.builtin.template: + src: "../templates/{{ item }}.j2" + dest: "/etc/systemd/system/{{ item }}" + mode: "0644" + loop: + - backup-verify.service + - backup-verify.timer + - name: Reload systemd ansible.builtin.systemd: daemon_reload: true @@ -68,3 +125,4 @@ - backup-nas.timer - backup-etcd.timer - backup-offsite.timer + - backup-verify.timer diff --git a/ansible/playbooks/healthchecks.yml b/ansible/playbooks/healthchecks.yml index 761f518..c846bcf 100644 --- a/ansible/playbooks/healthchecks.yml +++ b/ansible/playbooks/healthchecks.yml @@ -9,7 +9,8 @@ # 2. Create checks with the recommended settings below: # backup-cloud Period 25h, Grace 1h (daily 03:00 UTC on h4-core) # backup-nas Period 25h, Grace 1h (daily 01:30 UTC on h4-core) -# backup-etcd Period 8d, Grace 4h (weekly on h4-core) +# backup-etcd Period 25h, Grace 1h (daily 00:30 UTC on h4-core) +# backup-verify Period 8d, Grace 12h (weekly Sun 04:00 UTC on h4-core) # backup-vault Period 25h, Grace 1h (daily 02:30 UTC on rpi5) # h4-core Period 5min, Grace 5min (host-alive heartbeat) # 3. Store all ping URLs in Vault: @@ -40,6 +41,7 @@ - backup-cloud - backup-nas - backup-etcd + - backup-verify - h4-core tasks: @@ -74,6 +76,11 @@ owner: root group: root loop: "{{ h4_checks }}" + # Skip checks that have no URL in Vault yet, so adding a new check to + # h4_checks does not break the play before the operator has created the + # healthchecks.io check and run `vault kv patch secret/lab/healthchecks`. + # hc-ping already no-ops safely when a .url file is absent. + when: item in hc no_log: true - name: Install hc-ping helper @@ -110,6 +117,7 @@ - backup-cloud - backup-nas - backup-etcd + - backup-verify - name: Install healthchecks drop-ins for H4 backup services ansible.builtin.copy: @@ -124,6 +132,7 @@ - backup-cloud - backup-nas - backup-etcd + - backup-verify notify: reload systemd daemon - name: Install h4-heartbeat systemd service diff --git a/ansible/templates/backup-etcd.service.j2 b/ansible/templates/backup-etcd.service.j2 index 0ebeb52..845272e 100644 --- a/ansible/templates/backup-etcd.service.j2 +++ b/ansible/templates/backup-etcd.service.j2 @@ -1,16 +1,12 @@ [Unit] -Description=k3s state backup to cold tier (8TB) +Description=Copy k3s etcd snapshots to cold tier (8TB) RequiresMountsFor={{ cold_primary_mount }} [Service] Type=oneshot -# k3s single-node uses SQLite (state.db) — no embedded etcd. -# sqlite3 .backup does an online hot-copy (safe while k3s is running). -# Keeps the last 7 copies; prunes older ones. -ExecStart=/bin/bash -c '\ - DB=/var/lib/rancher/k3s/server/db/state.db && \ - DEST={{ cold_primary_mount }}/k3s-etcd-snapshots && \ - SNAP="$DEST/state-$(date +%%F_%%H%%M).db" && \ - mkdir -p "$DEST" && \ - sqlite3 "$DB" ".backup $SNAP" && \ - ls -1t "$DEST"/state-*.db | tail -n +8 | xargs -r rm -f' +# k3s owns snapshotting itself (embedded etcd, its own schedule and retention, +# written to the NVMe). This unit only replicates those snapshots onto the cold +# mirror and verifies what landed. The logic lives in a script rather than an +# inline ExecStart so it can fail with a real exit code — see the header of +# backup-etcd.sh for why that matters here. +ExecStart=/usr/local/sbin/backup-etcd.sh diff --git a/ansible/templates/backup-etcd.sh.j2 b/ansible/templates/backup-etcd.sh.j2 new file mode 100644 index 0000000..dcc4042 --- /dev/null +++ b/ansible/templates/backup-etcd.sh.j2 @@ -0,0 +1,108 @@ +#!/bin/bash +# {{ ansible_managed | default('Managed by Ansible — do not edit by hand') }} +# +# Replicate k3s embedded-etcd snapshots onto the cold tier. +# +# k3s takes its own etcd snapshots on a schedule (default every 12h, retention 5) +# into ${SRC} on the NVMe. That is the *same failure domain* as the cluster they +# protect, so this script's only job is to copy them onto the cold mirror. It does +# NOT take snapshots itself — k3s owns that. +# +# History (2026-07-25): this unit previously ran +# sqlite3 /var/lib/rancher/k3s/server/db/state.db ".backup ..." +# which was correct only while k3s used the SQLite datastore. The 2026-07-02 HA +# migration switched the H4 to embedded etcd and renamed state.db to +# state.db.migrated. sqlite3 then silently recreated an empty state.db and emitted +# 4 KB "backups" every week while still exiting 0 — so the unit reported success, +# healthchecks.io stayed green, and backup-cloud shipped the empty files offsite. +# Every failure path below therefore exits non-zero on purpose: a backup that +# cannot prove it worked must fail loudly. +# +# NOTE FOR EDITORS: this file is a Jinja2 template. Do not use bash's array-length +# syntax (dollar-brace-hash-name) here — brace followed by hash opens a Jinja +# comment and the template fails to render with "Missing end of comment tag". +# That is why the retention logic below uses a mktemp listing rather than an array. + +set -euo pipefail + +SRC={{ k3s_snapshot_dir | quote }} +DEST={{ etcd_snapshot_dest | quote }} +KEEP={{ etcd_snapshot_keep }} +MIN_BYTES={{ etcd_snapshot_min_bytes }} +MAX_AGE={{ etcd_snapshot_max_age_seconds }} + +if [ ! -d "$SRC" ]; then + echo "FAIL: k3s snapshot dir $SRC does not exist." >&2 + echo " Is this host still an etcd server node? Check:" >&2 + echo " kubectl get node \$(hostname) -o jsonpath='{.metadata.labels}'" >&2 + exit 1 +fi + +mkdir -p "$DEST" + +# ---- Copy anything not already on cold ------------------------------------ +# k3s retains only ~5 snapshots (2.5 days at the 12h default), so a daily run +# that copies every not-yet-present file captures the full series without gaps. +shopt -s nullglob +copied=0 +seen=0 +for src_file in "$SRC"/etcd-snapshot-*; do + seen=$((seen + 1)) + base=$(basename "$src_file") + [ -e "$DEST/$base" ] && continue + # Stage then rename, so an interrupted run never publishes a truncated file. + cp -p -- "$src_file" "$DEST/.${base}.partial" + mv -- "$DEST/.${base}.partial" "$DEST/$base" + copied=$((copied + 1)) + echo "copied $base" +done + +if [ "$seen" -eq 0 ]; then + echo "FAIL: no etcd-snapshot-* files in $SRC — k3s is not producing snapshots." >&2 + echo " Check: journalctl -u k3s | grep -i snapshot" >&2 + exit 1 +fi + +# ---- Verify what actually landed ------------------------------------------ +# The old unit's fatal flaw was trusting exit 0. Prove freshness and size instead. +listing=$(mktemp) +trap 'rm -f -- "$listing"' EXIT +ls -1t -- "$DEST"/etcd-snapshot-* > "$listing" 2>/dev/null || true + +count=$(wc -l < "$listing" | tr -d '[:space:]') +newest=$(head -n 1 "$listing") + +if [ -z "$newest" ]; then + echo "FAIL: no snapshots present in $DEST after copy." >&2 + exit 1 +fi + +size=$(stat -c %s -- "$newest") +age=$(( $(date +%s) - $(stat -c %Y -- "$newest") )) + +if [ "$size" -lt "$MIN_BYTES" ]; then + echo "FAIL: newest snapshot $newest is only ${size} bytes (min ${MIN_BYTES})." >&2 + echo " A 4 KB file here means the empty-SQLite bug has regressed." >&2 + exit 1 +fi + +if [ "$age" -gt "$MAX_AGE" ]; then + echo "FAIL: newest snapshot $newest is ${age}s old (max ${MAX_AGE}s)." >&2 + echo " k3s has stopped snapshotting even though old files remain." >&2 + exit 1 +fi + +echo "ok: ${count} snapshots on cold, newest $(basename "$newest") (${size} bytes, ${age}s old), ${copied} new this run" + +# ---- Retention on the cold copy ------------------------------------------- +# Scoped to the etcd-snapshot-* glob inside $DEST only. Deliberately does NOT +# match the legacy state-*.db files — those are removed by hand, once, after +# review. Nothing here touches the restic repos; per CLAUDE.md, restic retention +# lives exclusively in backup-nas.service. +if [ "$count" -gt "$KEEP" ]; then + tail -n +$((KEEP + 1)) "$listing" | while IFS= read -r old; do + [ -n "$old" ] || continue + rm -f -- "$old" + echo "pruned $(basename "$old")" + done +fi diff --git a/ansible/templates/backup-etcd.timer.j2 b/ansible/templates/backup-etcd.timer.j2 index c429c28..95195ae 100644 --- a/ansible/templates/backup-etcd.timer.j2 +++ b/ansible/templates/backup-etcd.timer.j2 @@ -1,8 +1,15 @@ [Unit] -Description=Weekly k3s etcd state backup +Description=Daily copy of k3s etcd snapshots to cold tier [Timer] -OnCalendar=Sun *-*-* 03:00:00 +# Daily, not weekly. k3s retains only ~5 snapshots (2.5 days at its 12h default), +# so a weekly copy would sample one arbitrary point per week and let the rest age +# off the NVMe uncopied. +# +# 00:30 sits 30 min after k3s' 00:00 snapshot and well before backup-cloud (03:00), +# which uploads this directory to R2 — so the offsite copy always sees fresh data. +# The old Sun 03:00 slot collided with backup-cloud. +OnCalendar=*-*-* 00:30:00 Persistent=true [Install] diff --git a/ansible/templates/backup-nas-copy.sh.j2 b/ansible/templates/backup-nas-copy.sh.j2 new file mode 100644 index 0000000..8bfce38 --- /dev/null +++ b/ansible/templates/backup-nas-copy.sh.j2 @@ -0,0 +1,103 @@ +#!/bin/bash +# {{ ansible_managed | default('Managed by Ansible — do not edit by hand') }} +# +# Replicate the primary restic repo onto the secondary cold mirror, then apply +# retention to that copy. +# +# WHY THIS IS A SCRIPT AND NOT AN ExecStartPost ONE-LINER +# +# 1. restic changed the copy command's argument semantics. Older versions +# (the H4 runs 0.12.1) copy FROM the -r repo INTO --repo2: +# restic -r SRC copy --repo2 DST --password-file2 PW +# Current restic documents the opposite shape — the destination is -r and the +# source is named explicitly: +# restic -r DST copy --from-repo SRC --from-password-file PW +# --repo2 no longer appears in the current documentation at all. An unattended +# `apt upgrade` that crosses that boundary could therefore silently reverse the +# copy direction and overwrite the primary from the secondary. Rather than rely +# on how any given version interprets --repo2, this script detects the version +# and uses the form that is unambiguous for it. +# +# 2. The old ExecStartPost was prefixed with '-', so systemd ignored its exit +# status. The copy could fail every night without ever surfacing. It is now +# fatal: if replication breaks, backup-nas.service fails, hc-ping does not run, +# and healthchecks.io raises the alarm. +# +# 3. Retention never ran against the copy, so it grew without bound (40 snapshots +# vs the primary's 19 as of 2026-07-25). The copy is deliberately kept DEEPER +# than the primary — it is the longer-history archive — but it is now bounded. +# +# NOTE FOR EDITORS: this is a Jinja2 template. Do not use bash's array-length +# syntax (dollar-brace-hash-name); brace followed by hash opens a Jinja comment. + +set -euo pipefail + +SRC={{ restic_repo | quote }} +DST={{ restic_copy_repo | quote }} +PW={{ restic_password_file | quote }} + +KEEP_DAILY={{ copy_keep_daily }} +KEEP_WEEKLY={{ copy_keep_weekly }} +KEEP_MONTHLY={{ copy_keep_monthly }} + +if [ ! -r "$PW" ]; then + echo "FAIL: restic password file $PW is missing or unreadable." >&2 + exit 1 +fi + +# ---- Which copy syntax does this restic understand? ------------------------ +# "restic 0.12.1 compiled with go1.18.1 on linux/amd64" -> 0.12.1 +version_string=$(restic version | awk '{print $2}') +major=${version_string%%.*} +remainder=${version_string#*.} +minor=${remainder%%.*} + +# Must be numeric. A non-numeric value would make the [ -eq ] test below error +# out; because that test sits inside an `if` condition, set -e would NOT catch it +# and the script would silently fall through to the --from-repo branch. +if ! [[ "$major" =~ ^[0-9]+$ ]] || ! [[ "$minor" =~ ^[0-9]+$ ]]; then + echo "FAIL: could not parse a numeric restic version from '$version_string'." >&2 + echo " Refusing to guess which copy syntax is safe." >&2 + exit 1 +fi + +use_from_repo=1 +if [ "$major" -eq 0 ] && [ "$minor" -lt 14 ]; then + use_from_repo=0 +fi + +echo "restic ${version_string}: using $([ "$use_from_repo" -eq 1 ] && echo '--from-repo' || echo '--repo2') copy syntax" + +# ---- Ensure the destination repo exists ------------------------------------ +# NOTE: a repo initialised plainly will not share chunker parameters with the +# source, so deduplication across the two repos is imperfect. That is acceptable +# here (the copy is a mirror, not a space optimisation). If you ever rebuild the +# secondary from scratch and care about size, init it with --copy-chunker-params +# against the primary BEFORE the first copy — chunker params cannot be changed +# after the fact. +if ! restic -r "$DST" --password-file "$PW" cat config >/dev/null 2>&1; then + echo "destination repo $DST not initialised — initialising" + restic -r "$DST" --password-file "$PW" init +fi + +# ---- Copy ------------------------------------------------------------------- +if [ "$use_from_repo" -eq 1 ]; then + restic -r "$DST" --password-file "$PW" \ + copy --from-repo "$SRC" --from-password-file "$PW" +else + restic -r "$SRC" --password-file "$PW" \ + copy --repo2 "$DST" --password-file2 "$PW" +fi + +# ---- Retention on the copy -------------------------------------------------- +# Deliberately deeper than the primary's 7/4/6 — this repo is the long-history +# archive. Default grouping (host,paths) is correct here: the h4-core NAS +# snapshots and the lldap-k8s snapshots are retained as separate series. +# Per CLAUDE.md, retention lives in the backup units and nowhere else. Never run +# forget/prune against these repos by hand. +restic -r "$DST" --password-file "$PW" forget --prune \ + --keep-daily "$KEEP_DAILY" \ + --keep-weekly "$KEEP_WEEKLY" \ + --keep-monthly "$KEEP_MONTHLY" + +echo "copy + retention complete for $DST" diff --git a/ansible/templates/backup-nas.service.j2 b/ansible/templates/backup-nas.service.j2 index c7d4697..a392116 100644 --- a/ansible/templates/backup-nas.service.j2 +++ b/ansible/templates/backup-nas.service.j2 @@ -14,6 +14,9 @@ ExecStart=/usr/bin/restic backup {{ nas_mount }} {{ cold_primary_mount }}/VMs \ # Retention is policy-driven and lives HERE only — never run forget/prune by hand. ExecStartPost=/usr/bin/restic forget --prune \ --keep-daily 7 --keep-weekly 4 --keep-monthly 6 -# Redundancy: replicate the primary repo to the 6TB disk (two-disk safety, no RAID needed). -# restic copy treats --repo2 as fully independent, so it needs its own password source. -ExecStartPost=-/usr/bin/restic copy --repo2 {{ restic_copy_repo }} --password-file2 {{ restic_password_file }} +# Redundancy: replicate the primary repo to the 6TB disk (two-disk safety, no RAID +# needed), then bound the copy's retention. Deliberately NOT prefixed with '-' — +# a failed replication must fail this unit so hc-ping is skipped and +# healthchecks.io alerts. The script also picks the copy syntax matching the +# installed restic version; see its header for why that matters. +ExecStartPost=/usr/local/sbin/backup-nas-copy.sh diff --git a/ansible/templates/backup-verify.service.j2 b/ansible/templates/backup-verify.service.j2 new file mode 100644 index 0000000..d94a494 --- /dev/null +++ b/ansible/templates/backup-verify.service.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Weekly verification that backups are real (not merely running) +RequiresMountsFor={{ cold_primary_mount }} {{ cold_secondary_mount }} + +[Service] +Type=oneshot +# Read-only. Never writes to, forgets from, or prunes any repository. +# Exits non-zero if any check fails, so the healthchecks.io drop-in is skipped +# and the dead-man switch fires. +ExecStart=/usr/local/sbin/backup-verify.sh diff --git a/ansible/templates/backup-verify.sh.j2 b/ansible/templates/backup-verify.sh.j2 new file mode 100644 index 0000000..a012b4e --- /dev/null +++ b/ansible/templates/backup-verify.sh.j2 @@ -0,0 +1,120 @@ +#!/bin/bash +# {{ ansible_managed | default('Managed by Ansible — do not edit by hand') }} +# +# Weekly verification that the backups are not merely running, but real. +# +# This exists because three separate mechanisms in this lab were found reporting +# success while doing nothing (2026-07-24 review, docs/REVIEW-2026-07-24.md): +# - backup-etcd snapshotted a SQLite file that no longer existed, producing +# 4 KB files and pinging healthchecks.io green for a month; +# - backup-offsite rendered to a no-op that logged "skipping" and exited 0; +# - backup-cloud's retention never deleted anything because a random staging +# path put every snapshot in its own retention group. +# In every case the *job* was healthy and the *outcome* was not. This script +# checks outcomes. +# +# It is read-only. It never writes to, forgets from, or prunes any repository. +# +# NOTE FOR EDITORS: this is a Jinja2 template. Do not use bash's array-length +# syntax (dollar-brace-hash-name); brace followed by hash opens a Jinja comment. + +set -uo pipefail # NOT -e: we want every check to run, then report all failures + +PRIMARY={{ restic_repo | quote }} +COPY={{ restic_copy_repo | quote }} +PW={{ restic_password_file | quote }} +CLOUD_ENV={{ cloud_env_file | quote }} +ETCD_DIR={{ etcd_snapshot_dest | quote }} + +failures=0 +fail() { echo "FAIL: $*" >&2; failures=$((failures + 1)); } +pass() { echo "ok: $*"; } + +# ---- 1. Structural integrity of the two local repos ------------------------ +for repo in "$PRIMARY" "$COPY"; do + if restic -r "$repo" --password-file "$PW" check > /tmp/verify-check.$$ 2>&1; then + pass "restic check clean — $repo" + else + fail "restic check FAILED — $repo" + sed 's/^/ /' /tmp/verify-check.$$ >&2 + fi + rm -f /tmp/verify-check.$$ +done + +# ---- 2. The two local repos should both have a recent snapshot ------------- +# The copy legitimately holds MORE snapshots than the primary (deeper retention), +# so we compare recency, not counts. +for repo in "$PRIMARY" "$COPY"; do + latest_epoch=$(restic -r "$repo" --password-file "$PW" snapshots --tag nas --json 2>/dev/null \ + | tr ',' '\n' | grep -o '"time":"[^"]*"' | tail -1 | cut -d'"' -f4 \ + | xargs -r -I{} date -d {} +%s 2>/dev/null) + if [ -z "${latest_epoch:-}" ]; then + fail "could not determine newest 'nas' snapshot in $repo" + else + age=$(( $(date +%s) - latest_epoch )) + if [ "$age" -gt {{ verify_max_snapshot_age_seconds }} ]; then + fail "newest 'nas' snapshot in $repo is ${age}s old (max {{ verify_max_snapshot_age_seconds }}s)" + else + pass "newest 'nas' snapshot in $repo is ${age}s old" + fi + fi +done + +# ---- 3. Cold etcd snapshots are real, not the 4 KB ghosts ------------------ +newest_etcd=$(ls -1t -- "$ETCD_DIR"/etcd-snapshot-* 2>/dev/null | head -n 1) +if [ -z "$newest_etcd" ]; then + fail "no etcd-snapshot-* files in $ETCD_DIR" +else + etcd_size=$(stat -c %s -- "$newest_etcd") + if [ "$etcd_size" -lt {{ etcd_snapshot_min_bytes }} ]; then + fail "newest etcd snapshot is only ${etcd_size} bytes — the empty-file bug is back" + else + pass "newest etcd snapshot $(basename "$newest_etcd") is ${etcd_size} bytes" + fi +fi + +# ---- 4. R2: integrity, and that it actually contains cluster state --------- +# The content assertion is the important half. A clean `check` only proves the +# repo is well-formed; it says nothing about whether the right files are in it. +# backup-cloud sources /mnt/cold-8t/k3s-etcd-snapshots, so if that directory were +# ever wrong again, R2 would faithfully back up the wrong thing and still pass. +if [ ! -r "$CLOUD_ENV" ]; then + echo "warn: $CLOUD_ENV not present — skipping R2 checks (run backup-cloud.yml?)" >&2 +else + # shellcheck disable=SC1090 + set -a; . "$CLOUD_ENV"; set +a + + if restic check > /tmp/verify-cloud.$$ 2>&1; then + pass "restic check clean — R2" + else + fail "restic check FAILED — R2" + sed 's/^/ /' /tmp/verify-cloud.$$ >&2 + fi + rm -f /tmp/verify-cloud.$$ + + etcd_in_cloud=$(restic ls latest "$ETCD_DIR" 2>/dev/null | grep -c 'etcd-snapshot-') + if [ "${etcd_in_cloud:-0}" -lt 1 ]; then + fail "newest R2 snapshot contains NO etcd-snapshot-* files under $ETCD_DIR" + else + pass "newest R2 snapshot contains ${etcd_in_cloud} etcd-snapshot-* files" + fi + + # The k3s server token must be in the backup or the etcd snapshots above cannot + # be restored onto new hardware — k3s derives the datastore's encryption key + # from it. Until 2026-07-25 it was backed up nowhere. + if restic ls latest {{ cloud_stage_dir | quote }} 2>/dev/null | grep -q 'k3s-server-token'; then + pass "newest R2 snapshot contains the k3s server token" + else + fail "newest R2 snapshot has NO k3s server token — etcd snapshots are unrestorable on new hardware" + fi + + cloud_count=$(restic snapshots --tag cloud --json 2>/dev/null | grep -o '"time":' | wc -l) + echo "info: R2 holds ${cloud_count} cloud snapshots" +fi + +# ---- Verdict ---------------------------------------------------------------- +if [ "$failures" -gt 0 ]; then + echo "backup-verify: ${failures} check(s) FAILED" >&2 + exit 1 +fi +echo "backup-verify: all checks passed" diff --git a/ansible/templates/backup-verify.timer.j2 b/ansible/templates/backup-verify.timer.j2 new file mode 100644 index 0000000..bb8e5a5 --- /dev/null +++ b/ansible/templates/backup-verify.timer.j2 @@ -0,0 +1,12 @@ +[Unit] +Description=Weekly backup verification + +[Timer] +# Sunday 04:00 — after backup-etcd (00:30), backup-nas (01:30), backup-vault +# (02:30) and backup-cloud (03:00), so it inspects the current night's results. +OnCalendar=Sun *-*-* 04:00:00 +RandomizedDelaySec=300 +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/docs/BACKUP-RESTORE.md b/docs/BACKUP-RESTORE.md new file mode 100644 index 0000000..13422f8 --- /dev/null +++ b/docs/BACKUP-RESTORE.md @@ -0,0 +1,367 @@ +# Backups, Restores, and Validation + +The authoritative reference for what is backed up, how to get it back, and how to +prove any of it works. Supersedes the backup and restore sections of +[RUNBOOK.md](RUNBOOK.md). + +> **Guiding principle, learned the hard way.** In July 2026 a review found three +> separate mechanisms in this lab reporting success while doing nothing: an etcd +> backup that snapshotted a file deleted a month earlier, an offsite timer that +> rendered to `exit 0`, and a cloud retention policy that never deleted anything. +> All three passed every check that existed. **A backup you have not restored is a +> hypothesis, not a backup.** Everything below is built to fail loudly rather than +> succeed quietly. + +--- + +## 1. What is backed up + +| Stream | When | Source | Destination | Retention | +|---|---|---|---|---| +| `backup-etcd` | daily 00:30 | k3s' own etcd snapshots from `/var/lib/rancher/k3s/server/db/snapshots/` | `/mnt/cold-8t/k3s-etcd-snapshots/` | 30 files (~15 days) | +| `backup-nas` | daily 01:30 | `/srv/nas`, `/mnt/cold-8t/VMs`, `/mnt/cold-8t/immich` | restic → `/mnt/cold-8t/restic` | 7d / 4w / 6m | +| `backup-nas` (copy step) | daily 01:30 | the primary restic repo | restic → `/mnt/cold-sec/restic` | 14d / 8w / 12m | +| `backup-vault` | daily 02:30 | Vault raft snapshot (on rpi5) | `/mnt/cold-8t/vault-snapshots/` | 30 days | +| `lldap-backup` (k8s CronJob) | daily 02:30 | lldap `users.db` | restic → `/mnt/cold-8t/restic` via NFS | via `backup-nas` policy | +| Immich DB dump (k8s CronJob) | daily 01:30 | `pg_dump` | `/mnt/cold-8t/immich/backups/` | captured by `backup-nas` | +| `backup-cloud` | daily 03:00 | etcd snapshots, Vault snapshots, lldap DB, Postgres dumps (Authelia/Immich/Semaphore), OpenTofu state, **k3s server token** | restic → Cloudflare R2 `homelab-backup` | 7d / 4w / 3m | +| `backup-verify` | weekly Sun 04:00 | *verification only, read-only* | — | — | + +`backup-offsite.timer` also exists but is a **no-op** — `restic_offsite_repo` +defaults to empty and the unit logs `skipping` and exits 0. `backup-cloud` → R2 is +the real offsite. Delete one of the two; see REVIEW-2026-07-24.md H20. + +### The three tiers + +- **HOT** — NVMe. k3s' own etcd snapshots (12h cadence, 5 retained ≈ 2.5 days) and + live data. Same failure domain as the cluster; not a backup. +- **COLD** — two mdadm RAID 1 mirrors on the H4. `/mnt/cold-8t` is the + copy-of-record; `/mnt/cold-sec` is a deeper-history restic copy. Survives disk + failure, **not** loss of the H4. +- **OFFSITE** — Cloudflare R2, encrypted by restic. The only tier that survives + losing the box, the building, or both. + +--- + +## 2. Things that will ruin your day if you don't know them + +### The k3s server token is required to restore etcd + +k3s derives an AES-256 key from the server join token (PBKDF2) and uses it to +encrypt confidential data — CA private keys, bootstrap data — **inside the +datastore itself**. A snapshot without its token cannot be restored onto new +hardware. Per +[k3s docs](https://docs.k3s.io/datastore/backup-restore): *"If you do not use the +same token value when restoring, the snapshot will be unusable."* + +Where it lives: + +- `/var/lib/rancher/k3s/server/token` on **all three** server nodes (h4-core, + n150-1, n150-2). Losing one box is therefore survivable. +- Inside the R2 backup as `k3s-server-token` (added 2026-07-25 — before that it + was backed up nowhere, and every offsite etcd snapshot was decorative). +- It **should** also be in your offline envelope (below). + +### Snapshot + token together is total compromise + +Anyone holding both can extract every secret and the cluster CA private keys. They +are co-located in the R2 repo only because that repo is encrypted as a whole and +its password is not stored in R2. Treat the R2 restic password as a crown jewel. + +### The break-glass envelope + +Recovery from total loss needs credentials that are themselves stored inside the +thing you lost. Keep an **offline copy** — printed, or on an encrypted USB kept +off-site — of: + +1. The restic password (`/etc/restic/password`) +2. The R2 restic password (`/etc/restic/cloud-password`) and the R2 API credentials +3. The k3s server token +4. The Vault unseal keys and root token +5. The Ansible vault password + +Without #1 and #2 the backups are undecryptable ciphertext. Without #3 the etcd +snapshots are unrestorable. Without #4 every ExternalSecret stays empty. + +### Vault is a circular dependency + +`RUNBOOK.md` says to get the restic password from Vault. Vault runs on rpi5 and its +raft snapshots live on `/mnt/cold-8t` — on the H4. If the H4 is gone you need +restic to recover, and you need Vault to get the restic password. **Break the loop +with the offline envelope.** This is not yet solved in automation. + +--- + +## 3. Restores + +> Read the whole procedure before starting. Restores are the one operation with no +> undo. + +### 3.1 Restore a file or directory from the NAS backup + +Most common case, entirely safe — restic restores never touch the source. + +```bash +# Find the snapshot +sudo restic -r /mnt/cold-8t/restic --password-file /etc/restic/password snapshots --tag nas + +# Look inside one +sudo restic -r /mnt/cold-8t/restic --password-file /etc/restic/password ls /srv/nas + +# Restore to a scratch path — NEVER restore over live data on the first attempt +sudo restic -r /mnt/cold-8t/restic --password-file /etc/restic/password \ + restore --target /mnt/cold-8t/restore-scratch --include /srv/nas/somedir + +# Verify, then move into place by hand +``` + +If `/mnt/cold-8t` is unreadable, substitute `/mnt/cold-sec/restic` — same password, +deeper history. + +### 3.2 Restore cluster state (etcd) — 3-server HA + +**This supersedes the procedure previously in RUNBOOK.md, which was wrong**: it used +a non-existent `k3s etcd-snapshot restore` subcommand, pointed at the wrong +directory, and gave the single-node steps for a 3-server cluster. + +Verified against [k3s docs](https://docs.k3s.io/cli/etcd-snapshot#restoring-snapshots). +Nodes below: **S1** = odroid-nas, **S2** = n150-1, **S3** = n150-2. + +```bash +# 0. Pick a snapshot and confirm it is real (~30 MB, not 4 KB) +ls -lh /mnt/cold-8t/k3s-etcd-snapshots/ + +# 1. Stop k3s on ALL THREE servers +sudo systemctl stop k3s # on S1, S2 and S3 + +# 2. On S1 ONLY — reset the cluster and restore +sudo k3s server \ + --cluster-reset \ + --cluster-reset-restore-path=/mnt/cold-8t/k3s-etcd-snapshots/ + +# Wait for: "Managed etcd cluster membership has been reset, restart without +# --cluster-reset flag now." Then Ctrl-C. + +# 3. On S1 — start normally +sudo systemctl start k3s +kubectl get nodes # S1 Ready; S2/S3 NotReady — expected + +# 4. On S2 and S3 — discard their old datastore +sudo rm -rf /var/lib/rancher/k3s/server/db/ + +# 5. On S2 and S3 — rejoin +sudo systemctl start k3s +kubectl get nodes # all three Ready +``` + +k3s moves the existing database to `${data-dir}/server/db/etcd-old-$TIMESTAMP/` +rather than deleting it, and writes `/var/lib/rancher/k3s/server/db/reset-flag` to +prevent a second accidental reset. That file clears on normal start. + +**Restoring onto new hardware** additionally requires the token: + +```bash +sudo k3s server \ + --cluster-reset \ + --cluster-reset-restore-path= \ + --token= +``` + +Node objects are inside the snapshot, so after restoring to new machines you must +`kubectl delete node ` for hosts that no longer exist. + +**Workloads need no restore.** Argo CD reconciles everything in `gitops/` from git +with `selfHeal`. Restoring etcd brings back cluster state; Argo brings back apps. +PVC *data* is a separate matter — that comes from restic. + +### 3.3 Restore from R2 (total loss of the H4) + +```bash +export RESTIC_REPOSITORY='s3:https://.r2.cloudflarestorage.com/homelab-backup' +export AWS_ACCESS_KEY_ID= # from the envelope +export AWS_SECRET_ACCESS_KEY= +export RESTIC_PASSWORD= + +restic snapshots --tag cloud +restic restore latest --target /recovery +``` + +`/recovery` then contains the etcd snapshots, `k3s-server-token`, Vault raft +snapshots, Postgres dumps, the lldap DB, and OpenTofu state. Rebuild order: + +1. Base OS + storage — `ansible/playbooks/storage.yml` (run `--check` first) +2. k3s — `ansible/playbooks/k3s-h4.yml`, then restore etcd per §3.2 using the + recovered token +3. Vault — restore the raft snapshot (§3.4), unseal with the envelope keys +4. Argo CD — `ansible/playbooks/argocd.yml`; workloads reconcile from git +5. Postgres/lldap — restore the dumps into the running pods +6. NAS data — restic restore from cold, or R2 if the cold disks are also gone + +> **Untested.** No one has performed this end to end. Until someone does, treat the +> timings and the ordering as informed guesses. See §5. + +### 3.4 Restore Vault + +Live backups are raft snapshots in `/mnt/cold-8t/vault-snapshots/vault-snap-*.snap` +(30-day retention). + +```bash +ls -lt /mnt/cold-8t/vault-snapshots/ | head +vault operator raft snapshot restore /path/to/vault-snap-.snap +# then unseal with the keys from the envelope +``` + +> `ansible/playbooks/vault-restore.yml` is **broken** — it hardcodes a dated tarball +> (`vault-backup-20260627.tar.gz`) in a directory nothing maintains, and uses a +> controller-side `src` for a file that lives on the H4. Do not use it until fixed. +> See REVIEW-2026-07-24.md H21. + +### 3.5 Restore a Postgres database + +Dumps land in the R2 backup and (for Immich) on cold. + +```bash +gunzip -c /recovery/var/tmp/backup-cloud-stage/authelia.sql.gz \ + | kubectl exec -i -n authelia deploy/authelia-postgres -- \ + sh -c 'PGPASSWORD=$POSTGRES_PASSWORD psql -U $POSTGRES_USER $POSTGRES_DB' +``` + +Same shape for `semaphore.sql.gz` and `immich.sql.gz`. + +--- + +## 4. Manual validation + +### 4.1 The one-command check + +`backup-verify` runs weekly and does everything in §4.2 automatically. Run it by +hand any time — it is strictly read-only and never writes to, forgets from, or +prunes any repository: + +```bash +sudo systemctl start backup-verify.service +systemctl status backup-verify.service +``` + +Every line is either `ok:` or `FAIL:`, and a single failure makes the unit exit +non-zero, which suppresses the healthchecks.io ping and raises the alarm. + +### 4.2 Checking by hand + +**Are the timers alive?** + +```bash +systemctl list-timers 'backup-*' +systemctl status backup-etcd backup-nas backup-cloud backup-verify +``` + +**Are the etcd snapshots real?** The original bug produced 4 KB files for a month. +Real snapshots are ~30 MB. + +```bash +ls -lh /mnt/cold-8t/k3s-etcd-snapshots/ +sudo k3s etcd-snapshot ls +``` + +**Is the cold copy byte-identical?** + +```bash +sudo sha256sum \ + /var/lib/rancher/k3s/server/db/snapshots/ \ + /mnt/cold-8t/k3s-etcd-snapshots/ +``` + +**Do the restic repos have recent snapshots, and are they intact?** + +```bash +for r in /mnt/cold-8t/restic /mnt/cold-sec/restic; do + echo "== $r" + sudo restic -r "$r" --password-file /etc/restic/password snapshots --tag nas | tail -3 + sudo restic -r "$r" --password-file /etc/restic/password check +done +``` + +cold-sec legitimately holds **more** snapshots than the primary — the primary is +pruned nightly to 7d/4w/6m while the copy keeps 14d/8w/12m. More on the copy is +correct; fewer is a bug. + +**Is R2 healthy, and does it hold what actually matters?** + +```bash +sudo bash -c 'set -a; . /etc/restic/cloud.env; set +a; restic snapshots --tag cloud | tail -5' +sudo bash -c 'set -a; . /etc/restic/cloud.env; set +a; restic check' + +# The content assertions — a clean check proves the repo is well-formed, +# NOT that the right files are in it. +sudo bash -c 'set -a; . /etc/restic/cloud.env; set +a; \ + restic ls latest /mnt/cold-8t/k3s-etcd-snapshots' +sudo bash -c 'set -a; . /etc/restic/cloud.env; set +a; \ + restic ls latest /var/tmp/backup-cloud-stage | grep k3s-server-token' + +# Free-tier headroom (10 GB) +sudo bash -c 'set -a; . /etc/restic/cloud.env; set +a; restic stats --mode raw-data' +``` + +**Deep integrity check** (re-reads stored data; catches bit-rot the structural check +misses; slow): + +```bash +sudo restic -r /mnt/cold-8t/restic --password-file /etc/restic/password \ + check --read-data-subset=5% +``` + +### 4.3 Never do these by hand + +Retention lives in the backup units and nowhere else. Running these manually can +destroy the copy-of-record: + +- `restic forget` / `restic prune` on any repo +- `mkfs` / `wipefs` on `/dev/md0` or `/dev/md1` +- stopping `smbd`, `nfs-kernel-server`, or disabling any `backup-*` timer + +### 4.4 The restore drill + +Structural checks cannot tell you a restore works. Once a quarter: + +1. Restore one file from the NAS backup to a scratch dir and diff it against live. +2. Restore one etcd snapshot into a throwaway VM and confirm `kubectl get nodes`. +3. Restore a Postgres dump into a scratch database and count rows. +4. Decrypt one R2 snapshot **using only the offline envelope** — no Vault, no H4. + This is the only test that actually validates total-loss recovery. + +Record the date, duration, and anything surprising in the log below. + +| Date | What was tested | Result | Time taken | Notes | +|---|---|---|---|---| +| — | *no restore has ever been tested* | — | — | See §5 | + +--- + +## 5. Known gaps + +Honest list. Tracked in [REVIEW-2026-07-24.md](REVIEW-2026-07-24.md). + +| Gap | Impact | Ref | +|---|---|---| +| **No restore has ever been performed** | The entire chain above is untested. This is the single largest risk in the lab. | C2 | +| Vault ↔ restic circular dependency | Total-loss recovery depends on the offline envelope existing and being current | H7 | +| `vault-restore.yml` is broken | No working automated Vault restore | H21 | +| `backup-offsite` is a no-op competing with `backup-cloud` | Two offsite mechanisms, one fake | H20 | +| restic 0.12.1 (2021) | Upgrading past 0.14 silently reverses the cold-sec copy direction unless the version guard in `backup-nas-copy.sh` catches it | M-new-2 | +| cold-8t is one box | Cold tier does not survive loss of the H4; only R2 does | C5 | +| No alert reaches a human | Alertmanager's only non-null receiver is an M5Stack display; healthchecks.io covers timers only | H25 | + +--- + +## 6. File map + +| Thing | Path | +|---|---| +| etcd copy script | `ansible/templates/backup-etcd.sh.j2` → `/usr/local/sbin/backup-etcd.sh` | +| NAS→cold-sec copy + retention | `ansible/templates/backup-nas-copy.sh.j2` → `/usr/local/sbin/backup-nas-copy.sh` | +| Verification | `ansible/templates/backup-verify.sh.j2` → `/usr/local/sbin/backup-verify.sh` | +| NAS backup unit | `ansible/templates/backup-nas.{service,timer}.j2` | +| Cloud backup | `ansible/playbooks/backup-cloud.yml` (script inline) → `/usr/local/bin/backup-cloud.sh` | +| Vault backup | `ansible/playbooks/backup-vault.yml` | +| Dead-man switches | `ansible/playbooks/healthchecks.yml` | +| Restic passwords | `/etc/restic/password`, `/etc/restic/cloud-password` (0600 root, never in git) | diff --git a/docs/REVIEW-2026-07-24.md b/docs/REVIEW-2026-07-24.md new file mode 100644 index 0000000..9058205 --- /dev/null +++ b/docs/REVIEW-2026-07-24.md @@ -0,0 +1,441 @@ +# Lab review — 2026-07-24 + +Full-environment review: `gitops/`, `ansible/`, `tofu/`, `packer/`, CI, `docs/`, `.claude/`. +Findings only — no changes made. Every claim below was read from the file cited. + +## Verdict + +The design is sound and unusually well documented for a home lab. The GitOps discipline is +real (no OpenShift residue, correct StorageClass everywhere, Kyverno actually enforcing, +ESO for all secrets). The weakness is not architecture — it is **verification**. Several +mechanisms the lab depends on report success while doing nothing, and the CI that would +have caught them cannot fail. + +Three themes: + +1. **The backup story is partly fictional.** Cluster state, the secondary cold copy, and + the offsite leg are each broken in a way that returns exit 0. +2. **The HA story stops at the API server.** Ingress, NFS, and Vault are still single + points of failure on the H4/rpi5 — the failover that exists doesn't cover the paths you + would actually use during an outage. +3. **Guardrails have drifted out from under the docs.** CI jobs, alert routing, and + `.claude/settings.json` all describe protection that isn't wired up. + +--- + +## Critical + +> **Update 2026-07-25 — verification pass run on the H4.** C1 **confirmed** (worse than +> written). C6 **largely disproved** — the cold-sec copy is working. C7 **downgraded** — +> never committed. H20 **confirmed**. See the inline notes on each. + +### C1. `backup-etcd` snapshots a file that hasn't existed since the HA migration — PATCHED + +> **FIXED and verified on the H4 2026-07-25**, branch `fix/backup-etcd-embedded-etcd`. +> `backup-etcd` now copies k3s' own snapshots and verifies size + freshness, failing loudly +> on both; timer moved weekly → daily 00:30. First run copied 5 snapshots (29–32 MB each); +> `sha256sum` confirms the cold copy is byte-identical to the k3s original, and `hc-ping` +> now fires only *after* a verified success. n150-1 independently holds 31 MB snapshots on +> the same 12h cadence, so cluster state already survives H4 loss. +> +> healthchecks.io period corrected to 25h/1h and the legacy `state-*.db` files plus the stray +> zero-byte `/var/lib/rancher/k3s/server/db/state.db` removed (closing M-new-3). Retained by +> decision: `etcd-2026-06-24_1739-...` (12 MB), the only genuine etcd snapshot from the +> outage window. + +> **CONFIRMED 2026-07-25.** `/mnt/cold-8t/k3s-etcd-snapshots/` contains: +> `etcd-2026-06-24_1739-...` (12M, a real `k3s etcd-snapshot save`, one-off), +> `state-2026-06-25` (19M) and `state-2026-06-28` (421M) from a stale leftover `state.db`, +> then **4.0K every week from 07-05 onward** — empty SQLite databases. `sqlite3` creates one +> when the source path is missing; `.backup` on it succeeds, so the unit exits +> `status=0/SUCCESS` and `hc-ping` reports OK. healthchecks.io has been green for a month +> while producing nothing, and `backup-cloud.yml` has been shipping the 4K files to R2. +> **Last usable cluster-state backup: 2026-06-24, taken manually.** The two SQLite files are +> pre-HA-migration and cannot restore an embedded-etcd cluster. +> +> **Root cause dated exactly.** `/var/lib/rancher/k3s/server/db/` contains +> `state.db` at **0 bytes, mtime Jul 5 03:00** — `sqlite3` creating the file when the backup +> unit opened a path k3s had retired, at the first post-migration run — and +> `state.db.migrated` (22MB, **Jul 2 20:19**), k3s renaming the old SQLite datastore during +> the conversion to embedded etcd. That timestamp matches the documented HA migration date. +> The unit has also been littering a stray `state.db` into the live datastore directory +> every Sunday since. +> +> **k3s' own snapshots are healthy**, which makes the fix small: +> `/var/lib/rancher/k3s/server/db/snapshots/` holds 5 × 29–32MB snapshots on a 12h cadence, +> newest `Jul 25 00:00`. Do not reimplement snapshotting — copy these to cold. Two caveats: +> retention 5 × 12h = **2.5 days of history** against a *weekly* timer, so make the copy +> daily (32MB); and they live on the NVMe, the same failure domain as the cluster they +> protect, so either copy to cold or set `etcd-snapshot-dir` to the cold mount directly. +> +> **Check n150-1 / n150-2** — every embedded-etcd server takes its own snapshots. If theirs +> are also running, cluster state already survives H4 loss, which materially improves C2. + +`ansible/templates/backup-etcd.service.j2:7-16` runs +`sqlite3 /var/lib/rancher/k3s/server/db/state.db ".backup ..."`, with a comment stating +"k3s single-node uses SQLite — no embedded etcd". But `ansible/playbooks/k3s-h4.yml:34` +sets `cluster-init: true`, so k3s has run **embedded etcd** since 2026-07-02. `state.db` +is not present; the datastore is `db/etcd/`. + +Knock-on: `ansible/playbooks/backup-cloud.yml:151` ships `/mnt/cold-8t/k3s-etcd-snapshots` +to R2 as the offsite copy of cluster state — so the offsite copy is empty too. + +- **Verify first:** `ls -lh /mnt/cold-8t/k3s-etcd-snapshots/` on the H4. Expect multi-MB + files; 0-byte or absent confirms it. +- **Fix:** `ExecStart` → `k3s etcd-snapshot save --name lab --dir {{ cold_primary_mount }}/k3s-etcd-snapshots`, + or set `etcd-snapshot-schedule-cron` in `/etc/rancher/k3s/config.yaml` on all three servers. +- **Branch:** `fix/backup-etcd-embedded-etcd` + +### C8. The k3s server token was backed up nowhere — every etcd snapshot was unrestorable + +*(found 2026-07-25, while writing the restore procedure)* + +k3s derives an AES-256 key from the server join token via PBKDF2 and uses it to +encrypt confidential data — CA private keys, bootstrap data — **inside the datastore +itself**. Per [k3s docs](https://docs.k3s.io/datastore/backup-restore): *"If you do +not use the same token value when restoring, the snapshot will be unusable."* +Restoring onto new hardware requires `--token=`. + +`/var/lib/rancher/k3s/server/token` appeared in **no** backup: not `backup-nas` +(which covers only `/srv/nas`, `/mnt/cold-8t/VMs`, `/mnt/cold-8t/immich`), not +`backup-etcd`, not `backup-cloud`, and not Vault. It existed solely on the three +server nodes' local disks. + +Losing the H4 alone was survivable — n150-1 and n150-2 hold the same token. But a +total-site loss meant the R2 offsite copy, the one tier that exists precisely for +that scenario, held etcd snapshots that could never be restored. The backups +verified clean and were decorative. + +- **Fixed 2026-07-25:** `backup-cloud.yml` now stages the token into the R2 backup, + and `backup-verify` asserts its presence. Security tradeoff (snapshot + token + together = full compromise) documented inline and in `docs/BACKUP-RESTORE.md` §2. +- **Still required of the operator:** the offline break-glass envelope — + `docs/BACKUP-RESTORE.md` §2. Automation cannot solve the circular dependency + where recovery credentials live inside the thing you lost. + +### C2. Nothing has ever been restored, and the documented restore command is invalid + +> **Procedure fixed 2026-07-25; testing still outstanding.** A correct, docs-verified +> restore procedure now lives in `docs/BACKUP-RESTORE.md` §3 — the 3-server HA reset +> sequence, restoring to new hardware with `--token`, R2 total-loss recovery, Vault +> raft, and Postgres. What remains is that **nobody has run one**. §4.4 holds the +> quarterly drill and an empty results table awaiting its first entry. + +`docs/RUNBOOK.md:211-216` — `k3s etcd-snapshot restore ` is not a valid subcommand +(it's `k3s server --cluster-reset --cluster-reset-restore-path=...`), the path given +(`/mnt/cold-8t/etcd/`) doesn't match the actual snapshot dir, and the procedure is +single-node applied to a 3-server embedded-etcd cluster. Real recovery requires stopping +k3s on all three, `--cluster-reset` on one, wiping `db/etcd` on the other two, rejoining. + +There is no bare-metal H4 rebuild procedure at all (eMMC reflash → LVM loop reattach → +RAID assemble → restic password recovery → k3s rejoin → NFS re-export), and repo-wide there +is no `restic check`, no test restore, no drill log. The only `restic check` in the repo is +prose at `docs/OPS.md:539`. + +- **Fix:** write the H4-total-loss runbook as an executable procedure, then actually run one + restic restore and one etcd restore into scratch this month and record the date/duration. + +### C3. Every `*.apps` URL dies with the H4 — including the tools you'd diagnose with + +`ansible/inventory/hosts.yml:39` sets `ingress_vip: 192.168.1.160` (the H4 itself), and +`gitops/workloads/kube-vip/daemonset.yaml:53` has `svc_enable: "false"` — kube-vip covers +the **API only**. `gitops/workloads/coredns-custom/configmap.yaml:18` hard-codes `.160` too, +so pods resolve it that way as well. + +When the H4 dies: etcd keeps quorum, pods reschedule perfectly onto n150-1/2 — and Grafana, +Alertmanager, and ArgoCD are all unreachable. + +- **Fix:** `svc_enable: "true"` + a Traefik `LoadBalancer` Service on a second VIP (e.g. + `.201`); repoint `ingress_vip`, the Pi-hole/dnsmasq wildcard, and the CoreDNS template. + +### C4. CI validation jobs are structurally incapable of failing + +`.gitlab-ci.yml:61-71` and `.github/workflows/validate.yml:31-38` both use +`FAILED=0; find ... | while read f; do ... || FAILED=1; done; [ "$FAILED" -eq 0 ] || exit 1`. +The `while` runs in a pipeline subshell, so `FAILED=1` is discarded — **the dry-run and +yaml-lint jobs always pass.** Only `kubeconform` and `conftest` (which use `xargs -0`) +are real gates. + +Compounding: all validation is scoped to `gitops/workloads/` — the 17 Argo `Application` +manifests in `gitops/apps/` and `gitops/bootstrap/root-app.yaml`, the highest-blast-radius +YAML in the repo, are never checked. + +- **Fix:** `find ... -print0 | xargs -0 -n1 ...` (or `while ... done < <(find ...)`), add + `set -euo pipefail`, widen the find root to `gitops/`. Cheap, and everything else relies + on this gate. + +### C5. GitLab CI job pods mount the host Docker socket read-write + +`gitops/workloads/gitlab-runner/values.yaml:40-45` — `host_path = "/var/run/docker.sock"`, +`read_only = false`, into every job pod on n150-1. Any pipeline gets root on the node, +trivially defeating the `no-privileged` ClusterPolicy. (The node runs containerd, so the +socket may not even exist — in which case this is dead config that still needs deleting.) + +Same file, lines 32-39: the k8s executor sets no `cpu_limit`/`memory_limit`, and +`gitlab-runner` is **not** excluded from `require-resource-limits` — so job pods are denied +at admission anyway. + +- **Fix:** delete the `host_path` block, use Kaniko/Buildah or the existing Zot flow; add + `cpu_limit`/`memory_limit`/`helper_*`. Do not add a policy exclusion. + +### C6. ~~The "second copy of record" is almost certainly never written~~ — WRONG, it works + +> **CLEARED 2026-07-25.** `/mnt/cold-sec/restic` is initialized and populated, `index/` and +> `snapshots/` carry mtime `Jul 25 01:30` matching `backup-nas.timer`, and the snapshot +> counts are **cold-sec 40 vs primary 19** — which is the *healthy* signature, not a gap: +> `forget --prune --keep-daily 7 --keep-weekly 4 --keep-monthly 6` runs on the primary each +> night, then the copy replicates and is never pruned. The secondary is a deeper archive +> than the primary. `restic 0.12.1` confirms pre-0.14 `--repo2` semantics, so the copy +> direction is correct. +> +> Two real issues surfaced by this, tracked below as M-new-1 and M-new-2. Also still valid: +> the `-` prefix on that `ExecStartPost` will hide the first real failure, and +> `storage.yml` never inits the copy repo, so a rebuild would not recreate it. + +
Original finding, retained for the record + +`ansible/templates/backup-nas.service.j2:19`: +`ExecStartPost=-/usr/bin/restic copy --repo2 {{ restic_copy_repo }} ...` + +Two problems. The secondary repo is never `restic init`-ed (`storage.yml:75` only creates +the directory), and the leading `-` makes systemd ignore the failure — so it has been +failing silently while `backup-nas` pings healthchecks green. Separately, restic ≥ 0.14 +inverted `copy`: `--repo2` became the *source*. Check `restic version` on the H4 before +rewriting. + +- **Verify first:** `restic -r /mnt/cold-sec/restic snapshots` +- **Fix:** init the copy repo, rewrite as `restic -r copy --from-repo `, + drop the `-` prefix so failures surface. + +
+ +### C7. `ansible/.vault_pass` is present in the tree — downgraded, never committed + +> **DOWNGRADED 2026-07-25.** `git ls-files ansible/.vault_pass` returns empty — the file was +> never committed, so **no rotation is required**. What remains is that it travels with any +> *copy* of the working directory (it is present in the Windows `Downloads` copy of this +> repo), as does the gitignored-but-present `LAB-URLS.md`. Relevant before zipping, syncing, +> or sharing that folder. The `ansible/.gitignore` path bug below is still worth fixing as +> defense in depth. + +`ansible/.vault_pass` (25 bytes) decrypts `inventory/group_vars/all/secrets.yml` and +`kvm_hosts/secrets.yml`. The root `.gitignore:2` covers it, but `ansible/.gitignore:2-4` +uses root-relative paths (`ansible/.vault_pass`) *inside* `ansible/`, so those rules match +`ansible/ansible/.vault_pass` — they protect nothing. Any export or archive of this +directory carries the password. `LAB-URLS.md` has the same shape of problem +(gitignored but present, contains credential material). + +- **Fix:** confirm `git ls-files ansible/.vault_pass` is empty; if not, rotate and + re-encrypt. Fix `ansible/.gitignore` to use paths relative to its own directory, or + delete it and rely on the root file. + +--- + +## High + +### Argo CD / cluster + +| # | Finding | File | +|---|---------|------| +| H1 | `immich-server` mounts ConfigMap `lab-ca-cert`, which is defined only in the `minio` and `monitoring` namespaces — ConfigMaps are namespaced, so the pod cannot start | `gitops/workloads/immich/server.yaml:43` | +| H2 | The cluster-scoped `ClusterSecretStore/vault-backend` — referenced by all 16 ExternalSecrets across 9 namespaces — is defined *inside* the immich app. Rename or remove that directory and `prune: true` deletes secret refresh lab-wide | `gitops/workloads/immich/external-secret.yaml:13-27` | +| H3 | No sync waves anywhere (`grep sync-wave` → nothing) and no `retry` policy on any Application. cert-manager→issuers→Ingresses, ESO→SecretStore→ExternalSecrets→Deployments, and kyverno→policies all rely on retry luck | `gitops/apps/*.yaml` | +| H4 | Namespace and PVC manifests have no `Prune=false`; the AppSet has `prune: true` + `selfHeal: true`. A bad generator result prunes a Namespace, cascading to `local-path` PVs with `Delete` reclaim — Immich Postgres, MinIO (holds OpenTofu state), lldap SQLite, HA config | `gitops/apps/workloads-appset.yaml:66-68` | +| H5 | cert-manager has `ServerSideApply=true` + `selfHeal` + `crds.enabled` and **no `ignoreDifferences`** — cainjector writes `caBundle` into the CRDs, Argo reverts it every reconcile. Same likely needed for kyverno and monitoring | `gitops/apps/cert-manager.yaml:21-27` | +| H6 | ai-gateway NetworkPolicies default-deny then allow only 11434 + DNS + 443 — so the **primary NPU backends** (`rkllama-opi*:8080`) and the m5stack-adapter (`:8080`) are blocked, and Prometheus' litellm scrape on `:4000` is silently dropped | `gitops/workloads/ai-gateway/network-policy.yaml` | +| H7 | Three PDBs with `minAvailable: 1` against `replicas: 1` — `kubectl drain` hangs forever on the exact nodes you want to patch | `authelia/pdb.yaml:7`, `lldap/pdb.yaml:7`, `immich/pdb.yaml:7` | +| H8 | Vault is contacted over plaintext HTTP with a static, never-rotated token that isn't in git (so the store is unreproducible) | `gitops/workloads/immich/external-secret.yaml:20` | +| H9 | PodSecurityAdmission is documented but no namespace carries `pod-security.kubernetes.io/*` labels — everything is at k3s default (privileged). The only gate is Kyverno, whose `webhookFailurePolicy: Ignore` fails **open** | `gitops/apps/kyverno.yaml:35`, all `namespace.yaml` | +| H10 | Stateful workloads with `local-path` PVCs and no `nodeSelector` can land on the arm64 OPi agents (which are labelled but **not tainted**) and get pinned there by PV node affinity | `immich/*.yaml`, `authelia/*.yaml`, `lldap/deployment.yaml`, `home-assistant/deployment.yaml` | +| H11 | Kyverno PolicyExceptions for ollama and whisper are still committed and inside Argo-synced directories with `selfHeal` — deleting them imperatively means Argo recreated them. Both images are now pinned, so they're pure attack surface. `TODO-2026-07-23.md:163` claiming "0 PolicyExceptions" is false | `ai-backends/policy-exceptions.yaml`, `whisper/policy-exceptions.yaml` | + +### Hosts / automation + +| # | Finding | File | +|---|---------|------| +| H12 | `write-kubeconfig-mode: "0644"` — every local user on the k3s servers can read cluster-admin credentials. `docs/SECURITY.md:38-39` claims a scoped kubeconfig that isn't implemented | `k3s-h4.yml:43`, `k3s-ha-join.yml:82` | +| H13 | `github-runner.yml` rewrites the **primary lab user's** home directory (`user: name: swares, home: .../actions-runner`), orphaning `~/.ssh/authorized_keys`, `~/.kube/config`, `~/.ansible` — then grants that account `NOPASSWD: ALL` | `ansible/playbooks/github-runner.yml:36-42,109-116` | +| H14 | k3s node token leaks: written to `/tmp/k3s-agent-install.sh` and never removed; slurped with **no `no_log`** into Semaphore's job log DB; interpolated onto a shell command line visible in `ps` | `k3s-agent.yml:40-56`, `update-non-apt.yml:135-144`, `k3s.yml:28` | +| H15 | `storage.yml` can `mkfs` a cold mirror: it runs `community.general.filesystem` against `/dev/md1`/`/dev/md0` by unstable device name. Arrays renumber across reboots when not pinned in `mdadm.conf`. This is the copy-of-record for all NAS data | `ansible/playbooks/storage.yml:47-65` | +| H16 | Pi-hole deployed with `pihole_web_password_hash: ""` — v6 treats empty as *no auth*, so the admin UI and API on the load-bearing resolver are unauthenticated | `ansible/playbooks/dns.yml:12`, `templates/pihole.toml.j2:21` | +| H17 | APT signature verification permanently disabled on the primary DNS host (`deb [trusted=yes]` + `Acquire::AllowInsecureRepositories "true"`), then `curl \| bash` for Pi-hole | `ansible/playbooks/dns.yml:33-55,135` | +| H18 | `sync-secrets-to-vault.yml` uses `vault kv put` then `vault kv patch` on the *same path* — the `put` wipes `kvm_become_password`. Also delegates to the decommissioned `ldap-1`, and passes plaintext secrets as argv (visible in `ps`) | `ansible/playbooks/sync-secrets-to-vault.yml:129-135,99-104,52-167` | +| H19 | `delegate_to: h4` — no such inventory host (it's `h4-core`). The vault-backup destination dir and `authorized_key` are therefore never created on the real H4 | `ansible/playbooks/backup-vault.yml:23,42` | +| H20 | **CONFIRMED 2026-07-25** — ran 02:30, logged `no offsite repo configured; skipping`, `status=0/SUCCESS`. Offsite restic leg defaults to `''` and renders to a no-op that exits 0, while the timer is enabled and reports success. `backup-offsite` and `backup-lldap` are also absent from the healthchecks list. Two competing offsite mechanisms now exist (`backup-offsite` vs `backup-cloud` → R2) | `backup.yml:15,70`, `templates/backup-offsite.service.j2:9`, `healthchecks.yml:39-43` | +| H21 | `vault-restore.yml` pins a hard-coded dated tarball (`vault-backup-20260627.tar.gz`) in a directory nothing maintains, while the live stream is `vault-snap-*.snap` with 30-day retention. It also uses a controller-side `src` for a file on the H4 — the play cannot work as written | `ansible/playbooks/vault-restore.yml:26,65-68` | +| H22 | `.terraform.lock.hcl` is gitignored — provider versions are not reproducible or checksum-verified despite `~> 0.8` constraints | `.gitignore:22` | +| H23 | No OpenTofu state locking on either S3/Minio backend (no `use_lockfile`, no lock table). Semaphore, the GitLab runner, and a laptop can all apply concurrently and clobber state — which is stored unencrypted over plaintext HTTP | `tofu/vms/terraform.tf:14-27`, `tofu/dns/terraform.tf:25-40` | +| H24 | WinRM: docs create a local Administrator with the literal password `StrongPassword`, enable **Basic** auth, and the inventory connects over `http`/5985 — admin credential in base64 on the wire every run | `docs/OPS.md:452-465`, `hosts.yml:145-151` | + +### Observability + +| # | Finding | File | +|---|---------|------| +| H25 | **Nothing pages a human.** Alertmanager has exactly two receivers: `null` and `m5stack` (an HTTP POST to an ESP32 on the LAN). ArgoCD notifications go to the same device. Asleep or display off = every alert silently discarded | `gitops/apps/monitoring.yaml:166-192` | +| H26 | `kubeEtcd: enabled: false` — **zero etcd metrics, therefore zero quorum-loss alerting** on a 3-node HA cluster | `gitops/apps/monitoring.yaml:221-222` | +| H27 | `LabBackupEtcdSilent` fires at >25 h against a **weekly** timer — permanently critical 6 days out of 7, training you to ignore the one alert group that matters most (see C1) | `lab-alerts.yaml:80-95` vs `backup-etcd.timer.j2:5` | +| H28 | Prometheus scrapes decommissioned `192.168.1.70` (ldap-1) — `LabExternalHostDown` critical has been firing for ~3 weeks — while `.116` (Pi-hole secondary), `.217` (4th DNS) and `.128` (**Vault**) are not scraped at all | `gitops/apps/monitoring.yaml:129-134` | +| H29 | No cert-manager ServiceMonitor and no expiry rule; no blackbox exporter, so nothing verifies that `api.lab.home.arpa` and `*.apps.lab.home.arpa` actually resolve — despite `CLAUDE.md:49` calling DNS load-bearing | `gitops/workloads/cert-manager/` | +| H30 | `.claude/settings.json` is a MicroShift artifact: it gates `oc apply`/`oc delete`/`oc adm`, which `CLAUDE.md:42` forbids using. `kubectl` appears **3 times, all in `allow`** — every `kubectl` mutation is unguarded, directly contradicting the table at `docs/SECURITY.md:52-54`. Also denies `systemctl stop nfs-server` while Ubuntu's unit is `nfs-kernel-server`, and no `sudo`-prefixed variants of any deny rule | `.claude/settings.json:4-57` | + +--- + +## Medium + +- **M-new-1: nothing ever trims the cold-sec copy repo.** *(found 2026-07-25)* Retention runs + only against the primary (`backup-nas.service.j2:15`); the copy accumulates forever — 40 + snapshots vs the primary's 19 already. `CLAUDE.md` correctly forbids running `restic forget` + by hand, so there is **no sanctioned mechanism to ever reclaim space on the 5.45TB mirror**. + Fix: add a policy-driven `forget` for the copy repo inside the unit (where retention + legitimately lives), or at minimum a free-space alert on `/mnt/cold-sec`. +- **M-new-4: R2 retention has never deleted a snapshot, and cannot.** *(found 2026-07-25)* + `backup-cloud.sh` passes `TMPDIR=$(mktemp -d -t backup-cloud-XXXXXX)` straight into + `SOURCES`, so every snapshot's path list ends with a uniquely-named staging directory + (`/tmp/backup-cloud-PX4ZQS`, `-pOvw3A`, `-V5xXsp`, ...). restic `forget` groups by + `host,paths` by default, so **each snapshot forms its own retention group** and + `--keep-daily 7 --keep-weekly 4 --keep-monthly 3` dutifully keeps the sole member of each. + Visible in the journal as four consecutive `1 snapshots` / `keep 1 snapshots` blocks. + Third instance of the house theme: runs, exits 0, prints reassuring output, does nothing. + Currently 360 MiB of the 10 GB free tier across 5 snapshots, so there is runway — roughly + months, not weeks. `backup-nas` is unaffected; its source paths are fixed, which is why the + primary correctly prunes to 19 while the unpruned cold-sec copy has grown to 40. + Fix, both halves: add `--group-by host,tags` to the `forget`, **and** stage into a stable + path (`/var/tmp/backup-cloud-stage`, cleared each run) so snapshot trees are consistent and + a restore script can hardcode a path. +- **M-new-2: restic 0.12.1 is a silent-data-loss landmine on upgrade.** *(found 2026-07-25)* + The H4 runs restic 0.12.1 (2021, Ubuntu 22.04 distro package). restic 0.14 **inverted + `--repo2`** to mean *source*. The moment restic is upgraded past 0.14 — by `apt upgrade`, + Renovate, or a host rebuild — `backup-nas.service.j2:19` silently reverses and overwrites + the primary from the secondary. Rewrite the unit to explicit `--from-repo` semantics **at + the same time** as any upgrade, never separately. Until then, hold restic at 0.12.x. +- **M-new-3: stray `state.db` litter in the live k3s datastore.** *(found 2026-07-25)* The + broken backup unit creates a 0-byte `/var/lib/rancher/k3s/server/db/state.db` on every run. + Low risk (k3s selects etcd from the `etcd/` dir) but it does not belong in the datastore + directory. Remove it once C1 is fixed, along with the 4.0K empty snapshots on cold and — + after a deliberate decision — the 22MB `state.db.migrated`. +- **Immich RWX PV makes the H4 a hard dependency** (`immich/library-pv.yaml:9-17`) with `hard` + NFS mount options — a pod on n150-1 blocks in uninterruptible I/O forever if the H4 is down. + The 2026-07-04 "schedules on any node" claim needs the caveat. Same box also exports + `/srv/libvirt-shared` for VM live migration. +- **Vault is a hard dependency for its own recovery** — `docs/RUNBOOK.md:131-133` says get the + restic password from Vault; Vault's raft snapshots live on `/mnt/cold-8t/vault-snapshots` + on the H4 that just died. Keep an offline copy of the restic password and unseal keys. +- **Vault unseal keys sit beside the sealed data** (`/etc/vault.d/unseal-keys`, `0400`, same + SD card as `/opt/vault/data`), with `tls_disable = true` on `0.0.0.0:8200` and no host + firewall. `docs/SECURITY.md:25` calls them "offline, physically secure" — that is false. +- **`restic forget --prune` runs unconditionally** on the primary repo every run + (`backup-nas.service.j2:15`), so a bug in a source path silently ages out good snapshots. +- **No host firewall anywhere** — the only rules in `ansible/` are legacy firewalld in the + dead `microshift.yml` and two `ufw` allows. The model is "the LAN is trusted". +- **Renovate uses removed syntax** — `regexManagers` (deprecated), `matchPackagePatterns`/ + `excludePackagePatterns` (removed in Renovate 38+), so those `packageRules` may be silently + dropped. `fileMatch` is scoped to `gitops/workloads/**`, so **`gitops/apps/` chart versions + aren't tracked at all**; no ansible-galaxy or packer managers. +- **Packer targets Proxmox** (`ubuntu-noble.pkr.hcl`) while the lab runs KVM/libvirt; also + `insecure_skip_tls_verify = true` and clones a mutable VM 9000 with no checksum, so builds + aren't reproducible. Likely dead code. +- **`tofu/dns` can't apply** — its pinned provider only speaks the Pi-hole v5 API (documented + at `terraform.tf:6-14`), and the two load-bearing DNS anchor records lack `prevent_destroy` + (which `tofu/vms/main.tf:27-33` gets right). +- **`backup-cloud.yml` holds MinIO root credentials** and uses `--no-verify-ssl` — a backup + job shouldn't have root on the store holding your OpenTofu state. +- **Committed credential material:** argon2id OIDC client-secret hashes for five SSO clients + (`authelia/configmap.yaml:72,86,98,111,125`) — offline-crackable, and rotation needs a PR. + A `kind: Secret` at `gitlab-runner/gitlab-tls-secret.yaml:22` (contents are the public CA, + so not sensitive, but it will trip scanners). +- **Kyverno `require-resource-limits` skips `initContainers`** — `disallow-latest-tag` handles + them correctly, this one doesn't. Live gap: the `seed-config` initContainer at + `home-assistant/deployment.yaml:51-53` has no resources block and is admitted. +- **Dead duplicate config trees** — `ansible/.gitlab-ci.yml` (still MicroShift-era `oc apply`), + `ansible/.github/workflows/`, `ansible/.claude/settings.json`. Neither GitLab nor GitHub + reads these paths; they will be mistaken for live config. +- **No AppProject** — everything runs in `default`, which permits any repo, any destination, + any cluster-scoped resource. `repoURL` is hardcoded in 18 places and inconsistent + (`gitops/apps/gitlab-runner.yaml:18,22` uses HTTPS where everything else uses SSH). +- **Drains without rescue** (`update-non-apt.yml:78-108`, `update-hosts.yml:104-130`) leave + nodes cordoned indefinitely if the middle step fails, with no alert. +- **`scripts/lab-check.sh` health-checks decommissioned `192.168.1.70`** and is run with + `|| true` in CI, so the permanent failures are invisible. + +--- + +## Documentation drift + +| # | Contradiction | Refs | +|---|---|---| +| D1 | octopi flash "confirmed 2026-07-13" vs "flash **pending**" — same file | `ARCHITECTURE.md:140` vs `:176` | +| D2 | "DNS needs a permanent host" / "secondary DNS is opi-zero2w-1" vs four documented permanent DNS servers | `CLAUDE.md:68-69` vs `ARCHITECTURE.md:136-143` | +| D3 | "The map's plaintext credentials must be rotated. See HARDWARE.md" — HARDWARE.md contains **no** credentials and states rotation is done. Stale pointer | `CLAUDE.md:71` vs `HARDWARE.md:6-8` | +| D4 | ldap-1 decommission date 07-**04** vs 07-**18** | `HARDWARE.md:37`, `README.md:175` vs `RUNBOOK.md:99`, `hosts.yml:189` | +| D5 | Home Assistant: k3s Deployment vs Docker on rpi4b `.116` | `README.md:45`, `CLAUDE.md:68` vs `hosts.yml:162-166` | +| D6 | Vault auto-unseal "on H4" vs rpi5 | `services.md:17` vs `vault.yml:10` | +| D7 | Unseal keys "offline, physically secure" vs on-disk `0400` | `SECURITY.md:25` vs `vault.yml:67-76` | +| D8 | Permission table describes `kubectl` rules that don't exist | `SECURITY.md:52-54` vs `.claude/settings.json` | +| D9 | Ollama pin `0.32.0` vs actual `0.32.1` | `README.md:40` vs `ollama.yaml:39` | +| D10 | backup-etcd "daily" vs weekly timer | `RUNBOOK.md:123` vs `backup-etcd.timer.j2:5` | +| D11 | `.160` called "the kube-vip VIP" (it's `.200`) | `SSO.md:63` vs `ARCHITECTURE.md:36-40` | +| D12 | Where etcd lives: eMMC vs NVMe. ARCHITECTURE argues co-location is safe *because NVMe* — if it's really eMMC that argument is void | `RUNBOOK.md:14` vs `ARCHITECTURE.md:111-113`, `CLAUDE.md:27` | +| D13 | README pins `.116` as Pi-hole secondary; inventory says it's `home_assistant` and puts DNS on `.148`/`.184`/`.217` | `README.md:62` vs `hosts.yml:74-85,163-166` | +| D14 | `HARDWARE.md:147` ends mid-sentence: `"— investigate(2"` | — | + +**TODO files:** `TODO-2026-07-14.md` is 25/26 done and should be archived (two of its "done" +claims are wrong — lines 84 and 102). `TODO-2026-07-23.md` is the live doc; its remaining +open items are deferred by choice with sound rationale. The `README.md:158-169` TODO section +is the stalest of the three and should collapse to a pointer. + +--- + +## Verified clean + +Worth stating, because it's a lot: + +- Every PVC uses `local-path`; zero `topolvm-provisioner` references. The only non-`local-path` + volume is the deliberate static NFS RWX PV (`storageClassName: ""`, `Retain`) — correct. +- No `:latest`, no untagged images, no `privileged: true`, no missing `resources.limits` on any + `spec.containers` across all 18 workload directories. +- No OpenShift residue in `gitops/` — no `kind: Route`, no SCCs. All Ingress is + `networking.k8s.io/v1` with `ingressClassName: traefik`. +- All 16 ExternalSecrets use consistent `refreshInterval: 1h` and `ClusterSecretStore`; + `creationPolicy: Merge` correctly used for `argocd-secret`. +- No orphaned workload directories, no `apps/`↔`workloads/` duplication; all 9 AppSet + exclusions are justified with matching individual Applications. +- Helm charts are properly version-pinned with Renovate annotations throughout. +- `backup-cloud.yml` uses `no_log: true` consistently with `0600`/`0700` modes. +- Packer's `proxmox_token` is correctly marked `sensitive`; the example vars file has + placeholders only. + +--- + +## Suggested order of work + +1. ~~**Verify, don't assume**~~ — **done 2026-07-25.** C1 and H20 confirmed, C6 cleared, + C7 downgraded; M-new-1/2/3 found. Only remaining check: whether n150-1/n150-2 are also + taking local etcd snapshots (if so, C2's story improves considerably). +2. **`fix/backup-etcd-embedded-etcd`** — now the top item, and smaller than first thought: + copy k3s' existing `/var/lib/rancher/k3s/server/db/snapshots/` to cold **daily** (not + weekly — NVMe retention is only 2.5 days), drop the sqlite3 logic, clean up the stray + `state.db` (M-new-3), fix H27's alert threshold and D10's doc row, and drop the unused + `etcd_backup_dir: .../microshift-backups` var (`backup.yml:14`) plus the MicroShift-era + header comment. +3. **`fix/ci-validation-gates`** — C4. One small PR; it's the gate everything else relies on, + and it currently can't fail. +4. **`fix/backup-retention-and-verification`** — M-new-1, M-new-2, M-new-4: retention for the + cold-sec copy, `--group-by host,tags` + a stable staging path for the R2 copy, hold restic + at 0.12.x with a comment explaining why, a weekly `restic check` timer across all three + repos with its own healthcheck, a content assertion that the newest cloud snapshot holds a + >1 MB `etcd-snapshot-*` file, and drop the `-` prefix on the cold-sec copy. + + Verified 2026-07-25, so this PR starts from a known-good baseline: all three repos pass + `restic check` with no errors (19 / 40 / 5 snapshots), and R2 snapshot `b8cc9c9b` contains + all five real `etcd-snapshot-*` files. +5. **`fix/alert-routing`** — H25/H26/H28. Add one receiver that reaches you when asleep, + enable `kubeEtcd`, fix the scrape targets. +6. **`fix/claude-permissions`** — H30. Rewrite `.claude/settings.json` around `kubectl`, + mirror the SECURITY.md table, add `sudo` variants. +7. **`fix/gitlab-runner-hardening`** — C5. +8. **`fix/argo-sync-waves-and-prune-guards`** — H2/H3/H4/H5, plus the immich `lab-ca-cert` + ConfigMap (H1). +9. **`docs/reconcile-drift`** — the D-table plus archiving `TODO-2026-07-14.md`. + +Then write and *run* the H4-total-loss restore (C2). Everything above is preparation for the +day that matters. diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md index 1e18bf9..b7297f7 100644 --- a/docs/RUNBOOK.md +++ b/docs/RUNBOOK.md @@ -117,10 +117,15 @@ Never `kubectl apply` directly against main — it drifts and ArgoCD reverts it. ## Backups +> **See [BACKUP-RESTORE.md](BACKUP-RESTORE.md)** for the authoritative reference — +> full inventory, restore procedures, and validation commands. The table below is a +> summary; the restore steps further down this file were found to be incorrect on +> 2026-07-24 and are superseded by BACKUP-RESTORE.md §3. + | Stream | When | What | Downtime | |--------|------|------|----------| | `backup-nas` | daily 01:30 | restic of `/srv/nas` + `/mnt/cold-8t/VMs` + `/mnt/cold-8t/immich` → cold-8t, then `restic copy` → cold-sec | none | -| `backup-etcd` | daily | k3s SQLite state → `/mnt/cold-8t/k3s-etcd-snapshots/`, 7 copies retained | none | +| `backup-etcd` | daily 00:30 | Copies k3s' own embedded-etcd snapshots (k3s writes them to `/var/lib/rancher/k3s/server/db/snapshots/` every 12h) → `/mnt/cold-8t/k3s-etcd-snapshots/`, 30 copies retained. Verifies size + freshness and **fails loudly** if either check trips. | none | | `backup-vault` | daily 02:30 | Vault raft snapshot → `/mnt/cold-8t/vault-snapshots/`, 30-day retention | none | | `lldap-backup` (k8s CronJob) | daily 02:30 | lldap `users.db` (SQLite) → `/mnt/cold-8t/restic` via NFS restic repo. Secret: `lldap-backup-secrets` (ESO from `secret/lab/restic`). Manifest: `gitops/workloads/lldap/backup-cronjob.yaml`. | none | | `backup-cloud` | daily 03:00 | **Offsite (Track 1).** restic → Cloudflare R2 `homelab-backup` bucket. Sources: etcd snapshots + Vault snapshots + lldap SQLite (via `kubectl cp`) + Postgres dumps (Authelia, Immich, Semaphore via `kubectl exec`). Credentials in Vault at `secret/lab/cloudflare-r2`. Playbook: `ansible/playbooks/backup-cloud.yml`. | none | @@ -210,10 +215,15 @@ For immediate relief before the revert lands: ### Full cluster loss — etcd restore - sudo systemctl stop k3s - sudo k3s etcd-snapshot restore /mnt/cold-8t/etcd/ - sudo systemctl start k3s - # ArgoCD re-syncs all workloads from git automatically +> **The procedure that was here was wrong and has been removed.** It used a +> `k3s etcd-snapshot restore` subcommand that does not exist, pointed at +> `/mnt/cold-8t/etcd/` (the real directory is `/mnt/cold-8t/k3s-etcd-snapshots/`), +> and gave single-node steps for a 3-server embedded-etcd cluster — where a restore +> also requires wiping and rejoining the other two servers. +> +> **Use [BACKUP-RESTORE.md §3.2](BACKUP-RESTORE.md#32-restore-cluster-state-etcd--3-server-ha)**, +> which is verified against the current k3s documentation. Restoring onto new +> hardware additionally requires the k3s server token — see §2 of that document. ### NAS data loss — restic restore