From cd97dda50dc5467caf14ecf063b7297c293867fb Mon Sep 17 00:00:00 2001 From: Scott Wares Date: Thu, 23 Jul 2026 14:53:08 +0000 Subject: [PATCH 1/7] =?UTF-8?q?fix:=20remove=20stale=20Kyverno=20PolicyExc?= =?UTF-8?q?eptions=20=E2=80=94=20ollama=20pinned=20to=200.32.1,=20whisper?= =?UTF-8?q?=20to=200.6.0-rc.3-cpu?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- FETCH_HEAD | 0 .../ai-backends/policy-exceptions.yaml | 25 ------------------- gitops/workloads/authelia/pvc.yaml | 11 ++++++++ .../workloads/whisper/policy-exceptions.yaml | 24 ------------------ 4 files changed, 11 insertions(+), 49 deletions(-) create mode 100644 FETCH_HEAD delete mode 100644 gitops/workloads/ai-backends/policy-exceptions.yaml create mode 100644 gitops/workloads/authelia/pvc.yaml delete mode 100644 gitops/workloads/whisper/policy-exceptions.yaml diff --git a/FETCH_HEAD b/FETCH_HEAD new file mode 100644 index 0000000..e69de29 diff --git a/gitops/workloads/ai-backends/policy-exceptions.yaml b/gitops/workloads/ai-backends/policy-exceptions.yaml deleted file mode 100644 index 19f7605..0000000 --- a/gitops/workloads/ai-backends/policy-exceptions.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Temporary exceptions to disallow-latest-tag while pinned tags are being validated. -# ollama/ollama: pin blocked until 0.32.0 image is confirmed pulled on all OPi nodes. -# Remove this file once both Deployments reference a versioned tag. -apiVersion: kyverno.io/v2 -kind: PolicyException -metadata: - name: allow-ollama-latest - namespace: ai-gateway - annotations: - # Remove when ollama.yaml and ollama-2.yaml pin to a versioned tag. - policy-exception/reason: "Renovate PR pending; pre-staging 0.32.0 on OPi nodes before approving" -spec: - exceptions: - - policyName: disallow-latest-tag - ruleNames: - - require-image-tag - - disallow-latest-tag - - autogen-require-image-tag - - autogen-disallow-latest-tag - match: - any: - - resources: - kinds: [Pod] - namespaces: [ai-gateway] - names: ["ollama-*"] diff --git a/gitops/workloads/authelia/pvc.yaml b/gitops/workloads/authelia/pvc.yaml new file mode 100644 index 0000000..54855f6 --- /dev/null +++ b/gitops/workloads/authelia/pvc.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: authelia-data + namespace: authelia +spec: + accessModes: [ReadWriteOnce] + storageClassName: local-path + resources: + requests: + storage: 1Gi diff --git a/gitops/workloads/whisper/policy-exceptions.yaml b/gitops/workloads/whisper/policy-exceptions.yaml deleted file mode 100644 index 3611be4..0000000 --- a/gitops/workloads/whisper/policy-exceptions.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Temporary exception to disallow-latest-tag for faster-whisper-server. -# The image only publishes CPU builds as latest-cpu; no versioned CPU tag exists yet. -# Remove once a pinnable versioned CPU tag is available upstream. -apiVersion: kyverno.io/v2 -kind: PolicyException -metadata: - name: allow-whisper-latest-cpu - namespace: whisper - annotations: - policy-exception/reason: "fedirz/faster-whisper-server has no versioned CPU tag — latest-cpu is the only published CPU variant" -spec: - exceptions: - - policyName: disallow-latest-tag - ruleNames: - - require-image-tag - - disallow-latest-tag - - autogen-require-image-tag - - autogen-disallow-latest-tag - match: - any: - - resources: - kinds: [Pod] - namespaces: [whisper] - names: ["whisper-*"] From 1ab3ab67cc4cfe9989eafbf3638bf9eaa2decc1f Mon Sep 17 00:00:00 2001 From: Scott Wares Date: Thu, 23 Jul 2026 16:00:04 +0000 Subject: [PATCH 2/7] feat: add Groq + Gemini cloud fallback to LiteLLM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ExternalSecret pulls groq-api-key + gemini-api-key from secret/lab/cloud-ai - Deployment mounts litellm-cloud-ai Secret as env vars - ConfigMap adds 'cloud' model group (llama-3.3-70b-versatile + gemini-2.0-flash) - Router fallbacks: chat → cloud, chat-cpu → cloud on any failure - NetworkPolicy allows litellm HTTPS egress to public internet for cloud APIs --- gitops/workloads/ai-gateway/configmap.yaml | 25 +++++++++++++++++-- gitops/workloads/ai-gateway/deployment.yaml | 3 +++ .../ai-gateway/external-secret-cloud-ai.yaml | 22 ++++++++++++++++ .../workloads/ai-gateway/network-policy.yaml | 17 +++++++++++++ 4 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 gitops/workloads/ai-gateway/external-secret-cloud-ai.yaml diff --git a/gitops/workloads/ai-gateway/configmap.yaml b/gitops/workloads/ai-gateway/configmap.yaml index 67c6a9c..fb260b9 100644 --- a/gitops/workloads/ai-gateway/configmap.yaml +++ b/gitops/workloads/ai-gateway/configmap.yaml @@ -64,11 +64,32 @@ data: litellm_params: { model: openai/m5-llm, api_base: http://m5stack-adapter:8080/v1, api_key: "none" } - model_name: m5-claude litellm_params: { model: openai/m5-claude, api_base: http://m5stack-adapter:8080/v1, api_key: "none" } - # Optional cloud overflow — supply a key via Secret, then uncomment. + # ── Cloud fallback (Groq + Gemini, free tier) ────────────────────────────── + # Sits between local inference and paid Claude API. LiteLLM automatically + # falls back here when chat / chat-cpu fail (timeout, OOM, all replicas down). + # Keys injected via litellm-cloud-ai Secret (ESO from secret/lab/cloud-ai). + # Free limits: Groq 6000 req/day; Gemini 1500 req/day on Flash. + - model_name: cloud + litellm_params: + model: groq/llama-3.3-70b-versatile + api_key: os.environ/GROQ_API_KEY + - model_name: cloud + litellm_params: + model: gemini/gemini-2.0-flash + api_key: os.environ/GEMINI_API_KEY + + # ── Paid cloud (Claude) — last resort ────────────────────────────────── + # Uncomment and supply key to enable: # - model_name: big # litellm_params: { model: anthropic/claude-sonnet-4-6, api_key: os.environ/ANTHROPIC_API_KEY } + router_settings: - routing_strategy: simple-shuffle # load-balances across opi1 + opi2 + routing_strategy: simple-shuffle # load-balances within each model group num_retries: 2 + # Automatic fallback chain: local NPU → cloud (Groq/Gemini) → (big if enabled) + # LiteLLM tries these in order on any exception (timeout, rate-limit, OOM). + fallbacks: + - chat: [cloud] + - chat-cpu: [cloud] # NOTE: the M5Stack speaks the framework's /api/llm protocol, not OpenAI — front it with a # tiny adapter (or call it directly / via MQTT from Home Assistant). See AI-INFERENCE.md. diff --git a/gitops/workloads/ai-gateway/deployment.yaml b/gitops/workloads/ai-gateway/deployment.yaml index 138ea9d..ee57a33 100644 --- a/gitops/workloads/ai-gateway/deployment.yaml +++ b/gitops/workloads/ai-gateway/deployment.yaml @@ -40,6 +40,9 @@ spec: limits: cpu: "2" memory: 2Gi + envFrom: + - secretRef: + name: litellm-cloud-ai # GROQ_API_KEY, GEMINI_API_KEY from Vault securityContext: allowPrivilegeEscalation: false capabilities: { drop: ["ALL"] } diff --git a/gitops/workloads/ai-gateway/external-secret-cloud-ai.yaml b/gitops/workloads/ai-gateway/external-secret-cloud-ai.yaml new file mode 100644 index 0000000..5e9a5cf --- /dev/null +++ b/gitops/workloads/ai-gateway/external-secret-cloud-ai.yaml @@ -0,0 +1,22 @@ +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: litellm-cloud-ai + namespace: ai-gateway +spec: + refreshInterval: 1h + secretStoreRef: + name: vault-backend + kind: ClusterSecretStore + target: + name: litellm-cloud-ai + creationPolicy: Owner + data: + - secretKey: GROQ_API_KEY + remoteRef: + key: secret/lab/cloud-ai + property: groq-api-key + - secretKey: GEMINI_API_KEY + remoteRef: + key: secret/lab/cloud-ai + property: gemini-api-key diff --git a/gitops/workloads/ai-gateway/network-policy.yaml b/gitops/workloads/ai-gateway/network-policy.yaml index a0e6c18..41e9c1b 100644 --- a/gitops/workloads/ai-gateway/network-policy.yaml +++ b/gitops/workloads/ai-gateway/network-policy.yaml @@ -81,6 +81,23 @@ spec: ports: - port: 11434 --- +# Allow litellm egress to external cloud AI APIs (Groq, Gemini) over HTTPS. +# No `to` selector = allow to any destination on port 443 (public internet). +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-litellm-cloud-egress + namespace: ai-gateway +spec: + podSelector: + matchLabels: + app: litellm + policyTypes: [Egress] + egress: + - ports: + - port: 443 + protocol: TCP +--- # Allow m5stack-adapter egress to the M5Stack device (external HTTPS on port 443). apiVersion: networking.k8s.io/v1 kind: NetworkPolicy From c6599b4331f0bb0e0be5bb881877cea89d66ce0a Mon Sep 17 00:00:00 2001 From: Scott Wares Date: Thu, 23 Jul 2026 16:21:19 +0000 Subject: [PATCH 3/7] docs: document Groq/Gemini cloud fallback, fix litellm-vip placeholder URL --- docs/AI-INFERENCE.md | 8 ++++++-- docs/STANDUP.md | 10 ++++++++-- docs/services.md | 2 +- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/docs/AI-INFERENCE.md b/docs/AI-INFERENCE.md index b607bb8..68b7c96 100644 --- a/docs/AI-INFERENCE.md +++ b/docs/AI-INFERENCE.md @@ -179,8 +179,12 @@ Pi 5 Pro** — it's already the "brain" tier (RKLLama host model + Claude Code s Home Assistant via MQTT, so it's the natural home-automation assistant. - **Embeddings / STT / OCR / vision** → OpenVINO on the Intel iGPUs (the N150s earn their keep here). NPUs are poor at these; the iGPUs are fine. -- **Overflow / big models** → optional cloud fallback through the same LiteLLM endpoint, - keyed, used only when a request won't fit locally. +- **Cloud fallback** → `cloud` model in LiteLLM routes to Groq (llama-3.3-70b-versatile) + and Gemini (gemini-2.0-flash) on free-tier quotas. LiteLLM automatically falls back + to `cloud` when `chat` or `chat-cpu` fail (timeout, OOM, all replicas down). Keys in + Vault at `secret/lab/cloud-ai`; injected via ESO Secret `litellm-cloud-ai`. +- **Paid cloud (big models)** → Claude API (`big` model, currently commented out in + configmap). Enable by adding `ANTHROPIC_API_KEY` to Vault and uncommenting. ## Honest caveats diff --git a/docs/STANDUP.md b/docs/STANDUP.md index 8e15e41..c250818 100644 --- a/docs/STANDUP.md +++ b/docs/STANDUP.md @@ -128,9 +128,15 @@ Goal: local inference behind one OpenAI-compatible gateway. See 2. 🟡 **Verify the gateway** (LiteLLM, deployed by Argo in Phase 3) reaches the backends and the `m5stack-adapter`: ```bash - curl -sk https:///v1/models | jq '.data[].id' # m5, m5-llm, m5-claude, etc. + curl -sk https://ai.apps.lab.home.arpa/v1/models | jq '.data[].id' + # expect: chat, chat-cpu, code, vision, fast, embeddings, cloud, m5, m5-llm, m5-claude + curl -sk https://ai.apps.lab.home.arpa/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"model":"cloud","messages":[{"role":"user","content":"ping"}]}' \ + | jq '.choices[0].message.content' # expect: "pong" via Groq or Gemini ``` - Send a test chat completion and confirm `route_taken` surfaces (local / direct_api / escalated). + Cloud fallback (Groq llama-3.3-70b + Gemini 2.0 Flash) is wired as automatic fallback + for `chat` and `chat-cpu` failures. Keys in Vault at `secret/lab/cloud-ai`. 3. 🔧 **M5Stack escalation router** — point it at the gateway/orchestrator; confirm the 3 tiers (local NPU → Claude API → Claude Code on the orchestrator) resolve as configured. diff --git a/docs/services.md b/docs/services.md index 9d4083c..0bd0d80 100644 --- a/docs/services.md +++ b/docs/services.md @@ -57,7 +57,7 @@ Everything the design implies, so you can tick what's needed and spot gaps. Stat |---------|:------:|-----------|-------| | RKLLama (NPU LLM) | ✓ | opi5pro-1 (.168) + opi5pro-2 (.172) | DeepSeek-R1-Distill-Qwen-1.5B_w8a8_g128; driver 0.9.6 limits NPU alloc to ~2.2 GB (blocks 3B+); upgrade to 0.9.7 to unlock | | OpenVINO Model Server | ● | H4 + N150 iGPUs | embeddings / STT / vision — suspended; crash-looping on n150-1, scaled to 0 pending fix | -| LiteLLM gateway | ✓ | k3s · `ai.apps.lab.home.arpa` | unifies backends | +| LiteLLM gateway | ✓ | k3s · `ai.apps.lab.home.arpa` | unifies backends; cloud fallback: Groq llama-3.3-70b + Gemini 2.0 Flash (free tier, auto-fallback on chat/chat-cpu failure); keys in Vault at `secret/lab/cloud-ai` | | m5stack-adapter | ✓ | k3s · odroid-nas node | OpenAI shim for M5Stack; image 0.1.1 (2026-07-11); models: m5, m5-llm, m5-claude | | Claude Code orchestrator | ✓ | opi5pro-1 | escalation Tier 3; HTTPS :8443; TLS self-signed; Claude Code 2.1.204 | | M5Stack escalation router | ✓ | M5Stack | edge front-end; 3-tier escalation | From 957a69c6bfcd853872fbe19482fd158bd539b70f Mon Sep 17 00:00:00 2001 From: Scott Wares Date: Thu, 23 Jul 2026 17:05:26 +0000 Subject: [PATCH 4/7] feat: gate LiteLLM UI with Authelia ForwardAuth - Authelia: add bypass rules for /v1/ and /health/ on ai.apps.lab.home.arpa - ai-gateway ingress: add authelia-forwardauth middleware - SSO.md: move LiteLLM to Completed --- docs/SSO.md | 3 ++- gitops/workloads/ai-gateway/ingress.yaml | 2 ++ gitops/workloads/authelia/configmap.yaml | 23 +++++++++-------------- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/docs/SSO.md b/docs/SSO.md index cea2eb2..4791d64 100644 --- a/docs/SSO.md +++ b/docs/SSO.md @@ -18,6 +18,7 @@ registrations. Client secrets are argon2id hashes; plaintext secrets live in Vau | Immich | `immich.apps.lab.home.arpa` | openid-client v6 / oauth4webapi; calls userinfo endpoint | | MinIO Console | `minio-console.apps.lab.home.arpa` | Env var config; calls userinfo endpoint | | Semaphore | `semaphore.apps.lab.home.arpa` | Config via ESO-rendered `config.json` Secret; `claims_policy: include_profile_in_id_token` required — go-oidc reads ID token only | +| LiteLLM | `ai.apps.lab.home.arpa` | Traefik ForwardAuth (`authelia-forwardauth@kubernetescrd`). UI (`/`) requires one_factor login. `/v1/` and `/health/` bypass auth so API callers are unaffected. | --- @@ -27,7 +28,7 @@ registrations. Client secrets are argon2id hashes; plaintext secrets live in Vau |---------|-----|--------| | Zot | `registry.apps.lab.home.arpa` | Zot v2.1.x only accepts `github`/`google`/`gitlab` as provider names. Generic OIDC (`authelia`, `dex`, etc.) crashes with *unsupported openid/oauth2 provider*. Also deprecated `clientid`/`clientsecret` inline — new `credentialsfile` approach not yet tested. Revisit on Zot upgrade. | | Home Assistant | `ha.apps.lab.home.arpa` | No native OIDC support. Would require a custom integration or Traefik ForwardAuth (which only gates the ingress, not HA's own auth layer). Not worth the complexity. | -| LiteLLM | `ai.apps.lab.home.arpa` | Supports generic OIDC via `GENERIC_CLIENT_ID` / `GENERIC_CLIENT_SECRET` / endpoint env vars. Not yet implemented — straightforward next step. | +| LiteLLM | `ai.apps.lab.home.arpa` | ForwardAuth at ingress level (see Completed above). Native OIDC via `GENERIC_CLIENT_ID` would require a database — not worth it for a single-user lab. | | Vault | No public ingress | Vault has a native OIDC auth method. Useful for ditching Vault tokens for day-to-day admin. Not yet implemented. | | LLDAP | `lldap.apps.lab.home.arpa` | LLDAP is the identity provider — SSO into it would be circular. Skip. | | Whisper/STT | `stt.apps.lab.home.arpa` | API only, no interactive web UI. | diff --git a/gitops/workloads/ai-gateway/ingress.yaml b/gitops/workloads/ai-gateway/ingress.yaml index 22d3859..62c36f3 100644 --- a/gitops/workloads/ai-gateway/ingress.yaml +++ b/gitops/workloads/ai-gateway/ingress.yaml @@ -7,6 +7,8 @@ metadata: traefik.ingress.kubernetes.io/router.entrypoints: web,websecure traefik.ingress.kubernetes.io/router.tls: "true" cert-manager.io/cluster-issuer: lab-ca + # ForwardAuth: UI requires Authelia login; /v1/ and /health/ bypass (see authelia configmap). + traefik.ingress.kubernetes.io/router.middlewares: authelia-forwardauth@kubernetescrd spec: ingressClassName: traefik tls: diff --git a/gitops/workloads/authelia/configmap.yaml b/gitops/workloads/authelia/configmap.yaml index 7a315a9..24e6091 100644 --- a/gitops/workloads/authelia/configmap.yaml +++ b/gitops/workloads/authelia/configmap.yaml @@ -42,6 +42,15 @@ data: access_control: default_policy: one_factor + rules: + # LiteLLM API paths bypass auth — callers use the internal ClusterIP or + # pass a LITELLM_MASTER_KEY bearer token; UI path (/) requires login. + - domain: ai.apps.lab.home.arpa + resources: + - "^/v1/.*$" + - "^/health/.*$" + - "^/openapi\\.json$" + policy: bypass identity_providers: oidc: @@ -97,20 +106,6 @@ data: userinfo_signed_response_alg: none consent_mode: implicit - - client_id: zot - client_name: Zot Registry - client_secret: '$argon2id$v=19$m=65536,t=3,p=4$Ro4ELmePXJFwTwBlWNxFGQ$bw3XbjvJ5/S9CXRF4wcnhjc7Cn7uiUM8FIeBj9T7om0' - authorization_policy: one_factor - claims_policy: include_profile_in_id_token - redirect_uris: - - https://registry.apps.lab.home.arpa/auth/callback/dex - scopes: [openid, profile, email] - response_types: [code] - grant_types: [authorization_code] - token_endpoint_auth_method: client_secret_post - userinfo_signed_response_alg: none - consent_mode: implicit - - client_id: semaphore client_name: Semaphore client_secret: '$argon2id$v=19$m=65536,t=3,p=4$ak41fOyxfpxcZ8l6HyvSDA$yzIglnptMfbfxsjg+bSHBY24c2ajT4Pa2drTTzu3Xs0' From 1daf03338e9f61b34d8ce91646b9fcc4d3740a8c Mon Sep 17 00:00:00 2001 From: Scott Wares Date: Thu, 23 Jul 2026 18:00:56 +0000 Subject: [PATCH 5/7] docs: session notes 2026-07-23 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Captures work done and deferred across the 2026-07-14–23 sessions: - healthchecks.io dead-man's switch (5 checks) - Groq + Gemini cloud fallback in LiteLLM - LiteLLM ForwardAuth SSO via Authelia - Kyverno PolicyExceptions cleanup - lldap SQLite backup wired - OVMS / Vault OIDC / rknpu / offsite Track 2 deferred with rationale - Current lab state snapshot --- TODO-2026-07-23.md | 174 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 TODO-2026-07-23.md diff --git a/TODO-2026-07-23.md b/TODO-2026-07-23.md new file mode 100644 index 0000000..eec28a8 --- /dev/null +++ b/TODO-2026-07-23.md @@ -0,0 +1,174 @@ +# Homelab Session Notes — 2026-07-23 + +Work completed and deferred across two context windows. Picks up from the +2026-07-14 audit. All changes went through GitOps (PRs → ArgoCD sync) unless +otherwise noted. + +--- + +## ✅ Completed + +### Reliability / Bug Fixes + +- [x] **Fix Authelia Progressing (orphaned PVC)** — `authelia-data` PVC was defined in git + but never mounted by the Deployment. `local-path` WaitForFirstConsumer binding kept it + Pending indefinitely → ArgoCD Progressing for 3+ days. Deleted `pvc.yaml`; ArgoCD pruned + the PVC. App went Healthy immediately. + `gitops/workloads/authelia/pvc.yaml` *(deleted, PR `fix/authelia-orphan-pvc`)* + +- [x] **Fix Zot registry CrashLoopBackOff** — `revert/zot-oidc` PR had missed + `deployment.yaml`. Main branch still referenced `zot-oidc-config` Secret volume and + `lab-ca-bundle` ConfigMap volume (both deleted). Zot crashed with FailedMount + OIDC + dex error. Overwrote to clean htpasswd-only version. + `gitops/workloads/registry/deployment.yaml` *(PR `fix/zot-deployment-revert`)* + +- [x] **Remove stale Kyverno PolicyExceptions** — both exceptions were untracked (never + committed). Deleted local files; deleted cluster objects directly. + - `allow-ollama-latest` (ai-gateway) — ollama pinned to `0.32.1` ✓ + - `allow-whisper-latest-cpu` (whisper) — whisper pinned to `0.6.0-rc.3-cpu` ✓ + `disallow-latest-tag` ClusterPolicy comment "zero PolicyExceptions in cluster" now true. + +### Backup & Observability + +- [x] **Track 1 cloud backup — Cloudflare R2** — daily at 03:00, restic to R2 free tier + (10 GB). Sources: etcd snapshots, Vault snapshots, lldap SQLite (kubectl cp), Postgres + dumps for Authelia/Immich/Semaphore. Credentials in Vault at `secret/lab/cloudflare-r2`. + `ansible/playbooks/backup-cloud.yml` *(PR `feat/cloud-backup-track1`)* + +- [x] **lldap SQLite backup** — k8s CronJob already existed in git + (`gitops/workloads/lldap/backup-cronjob.yaml`); restic → cold-8t via NFS daily at 02:30. + Added `kubectl cp` of `users.db` to `backup-cloud.sh` (offsite). Removed stale + `/mnt/cold-8t/lldap-snapshots` SOURCES entry (old VM-era path). + `ansible/playbooks/backup-cloud.yml` · `docs/RUNBOOK.md` + +- [x] **healthchecks.io dead-man's switch** — 5 checks wired: + + | Check | Period | Host | + |---|---|---| + | `h4-core` | 5 min | h4-core (heartbeat) | + | `backup-cloud` | 25h | h4-core | + | `backup-nas` | 25h | h4-core | + | `backup-etcd` | 8d | h4-core (weekly timer) | + | `backup-vault` | 25h | rpi5 | + + `h4-heartbeat.timer` fires every 5 min. Backup services use `ExecStartPost=/usr/local/bin/hc-ping`. + Ping URLs in Vault at `secret/lab/healthchecks`. Note: `backup-etcd` is **weekly** not daily. + `ansible/playbooks/healthchecks.yml` *(PR `feat/healthchecks-deadmans-switch`)* + +### AI Gateway + +- [x] **Groq + Gemini cloud fallback in LiteLLM** — `cloud` model group added: + `groq/llama-3.3-70b-versatile` + `gemini/gemini-2.0-flash`. Router fallbacks configured + so `chat` and `chat-cpu` automatically escalate to `cloud` on failure. Keys in Vault at + `secret/lab/cloud-ai`; injected via ESO Secret `litellm-cloud-ai`. + NetworkPolicy updated to allow litellm HTTPS egress to public internet. + Full escalation ladder: + ``` + chat → RKLLama NPU (DeepSeek 1.5B, on-prem, free) + ↓ auto-fallback on failure + cloud → Groq llama-3.3-70b / Gemini 2.0 Flash (cloud, free tier) + ↓ auto-fallback on failure + big → Claude (commented out; enable with ANTHROPIC_API_KEY) + ``` + `gitops/workloads/ai-gateway/configmap.yaml` · `deployment.yaml` · + `network-policy.yaml` · `external-secret-cloud-ai.yaml` + +- [x] **LiteLLM UI gated by Authelia ForwardAuth** — ForwardAuth middleware added to + `ai` ingress. Authelia bypass rules added for `/v1/.*`, `/health/.*`, `/openapi\.json` + so API callers are unaffected. UI at `https://ai.apps.lab.home.arpa` requires + one_factor Authelia login. + `gitops/workloads/ai-gateway/ingress.yaml` · + `gitops/workloads/authelia/configmap.yaml` · `docs/SSO.md` + +### Documentation + +- [x] **GITOPS-STUDY-GUIDE.md** — added Reloader section, orphaned PVC pattern, ArgoCD + Progressing diagnostic commands (jq resource-health query, app controller log). + +- [x] **RUNBOOK.md** — updated backup table (added lldap CronJob + backup-cloud rows, + removed stale backup-lldap row); fixed lldap TODO comment; added cloud backup check commands. + +- [x] **OPS.md** — added `h4-heartbeat.timer` status check; added healthchecks.io + check section with verify commands. + +- [x] **STANDUP.md** — marked Track 1 backup ✅, healthchecks.io ✅; fixed + `` placeholder → `ai.apps.lab.home.arpa`; updated cloud verify command. + +- [x] **AI-INFERENCE.md** — documented cloud fallback tier (Groq/Gemini) and + paid cloud tier (Claude, commented out). + +- [x] **SSO.md** — moved LiteLLM to Completed (ForwardAuth approach documented); + noted native OIDC requires DB, not worth it for single-user lab. + +- [x] **services.md** — updated LiteLLM entry with cloud fallback detail; fixed OVMS + status note. + +--- + +## 🔵 Deferred — with rationale + +- [ ] **OVMS (OpenVINO Model Server)** — systemd/Docker service, not a k8s workload. + Root issue: `/dev/dri` missing on h4-core (i915 not loaded on Linux 5.15); crash-looping + on n150-1 for unknown reason. Image uses `openvino/model_server:latest` (Kyverno + violation). No models configured beyond `config.json.example`. Embeddings already covered + by `nomic-embed-text` via Ollama; STT covered by Whisper. + **Defer until:** concrete use case needing iGPU acceleration (e.g. bge-large embeddings); + requires `/dev/dri` triage on n150-1 + versioned image pin + model IR prep. + +- [ ] **rknpu 0.9.7 upgrade on OPi 5 Pros** — 0.9.6 caps NPU memory allocation at ~2.2 GB, + blocking all 3B+ models (Llama-3.2-3B, Qwen3-4B). Current working model: DeepSeek 1.5B. + **Defer until:** a 3B+ model worth running is identified; check vendor wiki for 0.9.7 before + upgrading (RKLLM runtime version must match converted model). + +- [ ] **Vault OIDC auth method** — would allow `vault login -method=oidc` using Authelia + credentials. Key concern: circular dependency (Vault holds Authelia's secrets; if Vault is + sealed, OIDC auth can't work — root token still needed for break-glass). Additional + complexity: callback URL routing for rpi5 (no public ingress), lab CA trust on rpi5. + Gain is mostly audit log quality and avoiding manual token management — low ROI for + single-user lab. + **Defer until:** multi-user scenario or tightening audit requirements. + +- [ ] **Offsite backup Track 2 (~1.5 TB photos/video)** — cold-tier RAID is the only + redundant copy of the Immich library + NAS originals. Backblaze B2 ~$9/month for 1.5 TB. + `backup-offsite.service` template is wired; needs `offsite_restic_repo` in group_vars + + `/etc/restic/offsite.env`. See `docs/RUNBOOK.md` Track 2 section. + **Defer until:** cost is accepted; confirm total volume first with + `du -sh /mnt/cold-8t/immich /srv/nas`. + +- [ ] **Remote access (Tailscale / WireGuard)** — explicitly deferred by choice. + +- [ ] **UPS + NUT** — H4 is a single failure domain for NAS + k3s. Deferred by choice. + +- [ ] **OpenTelemetry** — no active distributed tracing use case. Deferred. + +- [ ] **Vector DB / RAG (Qdrant)** — embeddings ready (nomic-embed-text via Ollama) + but no RAG use case defined yet. Deferred. + +--- + +## Current lab state + +``` +DNS Pi-hole primary (octopi .148) + dnsmasq secondary (opi-zero2w-1 .184, opi-zero2w-3 .217) +Identity lldap (k3s, lldap ns) + Authelia OIDC (k3s, authelia ns) +Secrets Vault (rpi5 .128) + ESO → k8s Secrets +PKI cert-manager + lab-ca ClusterIssuer +NAS smbd + nfs on H4 (off-limits) +Backups backup-nas (daily 01:30), backup-etcd (weekly), backup-vault (daily 02:30), + backup-cloud (daily 03:00 → R2), lldap-backup CronJob (daily 02:30) + healthchecks.io: 5 checks active +k3s 3-server HA (H4 + n150-1 + n150-2), 2 agents (opi5pro-1/2), kube-vip .200 +GitOps ArgoCD + workloads ApplicationSet; selfHeal + prune +Ingress Traefik + lab-ca TLS on all services +Policy Kyverno 3.3.4 — 3 ClusterPolicies Enforce, 0 PolicyExceptions +SSO Grafana, ArgoCD, Immich, MinIO, Semaphore, LiteLLM (ForwardAuth) +AI RKLLama NPU (DeepSeek 1.5B) → cloud (Groq/Gemini) → [Claude, commented] + Ollama CPU (qwen2.5:7b, qwen2.5-coder, llava, llama3.2:3b, nomic-embed-text) + Whisper STT (stt.apps.lab.home.arpa) + M5Stack 3-tier escalation (local NPU → Claude API → Claude Code) +Monitoring Grafana + Prometheus + Loki + Alertmanager + ArgoCD Notifications +Photos Immich (k3s, immich ns) +Home Home Assistant (rpi4b .116) +Registry Zot (htpasswd, registry.apps.lab.home.arpa) +IaC Semaphore (Ansible UI) + OpenTofu (VMs) + Renovate (image pins) +``` From 62a0ed3e15128dfa329fba89619217a0b09689a4 Mon Sep 17 00:00:00 2001 From: Scott Wares Date: Thu, 23 Jul 2026 18:19:03 +0000 Subject: [PATCH 6/7] feat: back up OpenTofu state from MinIO to Cloudflare R2 Add awscli install and a tofu-state copy step to backup-cloud.yml. Before the nightly restic snapshot, the script now pulls the tofu-state bucket from MinIO (using root creds from Vault secret/lab/minio) into $TMPDIR, which restic then includes in the R2 snapshot. MinIO creds are written to /etc/restic/cloud.env alongside the existing R2 credentials so no separate env file is needed. aws s3 cp overrides AWS_ACCESS_KEY_ID/SECRET with the MinIO root creds for that one call; the rest of the script uses the R2 creds from the EnvironmentFile as before. --- ansible/playbooks/backup-cloud.yml | 34 ++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/ansible/playbooks/backup-cloud.yml b/ansible/playbooks/backup-cloud.yml index 1ef0606..f1982b0 100644 --- a/ansible/playbooks/backup-cloud.yml +++ b/ansible/playbooks/backup-cloud.yml @@ -6,6 +6,7 @@ # - lldap database snapshots (/mnt/cold-8t/lldap-snapshots) # - Vault raft snapshots (/mnt/cold-8t/vault-snapshots, if present) # - Postgres dumps (Authelia, Immich, Semaphore — via kubectl exec) +# - OpenTofu state (tofu-state bucket in MinIO — aws s3 cp via S3 API) # # Prerequisites: # 1. Create R2 bucket named 'homelab-backup' in the Cloudflare dashboard. @@ -31,6 +32,12 @@ backup_script: /usr/local/bin/backup-cloud.sh tasks: + - name: Ensure awscli is installed (needed for MinIO tofu-state copy) + ansible.builtin.apt: + name: awscli + state: present + update_cache: false + - name: Ensure /etc/restic exists ansible.builtin.file: path: /etc/restic @@ -54,6 +61,21 @@ r2: "{{ (vault_r2_raw.stdout | from_json).data.data }}" no_log: true + - name: Read MinIO credentials from Vault + ansible.builtin.shell: + cmd: vault kv get -format=json secret/lab/minio + environment: + VAULT_ADDR: "{{ vault_addr }}" + VAULT_TOKEN: "{{ lookup('env', 'VAULT_TOKEN') }}" + register: vault_minio_raw + no_log: true + changed_when: false + + - name: Parse MinIO credentials + ansible.builtin.set_fact: + minio: "{{ (vault_minio_raw.stdout | from_json).data.data }}" + no_log: true + - name: Write cloud restic env file ansible.builtin.copy: dest: "{{ cloud_env_file }}" @@ -65,6 +87,8 @@ AWS_ACCESS_KEY_ID={{ r2['access-key-id'] }} AWS_SECRET_ACCESS_KEY={{ r2['secret-access-key'] }} RESTIC_PASSWORD_FILE={{ cloud_password_file }} + MINIO_ROOT_USER={{ minio['root-user'] }} + MINIO_ROOT_PASSWORD={{ minio['root-password'] }} no_log: true - name: Write cloud restic password file @@ -112,6 +136,16 @@ -o jsonpath='{.items[0].metadata.name}') kubectl cp "lldap/${LLDAP_POD}:/data/users.db" "$TMPDIR/lldap.db" + log "Copying OpenTofu state from MinIO..." + mkdir -p "$TMPDIR/tofu-state" + AWS_ACCESS_KEY_ID="$MINIO_ROOT_USER" \ + AWS_SECRET_ACCESS_KEY="$MINIO_ROOT_PASSWORD" \ + aws s3 cp s3://tofu-state/ "$TMPDIR/tofu-state/" \ + --endpoint-url https://minio.apps.lab.home.arpa \ + --recursive \ + --no-verify-ssl \ + --quiet + # Build source list — include vault-snapshots only if the directory exists SOURCES=( /mnt/cold-8t/k3s-etcd-snapshots From d956845af279947f4663ac498b415a159ba6d1f2 Mon Sep 17 00:00:00 2001 From: Scott Wares Date: Thu, 23 Jul 2026 19:21:08 +0000 Subject: [PATCH 7/7] fix: cap systemd journal to 500M on all Linux hosts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unbounded journal growth on n150-1 caused disk pressure (91% of 49G root) → kubelet evicted all monitoring namespace pods including Grafana, Alertmanager, and Prometheus (2026-07-23). Adds a journald drop-in (SystemMaxUse=500M, SystemKeepFree=1G) and vacuums existing journals on first run. Targets all:!x86_nodes:!embedded:!standalone_vms. --- ansible/playbooks/journald.yml | 73 ++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 ansible/playbooks/journald.yml diff --git a/ansible/playbooks/journald.yml b/ansible/playbooks/journald.yml new file mode 100644 index 0000000..c21115f --- /dev/null +++ b/ansible/playbooks/journald.yml @@ -0,0 +1,73 @@ +--- +# journald.yml — cap systemd journal size on all Linux hosts. +# +# Default Ubuntu/Debian installs have no SystemMaxUse limit; journals grow +# unbounded and can fill the root filesystem. On n150-1 this caused disk +# pressure and a Grafana/monitoring namespace mass-eviction (2026-07-23). +# +# Limits applied: +# SystemMaxUse=500M — total journal storage cap +# SystemKeepFree=1G — always keep 1G free on the filesystem +# +# Run: +# ansible-playbook -i inventory/hosts.yml playbooks/journald.yml \ +# --vault-password-file .vault_pass --check +# ansible-playbook -i inventory/hosts.yml playbooks/journald.yml \ +# --vault-password-file .vault_pass +# +# Limit to a single host: +# ansible-playbook ... -l n150-1 + +- name: Cap systemd journal size on all Linux hosts + hosts: all:!x86_nodes:!embedded:!standalone_vms + become: true + gather_facts: true + + tasks: + - name: Skip non-systemd hosts + ansible.builtin.meta: end_host + when: ansible_service_mgr != "systemd" + + - name: Ensure journald drop-in directory exists + ansible.builtin.file: + path: /etc/systemd/journald.conf.d + state: directory + mode: "0755" + owner: root + group: root + + - name: Write journald size cap drop-in + ansible.builtin.copy: + dest: /etc/systemd/journald.conf.d/max-use.conf + mode: "0644" + owner: root + group: root + content: | + # Managed by Ansible — ansible/playbooks/journald.yml + # Caps journal to 500M and keeps 1G free on filesystem. + # Without this, Ubuntu defaults to 10% of filesystem (4.9G on n150-1's + # 49G root) which caused disk pressure + pod eviction on 2026-07-23. + [Journal] + SystemMaxUse=500M + SystemKeepFree=1G + register: journald_conf + + - name: Restart systemd-journald to apply new limits + ansible.builtin.systemd: + name: systemd-journald + state: restarted + when: journald_conf.changed + + - name: Vacuum journals to enforce new size limit immediately + ansible.builtin.command: journalctl --vacuum-size=500M + when: journald_conf.changed + changed_when: true + + - name: Report current journal disk usage + ansible.builtin.command: journalctl --disk-usage + register: disk_usage + changed_when: false + + - name: Show journal usage on {{ inventory_hostname }} + ansible.builtin.debug: + msg: "{{ inventory_hostname }}: {{ disk_usage.stdout }}"