From a8663a5233ee8d24f116ef272ffea0d30d69a194 Mon Sep 17 00:00:00 2001 From: Scott Wares Date: Sat, 25 Jul 2026 19:16:32 +0000 Subject: [PATCH 1/2] fix(monitoring): add the ExternalSecret PR #NNN missed alertmanagerSpec.secrets referenced alertmanager-endpoints, but the ExternalSecret that creates it was a new file and never got staged (git commit -am only stages tracked files). Alertmanager has been stuck on FailedMount since the merge, so the cluster currently has no alerting at all. --- .../alertmanager-externalsecret.yaml | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 gitops/workloads/monitoring/alertmanager-externalsecret.yaml diff --git a/gitops/workloads/monitoring/alertmanager-externalsecret.yaml b/gitops/workloads/monitoring/alertmanager-externalsecret.yaml new file mode 100644 index 0000000..2b8dd28 --- /dev/null +++ b/gitops/workloads/monitoring/alertmanager-externalsecret.yaml @@ -0,0 +1,49 @@ +--- +# Alertmanager notification endpoints, pulled from Vault. +# +# WHY THESE ARE SECRETS +# +# ntfy has no accounts — the topic name IS the password. Anyone who knows the +# topic can both read every alert this lab emits and publish fake ones. Same for +# the healthchecks.io ping URL: its UUID is the only credential. Neither may be +# committed, so both live in Vault and are mounted into the Alertmanager pod. +# +# Storing the FULL URL (not just the topic) also means the query string — +# including ntfy's message-template parameters, priority and tags — can be tuned +# without a commit and an Argo sync. Alertmanager reads these files at +# /etc/alertmanager/secrets/alertmanager-endpoints/ via `url_file`. +# +# SEED VAULT BEFORE ARGO SYNCS THIS: +# +# # Pick an unguessable topic — this is the password. +# TOPIC="lab-$(openssl rand -hex 16)" +# +# vault kv patch secret/lab/alertmanager \ +# ntfy-url="https://ntfy.sh/${TOPIC}?priority=5&tags=rotating_light" \ +# watchdog-url="https://hc-ping.com/" +# +# Then subscribe to ${TOPIC} in the ntfy app, and create a healthchecks.io check +# named `alertmanager-watchdog` with Period 15m / Grace 10m — see the Watchdog +# receiver in gitops/apps/monitoring.yaml for why that cadence. +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: alertmanager-endpoints + namespace: monitoring +spec: + refreshInterval: 1h + secretStoreRef: + name: vault-backend + kind: ClusterSecretStore + target: + name: alertmanager-endpoints + creationPolicy: Owner + data: + - secretKey: ntfy-url + remoteRef: + key: secret/lab/alertmanager + property: ntfy-url + - secretKey: watchdog-url + remoteRef: + key: secret/lab/alertmanager + property: watchdog-url From 236d05ed41080fc33198a6a0851c5a8c1bbcb5e5 Mon Sep 17 00:00:00 2001 From: Scott Wares Date: Sat, 25 Jul 2026 21:19:31 +0000 Subject: [PATCH 2/2] feat(lldap): move from SQLite to PostgreSQL so the pod can reschedule lldap's SQLite database lived on a local-path PVC, which carries node affinity. On 2026-07-25 n150-2 wedged twice; both times the replacement lldap pod was unschedulable because its volume only existed on that node, and ALL SSO went down with it - ArgoCD, Grafana, Immich, MinIO and Semaphore all authenticate through Authelia, which needs lldap. With Postgres, lldap itself is stateless and schedules anywhere. Scope, honestly: Postgres still uses a local-path PVC, so it is node-bound. What changes is that the binding is deliberate rather than accidental, and Postgres survives unclean shutdown far better than SQLite - which matters, since a forced reboot corrupted an etcd bbolt file on this same hardware the same day. Pinned to n150-1 beside authelia-postgres: Authelia and lldap are a coupled unit, so co-locating their databases means SSO depends on one node failing rather than two. Also updated, each of which would have failed silently: - NetworkPolicy: lldap has default-deny egress, so without a 5432 rule it cannot reach its own database and simply never becomes ready. - backup CronJob: it mounted the SQLite PVC and copied users.db, a file that no longer exists. Now runs pg_dump, and asserts the dump exceeds 1000 bytes and contains CREATE TABLE - an empty dump is more dangerous than a missing one because it looks like success. lldap-data PVC is intentionally left in git as the rollback path; it still holds users.db. Remove it in a follow-up once verified - with prune: true, deleting it from git destroys the data. There is no SQLite->Postgres migration path in lldap, so users, groups and memberships are recreated by hand and passwords reset. The admin account rebuilds from LLDAP_LDAP_USER_PASS, so Authelia's bind keeps working. Refs: docs/REVIEW-2026-07-24.md H10 --- docs/REVIEW-2026-07-24.md | 2 +- gitops/workloads/lldap/backup-cronjob.yaml | 53 ++++++--- gitops/workloads/lldap/deployment.yaml | 34 ++++-- gitops/workloads/lldap/external-secret.yaml | 8 ++ gitops/workloads/lldap/network-policy.yaml | 43 +++++++- gitops/workloads/lldap/postgres.yaml | 116 ++++++++++++++++++++ 6 files changed, 231 insertions(+), 25 deletions(-) create mode 100644 gitops/workloads/lldap/postgres.yaml diff --git a/docs/REVIEW-2026-07-24.md b/docs/REVIEW-2026-07-24.md index a9a9733..c400887 100644 --- a/docs/REVIEW-2026-07-24.md +++ b/docs/REVIEW-2026-07-24.md @@ -302,7 +302,7 @@ directory carries the password. `LAB-URLS.md` has the same shape of problem | # | Finding | File | |---|---------|------| -| H25 | **FIXED 2026-07-25** (branch `fix/alert-routing`). Was: Alertmanager had exactly two receivers, `null` and `m5stack` (an HTTP POST to an ESP32 on the LAN), so asleep or display-off meant every alert was silently discarded. Now `severity=critical` also routes to ntfy (topic URL in Vault, `continue: true` so the display still sees it), and the always-firing `Watchdog` alert — previously discarded to `null` — forwards to a healthchecks.io check, making the monitoring stack its own dead-man's switch. ArgoCD notifications still go only to the M5Stack; separate item. | `gitops/apps/monitoring.yaml` | +| H25 | **FIXED and verified on the cluster 2026-07-25** — a test alert at `severity=critical` reached the ntfy tablet, the ntfy web app, AND the M5Stack simultaneously, confirming `continue: true` routes to both rather than replacing one with the other. (branch `fix/alert-routing`). Was: Alertmanager had exactly two receivers, `null` and `m5stack` (an HTTP POST to an ESP32 on the LAN), so asleep or display-off meant every alert was silently discarded. Now `severity=critical` also routes to ntfy (topic URL in Vault, `continue: true` so the display still sees it), and the always-firing `Watchdog` alert — previously discarded to `null` — forwards to a healthchecks.io check, making the monitoring stack its own dead-man's switch. ArgoCD notifications still go only to the M5Stack; separate item. | `gitops/apps/monitoring.yaml` | | H26 | `kubeEtcd: enabled: false` — **zero etcd metrics, therefore zero quorum-loss alerting** on a 3-node HA cluster | `gitops/apps/monitoring.yaml:221-222` | | H27 | `LabBackupEtcdSilent` fires at >25 h against a **weekly** timer — permanently critical 6 days out of 7, training you to ignore the one alert group that matters most (see C1) | `lab-alerts.yaml:80-95` vs `backup-etcd.timer.j2:5` | | H28 | **FIXED 2026-07-25.** Dropped decommissioned `192.168.1.70`; added `.116` (Pi-hole secondary), `.217` (4th DNS), `.128` (**Vault** — every ExternalSecret depends on it) and `.64` (build agent). Also found and fixed a second stale reference: `LabVMUnexpectedShutoff` filtered on `domain=~"ldap-1"`, a VM decommissioned 2026-07-04, so the rule had been incapable of firing for three weeks — retargeted at `gitlab-1`. | `gitops/apps/monitoring.yaml`, `lab-alerts.yaml:137` | diff --git a/gitops/workloads/lldap/backup-cronjob.yaml b/gitops/workloads/lldap/backup-cronjob.yaml index c88833f..16c434f 100644 --- a/gitops/workloads/lldap/backup-cronjob.yaml +++ b/gitops/workloads/lldap/backup-cronjob.yaml @@ -4,7 +4,11 @@ metadata: name: lldap-backup namespace: lldap spec: - schedule: "30 2 * * *" # 02:30 daily — after backup-nas starts, before backup-vault + # 02:30 daily. NOTE: this overlaps the host backup-nas timer (01:30) writing to + # the SAME restic repo. Two restic processes on one repo contend for the repo + # lock. Tracked in docs/REVIEW-2026-07-24.md; left as-is here to keep this PR + # to the Postgres migration. + schedule: "30 2 * * *" concurrencyPolicy: Forbid successfulJobsHistoryLimit: 3 failedJobsHistoryLimit: 3 @@ -16,14 +20,24 @@ spec: app: lldap-backup spec: restartPolicy: OnFailure - # No nodeSelector — PV affinity schedules this to the correct node automatically. - # NFS volume is mounted by kubelet on the host; no SSH or pod-level egress needed. + # No PVC mount any more — lldap's data lives in Postgres as of + # 2026-07-25, so this dumps the database over the network instead of + # reading a SQLite file. That also means this job is no longer pinned + # by PV affinity and can run on any node. securityContext: runAsUser: 0 containers: - name: backup - image: restic/restic:0.19.1 + # postgres image for pg_dump; restic is fetched from the same + # alpine base. Using one image avoids an init container. + image: postgres:16-alpine imagePullPolicy: IfNotPresent + env: + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: lldap-secrets + key: db_password command: - /bin/sh - -c @@ -32,20 +46,35 @@ spec: REPO="/restic-repo" export RESTIC_PASSWORD_FILE=/secrets/restic-password - # Initialize repo if not already (no-op if it exists) + apk add --no-cache restic > /dev/null + restic -r "$REPO" snapshots > /dev/null 2>&1 || restic -r "$REPO" init - echo "Starting lldap SQLite backup..." - restic -r "$REPO" backup /data/users.db \ + echo "Dumping lldap database..." + pg_dump -h lldap-postgres -U lldap -d lldap > /tmp/lldap.sql + + # A dump that is well-formed but EMPTY is more dangerous than a + # missing one, because it looks like a successful backup. This + # is the same failure shape as the 4 KB etcd "snapshots" and the + # empty pg_dumps found on 2026-07-25 — assert on content. + size=$(wc -c < /tmp/lldap.sql) + if [ "$size" -lt 1000 ]; then + echo "FAIL: dump is only ${size} bytes — refusing to back up an empty database" >&2 + exit 1 + fi + grep -q "CREATE TABLE" /tmp/lldap.sql || { + echo "FAIL: dump contains no CREATE TABLE statements" >&2 + exit 1 + } + echo " dump ok (${size} bytes)" + + restic -r "$REPO" backup /tmp/lldap.sql \ --tag lldap \ --host lldap-k8s echo "Backup complete. Recent lldap snapshots:" restic -r "$REPO" snapshots --tag lldap --last 5 volumeMounts: - - name: lldap-data - mountPath: /data - readOnly: true - name: backup-secrets mountPath: /secrets readOnly: true @@ -59,10 +88,6 @@ spec: cpu: 500m memory: 256Mi volumes: - - name: lldap-data - persistentVolumeClaim: - claimName: lldap-data - readOnly: true - name: backup-secrets secret: secretName: lldap-backup-secrets diff --git a/gitops/workloads/lldap/deployment.yaml b/gitops/workloads/lldap/deployment.yaml index db6c734..c33a0fb 100644 --- a/gitops/workloads/lldap/deployment.yaml +++ b/gitops/workloads/lldap/deployment.yaml @@ -7,7 +7,11 @@ spec: replicas: 1 minReadySeconds: 30 strategy: - type: Recreate # SQLite is single-writer; never run two replicas concurrently + # Was Recreate because SQLite is single-writer. Now that state lives in + # Postgres, lldap itself is stateless and could roll normally — but keep + # Recreate for now: it is one small pod, rolling buys nothing, and Recreate + # keeps the restart behaviour identical to what has been running. + type: Recreate selector: matchLabels: app: lldap @@ -40,17 +44,29 @@ spec: secretKeyRef: name: lldap-secrets key: jwt_secret + # Postgres, not SQLite (2026-07-25). See postgres.yaml for why. + # + # DB_PASSWORD must be declared BEFORE LLDAP_DATABASE_URL — Kubernetes + # expands $(VAR) only from env vars defined earlier in this list. + # + # The password is interpolated into a URL, so it must contain no + # URL-reserved characters (@ : / ? # % &). Generate it with + # `openssl rand -hex 32`, which is hex and therefore always safe. + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: lldap-secrets + key: db_password - name: LLDAP_DATABASE_URL - value: "sqlite:///data/users.db?mode=rwc" + value: "postgres://lldap:$(DB_PASSWORD)@lldap-postgres:5432/lldap" - name: LLDAP_HTTP_URL value: "https://lldap.apps.lab.home.arpa" - name: LLDAP_LDAP_PORT value: "3890" - name: LLDAP_HTTP_PORT value: "17170" - volumeMounts: - - name: data - mountPath: /data + # No volumeMounts. State lives in Postgres now, which is the entire + # point: this pod is stateless and can schedule on any node. readinessProbe: httpGet: path: /health @@ -72,7 +88,7 @@ spec: limits: cpu: 200m memory: 128Mi - volumes: - - name: data - persistentVolumeClaim: - claimName: lldap-data + # `volumes:` removed with the SQLite migration. The lldap-data PVC is + # deliberately left in git (pvc.yaml) until the Postgres migration is + # verified — it still holds the old users.db and is the rollback path. + # Delete it in a follow-up PR once you are satisfied, not before. diff --git a/gitops/workloads/lldap/external-secret.yaml b/gitops/workloads/lldap/external-secret.yaml index 9144686..ba1b646 100644 --- a/gitops/workloads/lldap/external-secret.yaml +++ b/gitops/workloads/lldap/external-secret.yaml @@ -30,3 +30,11 @@ spec: remoteRef: key: secret/lab/lldap property: jwt_secret + # Added 2026-07-25 with the SQLite → Postgres migration. + # MUST be URL-safe: it is interpolated into LLDAP_DATABASE_URL, so no + # @ : / ? # % or & characters. Generate with `openssl rand -hex 32`. + # vault kv patch secret/lab/lldap db_password="$(openssl rand -hex 32)" + - secretKey: db_password + remoteRef: + key: secret/lab/lldap + property: db_password diff --git a/gitops/workloads/lldap/network-policy.yaml b/gitops/workloads/lldap/network-policy.yaml index bfe4365..9484967 100644 --- a/gitops/workloads/lldap/network-policy.yaml +++ b/gitops/workloads/lldap/network-policy.yaml @@ -78,8 +78,19 @@ spec: ports: - port: 22 protocol: TCP + # Added 2026-07-25: the backup now runs pg_dump against Postgres rather than + # reading a SQLite file off a mounted PVC, so it needs to reach the database. + - to: + - podSelector: + matchLabels: + app: lldap-postgres + ports: + - port: 5432 + protocol: TCP --- -# Allow lldap egress for DNS only (it doesn't call external services). +# Allow lldap egress: DNS, plus Postgres now that state lives there (2026-07-25). +# Without the 5432 rule the default-deny policy above silently blocks lldap from +# reaching its own database and the pod never becomes ready. apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: @@ -100,3 +111,33 @@ spec: protocol: UDP - port: 53 protocol: TCP + - to: + - podSelector: + matchLabels: + app: lldap-postgres + ports: + - port: 5432 + protocol: TCP +--- +# Postgres accepts connections only from lldap itself and from the backup job. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-postgres-from-lldap + namespace: lldap +spec: + podSelector: + matchLabels: + app: lldap-postgres + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: + app: lldap + - podSelector: + matchLabels: + app: lldap-backup + ports: + - port: 5432 + protocol: TCP diff --git a/gitops/workloads/lldap/postgres.yaml b/gitops/workloads/lldap/postgres.yaml new file mode 100644 index 0000000..af5f217 --- /dev/null +++ b/gitops/workloads/lldap/postgres.yaml @@ -0,0 +1,116 @@ +# PostgreSQL 16 for lldap. +# +# WHY lldap MOVED OFF SQLITE (2026-07-25) +# +# lldap's SQLite database lived on a `local-path` PVC, which carries node +# affinity. On 2026-07-25 n150-2 wedged twice; both times the lldap pod could not +# be rescheduled anywhere — its volume only existed on that node — so the +# replacement pod sat Pending and ALL SSO went down with it (ArgoCD, Grafana, +# Immich, MinIO, Semaphore all authenticate through Authelia, which needs lldap). +# See docs/REVIEW-2026-07-24.md H10. +# +# With Postgres, the lldap pod itself becomes stateless and schedules anywhere. +# +# BE HONEST ABOUT WHAT THIS DOES AND DOESN'T FIX: this database still uses a +# `local-path` PVC, so *Postgres* is node-bound. What changes is that the binding +# is now deliberate rather than accidental, and Postgres survives an unclean +# shutdown far better than SQLite — which matters, because the same day this was +# written a forced reboot corrupted an etcd bbolt file on the very same class of +# hardware. +# +# NODE PLACEMENT IS DELIBERATE: pinned to n150-1 alongside authelia-postgres. +# Authelia and lldap are already a coupled unit — neither is useful without the +# other — so co-locating their databases means SSO depends on ONE node failing, +# not two. Splitting them would double the number of nodes that can break logins. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: lldap-pgdata + namespace: lldap +spec: + accessModes: [ReadWriteOnce] + storageClassName: local-path + resources: + requests: + storage: 2Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: lldap-postgres + namespace: lldap +spec: + replicas: 1 + strategy: + type: Recreate # RWO PVC — never two writers on the same volume + selector: + matchLabels: + app: lldap-postgres + template: + metadata: + labels: + app: lldap-postgres + spec: + automountServiceAccountToken: false + # See the header: co-located with authelia-postgres on purpose. + nodeSelector: + kubernetes.io/hostname: n150-1 + containers: + - name: postgres + image: postgres:16-alpine + env: + - name: POSTGRES_DB + value: lldap + - name: POSTGRES_USER + value: lldap + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: lldap-secrets + key: db_password + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + ports: + - containerPort: 5432 + readinessProbe: + exec: + command: [pg_isready, -U, lldap, -d, lldap] + initialDelaySeconds: 5 + periodSeconds: 5 + failureThreshold: 6 + # Unlike authelia-postgres, this one has a liveness probe too — a wedged + # but listening Postgres would otherwise never be restarted, and this is + # the identity store for the whole lab. (authelia-postgres is missing + # one; tracked as H20 in the review.) + livenessProbe: + exec: + command: [pg_isready, -U, lldap, -d, lldap] + initialDelaySeconds: 30 + periodSeconds: 30 + failureThreshold: 3 + resources: + requests: + cpu: 10m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi + volumeMounts: + - name: pgdata + mountPath: /var/lib/postgresql/data + volumes: + - name: pgdata + persistentVolumeClaim: + claimName: lldap-pgdata +--- +apiVersion: v1 +kind: Service +metadata: + name: lldap-postgres + namespace: lldap +spec: + selector: + app: lldap-postgres + ports: + - port: 5432 + targetPort: 5432