From 9079cf5e97e8c4f7f234afb3ec36bdc9bc4665f3 Mon Sep 17 00:00:00 2001 From: Search Date: Sat, 4 Jul 2026 19:45:36 +0000 Subject: [PATCH 1/2] fix(helm): reduce hindsight DB pressure during liveness --- helm/hindsight/values.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/helm/hindsight/values.yaml b/helm/hindsight/values.yaml index 2ad8ec1f0..e8146c217 100644 --- a/helm/hindsight/values.yaml +++ b/helm/hindsight/values.yaml @@ -48,7 +48,7 @@ api: readinessProbe: httpGet: - path: /health + path: /version port: 8888 initialDelaySeconds: 10 periodSeconds: 5 @@ -149,7 +149,7 @@ worker: readinessProbe: httpGet: - path: /health + path: /metrics port: 8889 initialDelaySeconds: 10 periodSeconds: 5 @@ -166,6 +166,7 @@ worker: # Keep one shared slot available for reservation-less task types like retain. HINDSIGHT_API_WORKER_MAX_SLOTS: "2" HINDSIGHT_API_WORKER_CONSOLIDATION_MAX_SLOTS: "1" + HINDSIGHT_API_WORKER_RETAIN_MAX_SLOTS: "1" HINDSIGHT_API_RETAIN_MAX_CONCURRENT: "1" # Max retries before marking a task as failed HINDSIGHT_API_WORKER_MAX_RETRIES: "3" From f20515683e35bf75e4271087f1634b527d61dc95 Mon Sep 17 00:00:00 2001 From: Search Date: Wed, 8 Jul 2026 08:02:35 +0000 Subject: [PATCH 2/2] fix(helm): restore DB-aware readiness probes, fix shared slot starvation Address review feedback on PR #18: - readinessProbe now hits /health (checks DB connectivity) instead of /version|/metrics, so k8s pulls a pod from rotation on DB outage instead of routing traffic to it. livenessProbe keeps /version|/metrics to avoid restart storms from transient DB blips. - worker_max_slots bumped 2 -> 3 so the shared task-slot pool no longer collapses to zero now that retain has its own reservation; without this, file_convert_retain/refresh_mental_model/graph_maintenance/ import_documents could never be claimed. Co-Authored-By: Claude Sonnet 5 --- helm/hindsight/values.yaml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/helm/hindsight/values.yaml b/helm/hindsight/values.yaml index e8146c217..2067906a9 100644 --- a/helm/hindsight/values.yaml +++ b/helm/hindsight/values.yaml @@ -46,9 +46,12 @@ api: timeoutSeconds: 5 failureThreshold: 3 + # readinessProbe hits /health (checks DB connectivity) rather than /version: + # unlike liveness, readiness must detect a DB outage so k8s pulls the pod + # from rotation instead of routing traffic to a pod that can't serve it. readinessProbe: httpGet: - path: /version + path: /health port: 8888 initialDelaySeconds: 10 periodSeconds: 5 @@ -147,9 +150,12 @@ worker: timeoutSeconds: 5 failureThreshold: 3 + # readinessProbe hits /health (checks DB connectivity) rather than /metrics: + # unlike liveness, readiness must detect a DB outage so k8s pulls the pod + # from rotation instead of routing traffic to a pod that can't serve it. readinessProbe: httpGet: - path: /metrics + path: /health port: 8889 initialDelaySeconds: 10 periodSeconds: 5 @@ -163,8 +169,12 @@ worker: # Number of tasks to claim per poll cycle HINDSIGHT_API_WORKER_BATCH_SIZE: "1" # Bound per-worker task and DB write concurrency for the default embedded DB. - # Keep one shared slot available for reservation-less task types like retain. - HINDSIGHT_API_WORKER_MAX_SLOTS: "2" + # consolidation and retain each get a reserved slot; the remaining slot is + # shared by reservation-less task types (file_convert_retain, + # refresh_mental_model, graph_maintenance, import_documents). Keep + # MAX_SLOTS one greater than the sum of reservations below or the shared + # pool collapses to zero and those task types silently stop running. + HINDSIGHT_API_WORKER_MAX_SLOTS: "3" HINDSIGHT_API_WORKER_CONSOLIDATION_MAX_SLOTS: "1" HINDSIGHT_API_WORKER_RETAIN_MAX_SLOTS: "1" HINDSIGHT_API_RETAIN_MAX_CONCURRENT: "1"