From a5c707d5c5e7aa28e7ddb18ec83c44d070d8fbf6 Mon Sep 17 00:00:00 2001
From: Gyorgy Ruck <gyorgy@cast.ai>
Date: Tue, 5 May 2026 15:54:38 +0200
Subject: [PATCH 1/3] feat: Adding Kent enterprise support

---
 .../scripts/enable-reliability-stack.sh       | 170 +++++++++++++++++-
 1 file changed, 166 insertions(+), 4 deletions(-)

diff --git a/charts/kvisor/scripts/enable-reliability-stack.sh b/charts/kvisor/scripts/enable-reliability-stack.sh
index afadabe6..824895d4 100755
--- a/charts/kvisor/scripts/enable-reliability-stack.sh
+++ b/charts/kvisor/scripts/enable-reliability-stack.sh
@@ -557,6 +557,21 @@ def find_kvisor_path(obj, path=''):
     return None
 
 data = json.load(sys.stdin)
+
+# Karpenter Enterprise (kent) umbrellas ship an additional kent-specific
+# castai-kvisor copy alongside the generic one. Both render same-named
+# resources; only the kent copy is active for kent customers. Prefer it
+# explicitly — but validate kvisor-shaped children so a future stub or
+# rename falls through to the generic search instead of routing flags
+# into a black hole.
+kent = data.get('kent', {})
+kent_kvisor = kent.get('castai-kvisor', {}) if isinstance(kent, dict) else {}
+if isinstance(kent, dict) and kent.get('enabled') is True \
+        and isinstance(kent_kvisor, dict) \
+        and (kent_kvisor.keys() & {'agent', 'controller', 'castai', 'enabled'}):
+    print('kent.castai-kvisor')
+    sys.exit(0)
+
 path = find_kvisor_path(data)
 if path:
     print(path)
@@ -566,6 +581,9 @@ else:
 " 2>/dev/null) || VALUES_PREFIX="castai-kvisor"
       fi
       ok "Detected umbrella chart (release: $RELEASE, chart: $DETECTED_CHART_NAME)"
+      if [[ "$VALUES_PREFIX" == "kent.castai-kvisor" ]]; then
+        info "Detected kent (Karpenter Enterprise) mode — using kent.castai-kvisor prefix"
+      fi
       info "Auto-configured: --release $RELEASE --chart $CHART --values-prefix $VALUES_PREFIX"
       ;;
     *)
@@ -660,6 +678,11 @@ if kubectl $KUBECTL_CTX get crd clickhouseinstallations.clickhouse.altinity.com
     warn "ClickHouseInstallation CRD exists but operator is not running — will install operator"
   fi
 fi
+# Snapshot the pre-Phase-1 state. The second-chance detect_operator at line ~841
+# (needed for print-only mode) re-runs after Phase 1 may have installed the
+# operator, which would otherwise make Phase 2 misclassify our just-installed
+# operator as "external, pre-existing" and set operator.enabled=false on it.
+OPERATOR_PRE_EXISTING="$OPERATOR_RUNNING"
 
 if [[ -n "$OPERATOR_RUNNING" ]]; then
   : # nothing to do
@@ -880,6 +903,46 @@ fi
 # ── Phase 2: Enable Full Reliability Stack ───────────────────────────────────
 step "Phase 2: Enabling Full Reliability Stack"
 
+# Reliability metrics need both kvisor agent (OBI eBPF) and controller running.
+# Some umbrella profiles ship with agent.enabled=false by default — silently
+# leaving us without an agent. Detect that for upgrades and notify the user
+# we're flipping it on as a prerequisite. We always pass --set ...agent.enabled
+# / controller.enabled below so install mode picks up the right defaults
+# regardless of profile. Failures reading prior values are surfaced as a warn
+# (not silently swallowed) so a wedged release isn't masked.
+if [[ -z "$INSTALL_MODE" ]]; then
+  HELM_VALUES_ERR=$(mktemp)
+  PRIOR_STATE=$(helm $HELM_CTX get values -a "$RELEASE" -n "$NAMESPACE" -o json 2>"$HELM_VALUES_ERR" | VALUES_PREFIX="$VALUES_PREFIX" python3 -c "
+import sys, json, os
+data = json.load(sys.stdin)
+prefix = os.environ.get('VALUES_PREFIX', '')
+node = data
+for part in (prefix.split('.') if prefix else []):
+    node = node.get(part, {}) if isinstance(node, dict) else {}
+def norm(v):
+    if v is None: return 'unset'
+    if v is True or (isinstance(v, str) and v.lower() == 'true'): return 'enabled'
+    if v is False or (isinstance(v, str) and v.lower() == 'false'): return 'disabled'
+    return 'unknown'
+agent = norm(node.get('agent', {}).get('enabled') if isinstance(node, dict) else None)
+ctrl  = norm(node.get('controller', {}).get('enabled') if isinstance(node, dict) else None)
+print(f'{agent}|{ctrl}')
+" 2>>"$HELM_VALUES_ERR") || PRIOR_STATE="error|error"
+  PRIOR_AGENT="${PRIOR_STATE%|*}"
+  PRIOR_CTRL="${PRIOR_STATE#*|}"
+  if [[ "$PRIOR_AGENT" == "error" ]]; then
+    warn "Could not read prior release values ($(head -1 "$HELM_VALUES_ERR" 2>/dev/null || echo "unknown error")) — proceeding with chart defaults"
+  else
+    if [[ "$PRIOR_AGENT" == "disabled" ]]; then
+      info "kvisor agent is currently disabled (${VALUES_PREFIX:+${VALUES_PREFIX}.}agent.enabled=false) — enabling as a reliability-metrics prerequisite"
+    fi
+    if [[ "$PRIOR_CTRL" == "disabled" ]]; then
+      info "kvisor controller is currently disabled (${VALUES_PREFIX:+${VALUES_PREFIX}.}controller.enabled=false) — enabling as a reliability-metrics prerequisite"
+    fi
+  fi
+  rm -f "$HELM_VALUES_ERR"
+fi
+
 # Build the helm command with all reliability flags
 HELM_CMD="$(build_helm_base)"
 
@@ -896,14 +959,27 @@ else
 fi
 
 HELM_CMD="$HELM_CMD \\
+  $(setkey agent.enabled=true) \\
   $(setkey agent.reliabilityMetrics.enabled=true) \\
   $(setkey agent.reliabilityMetrics.obi.sizingProfile=$OBI_PROFILE) \\
   $(setkey agent.reliabilityMetrics.obi.dynamicSizing=$DYNAMIC_SIZING) \\
+  $(setkey controller.enabled=true) \\
   $(setkey controller.reliabilityMetrics.enabled=true) \\
   $(setkey reliabilityMetrics.enabled=true) \\
   $(setkey reliabilityMetrics.install.enabled=true) \\
   $(setkey reliabilityMetrics.exporter.enabled=true)"
 
+# In kent mode the umbrella also ships an autoscaler.castai-kvisor copy that
+# renders same-named resources but with reliability metrics off. Disabling it
+# prevents the strategic-merge collisions that produced orphan volumeMounts
+# (Deployment.apps invalid: containers[1].volumeMounts[0].name: Not found).
+# Safe for kent users — autoscaler's kvisor copy is unused there.
+if [[ "$VALUES_PREFIX" == "kent.castai-kvisor" ]]; then
+  HELM_CMD="$HELM_CMD \\
+  --set autoscaler.castai-kvisor.enabled=false"
+  info "Disabling duplicate autoscaler.castai-kvisor subchart (kent ships its own copy)"
+fi
+
 # Override the ch-exporter API key secret ref if we detected a non-default secret
 if [[ -z "$INSTALL_MODE" && -n "$DETECTED_API_KEY_SECRET" && "$DETECTED_API_KEY_SECRET" != "castai-kvisor" ]]; then
   HELM_CMD="$HELM_CMD \\
@@ -922,10 +998,11 @@ if [[ "$RELEASE" != "castai-kvisor" ]]; then
   info "Overriding ClickHouse address → $CH_ADDR"
 fi
 
-# Only skip installing our operator if an external one was already running before we started.
-# If we installed it in Phase 1 (either CRD was missing, or CRD existed but operator wasn't running),
-# we need operator.enabled=true so it stays managed by this release.
-if [[ -n "$OPERATOR_RUNNING" ]]; then
+# Only skip installing our operator if an external one was already running BEFORE
+# this script started — checked via OPERATOR_PRE_EXISTING (snapshot before Phase 1).
+# Using OPERATOR_RUNNING here would misclassify the operator that Phase 1 just
+# installed as "external" and tear it down, orphaning the ClickHouseInstallation.
+if [[ -n "$OPERATOR_PRE_EXISTING" ]]; then
   HELM_CMD="$HELM_CMD \\
   $(setkey reliabilityMetrics.operator.enabled=false)"
   info "Using existing ClickHouse operator (not installing chart's operator)"
@@ -1038,6 +1115,91 @@ if [[ -z "$DRY_RUN" ]]; then
       warn "OBI container not producing logs yet (may still be starting)"
     fi
   fi
+
+  # ── Data-pipeline verification ────────────────────────────────────────────
+  # Helm reports "deployed" the moment its manifests apply; that doesn't prove
+  # data is flowing. Probe Bronze → Silver → mothership end-to-end so a wedged
+  # collector / busted MV / unreachable mothership doesn't slip past us.
+  echo ""
+  info "Verifying data pipeline..."
+
+  # ClickHouse pod (operator-managed, label is stable across release names).
+  CH_POD=$(kubectl $KUBECTL_CTX get pod -n "$NAMESPACE" -l clickhouse.altinity.com/cluster -o name 2>/dev/null | head -1)
+  if [[ -z "$CH_POD" ]]; then
+    warn "No ClickHouse pod found (label clickhouse.altinity.com/cluster) — operator may still be reconciling the StatefulSet"
+  else
+    CH_POD_NAME="${CH_POD##*/}"
+    if kubectl $KUBECTL_CTX wait --for=condition=Ready -n "$NAMESPACE" "$CH_POD" --timeout=120s >/dev/null 2>&1; then
+      ok "ClickHouse pod ready ($CH_POD_NAME)"
+    else
+      warn "ClickHouse pod $CH_POD_NAME not Ready within 120s"
+    fi
+  fi
+
+  # Schema migrations (Helm hook Job from the ch-exporter chart).
+  MIGRATE_JOB=""
+  for cand in "${RELEASE}-clickhouse-migrate" "castai-clickhouse-migrate"; do
+    if kubectl $KUBECTL_CTX get job -n "$NAMESPACE" "$cand" >/dev/null 2>&1; then
+      MIGRATE_JOB="$cand"; break
+    fi
+  done
+  if [[ -n "$MIGRATE_JOB" ]]; then
+    if kubectl $KUBECTL_CTX wait --for=condition=Complete -n "$NAMESPACE" "job/$MIGRATE_JOB" --timeout=120s >/dev/null 2>&1; then
+      ok "Schema migrations applied (job/$MIGRATE_JOB)"
+    else
+      warn "Migration job/$MIGRATE_JOB did not Complete within 120s — kubectl logs job/$MIGRATE_JOB -n $NAMESPACE"
+    fi
+  fi
+
+  # End-to-end probe: gauge metrics from the controller's k8s_cluster receiver
+  # populate within seconds of startup, so they're the cheapest signal that the
+  # whole Bronze → Silver → exporter path is alive. Histogram tables stay empty
+  # until application traffic flows — don't gate on them at install time.
+  if [[ -n "$CH_POD" ]]; then
+    ch_query() {
+      kubectl $KUBECTL_CTX exec -n "$NAMESPACE" "$CH_POD" -c clickhouse -- \
+        clickhouse-client -d metrics -q "$1" 2>/dev/null | tr -d '[:space:]'
+    }
+    PROBE_TIMEOUT=180
+    PROBE_ELAPSED=0
+    BRONZE_ROWS=""
+    SILVER_ROWS=""
+    EXPORT_TS=""
+    while (( PROBE_ELAPSED < PROBE_TIMEOUT )); do
+      if [[ -z "$BRONZE_ROWS" ]]; then
+        v=$(ch_query "SELECT count() FROM otel_metrics_gauge WHERE TimeUnix > now() - INTERVAL 2 MINUTE")
+        [[ "$v" =~ ^[0-9]+$ && "$v" -gt 0 ]] && BRONZE_ROWS="$v"
+      fi
+      if [[ -n "$BRONZE_ROWS" && -z "$SILVER_ROWS" ]]; then
+        v=$(ch_query "SELECT count() FROM reliability_metrics_gauge WHERE timestamp > now() - INTERVAL 5 MINUTE")
+        [[ "$v" =~ ^[0-9]+$ && "$v" -gt 0 ]] && SILVER_ROWS="$v"
+      fi
+      if [[ -n "$SILVER_ROWS" && -z "$EXPORT_TS" ]]; then
+        v=$(ch_query "SELECT toUnixTimestamp(max(last_exported_time)) FROM export_progress FINAL")
+        [[ "$v" =~ ^[0-9]+$ && "$v" -gt 0 ]] && EXPORT_TS="$v"
+      fi
+      [[ -n "$BRONZE_ROWS" && -n "$SILVER_ROWS" && -n "$EXPORT_TS" ]] && break
+      sleep 10
+      PROBE_ELAPSED=$((PROBE_ELAPSED + 10))
+    done
+
+    if [[ -n "$BRONZE_ROWS" ]]; then
+      ok "Bronze: $BRONZE_ROWS gauge rows in last 2 min"
+    else
+      warn "Bronze empty after ${PROBE_TIMEOUT}s — check agent OTel collector logs (kubectl logs -c otel-collector -l $AGENT_LABEL)"
+    fi
+    if [[ -n "$SILVER_ROWS" ]]; then
+      ok "Silver: $SILVER_ROWS gauge rows aggregated (1-min windows)"
+    else
+      warn "Silver empty — Bronze→Silver Materialized View may have failed; check migration logs"
+    fi
+    if [[ -n "$EXPORT_TS" ]]; then
+      ok "ch-exporter forwarding to mothership (cursor advanced past epoch)"
+    else
+      warn "Export cursor still at epoch — kubectl logs $CH_POD -c ch-exporter for connectivity issues"
+    fi
+    info "(Histogram tables — http/grpc/db/messaging — populate when application traffic flows; not checked here.)"
+  fi
 fi
 
 # ── Summary ──────────────────────────────────────────────────────────────────

From d437710e1ae92845f8b08ab73d88d8cb57b9ab17 Mon Sep 17 00:00:00 2001
From: Gyorgy Ruck <gyorgy@cast.ai>
Date: Tue, 5 May 2026 15:56:59 +0200
Subject: [PATCH 2/3] doc: Adding Kent enterprise documentation

---
 docs/reliability-stack-installation.md | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/docs/reliability-stack-installation.md b/docs/reliability-stack-installation.md
index de7f85ac..2c766f87 100644
--- a/docs/reliability-stack-installation.md
+++ b/docs/reliability-stack-installation.md
@@ -90,6 +90,12 @@ The `enable-reliability-stack.sh` script handles the ClickHouse Operator CRD boo
 
 The script **auto-detects** whether kvisor is installed standalone (`castai-kvisor` chart) or via the CAST AI umbrella chart (`castai` chart) and adjusts the release name, chart reference, and values prefix accordingly. No manual flags needed in most cases.
 
+It also auto-handles a few umbrella-specific edge cases:
+
+- **Karpenter Enterprise (kent) mode**: when `kent.enabled=true` is detected, the script picks the `kent.castai-kvisor` values prefix and auto-injects `--set autoscaler.castai-kvisor.enabled=false` to suppress the umbrella's duplicate `castai-kvisor` copy that would otherwise strategic-merge over kent's render.
+- **Disabled-by-default kvisor agent/controller**: some umbrella profiles ship with `agent.enabled=false` (e.g. kent's cluster-proxy-only flavor). The script detects this for upgrades, prints a notice, and always passes `--set ...agent.enabled=true / controller.enabled=true` so install mode picks up the right defaults regardless of profile.
+- **Phase 3 data-pipeline verification**: after the helm upgrade succeeds, the script waits for the migrate Job + ClickHouse StatefulSet, then probes Bronze → Silver → mothership end-to-end (gauge metrics) to confirm the pipeline is actually flowing — not just that helm marked the release `deployed`.
+
 ```bash
 # Basic usage — auto-detects standalone vs umbrella, auto-detects OBI profile
 ./charts/kvisor/scripts/enable-reliability-stack.sh
@@ -156,12 +162,15 @@ helm install castai-kvisor castai-helm/castai-kvisor \
 
 > **⚠️ Umbrella Chart**
 >
-> When kvisor is installed via the `castai` umbrella chart (not standalone `castai-kvisor`), three things differ:
+> When kvisor is installed via the `castai` umbrella chart (not standalone `castai-kvisor`), four things differ:
 > 1. The API key secret is `castai-credentials` (not `castai-kvisor`)
-> 2. All `--set` keys must be prefixed to route into the kvisor subchart (e.g. `autoscaler.castai-kvisor.*` — the exact prefix depends on the umbrella chart structure)
+> 2. All `--set` keys must be prefixed to route into the kvisor subchart. The exact prefix depends on the umbrella's active profile:
+>    - **Karpenter Enterprise (kent) mode** (`kent.enabled=true`): use `kent.castai-kvisor.*` AND set `autoscaler.castai-kvisor.enabled=false` to suppress the duplicate copy
+>    - **Other umbrella variants**: typically `autoscaler.castai-kvisor.*`
 > 3. The ClickHouse service name becomes `castai-clickhouse` (not `castai-kvisor-clickhouse`)
+> 4. Some profiles disable kvisor's `agent`/`controller` by default — you must explicitly set `agent.enabled=true` and `controller.enabled=true` under the chosen prefix
 >
-> **Recommended:** Use the `enable-reliability-stack.sh` script — it auto-detects umbrella vs standalone and discovers the correct values prefix automatically.
+> **Recommended:** Use the `enable-reliability-stack.sh` script — it handles all four points automatically.
 >
 > Manual example for reference (verify the prefix for your umbrella chart version):
 > ```bash
@@ -478,6 +487,8 @@ reliabilityMetrics:
 
 ## Verification
 
+> **💡 If you used `enable-reliability-stack.sh`**, its Phase 3 already ran most of the checks below: agent DaemonSet rollout, OBI logs, migrate Job completion, ClickHouse pod readiness, Bronze + Silver gauge populating, and ch-exporter cursor advancing past epoch. Steps 1–5 below are useful when investigating after a failure or when running a manual install path.
+
 ### 1. Check Pod Status
 
 ```bash

From c5691e007287395073531750879501f460b1c18c Mon Sep 17 00:00:00 2001
From: Gyorgy Ruck <gyorgy@cast.ai>
Date: Tue, 5 May 2026 16:18:14 +0200
Subject: [PATCH 3/3] PR review followup: address 8 review findings
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

#1 Replace stale "line ~841" reference in OPERATOR_PRE_EXISTING comment
   with a section anchor — line numbers rot when the script grows.

#2 Tighten kent fast-path shape check from {agent, controller, castai,
   enabled} to {agent, controller} only. Hoisted into a shared
   KVISOR_DISCRIMINATOR constant used by both fast-path and recursive
   walk so they stay in sync. 'castai' (cluster identity stub) and
   'enabled' (too generic) could otherwise route flags into a future
   minimal stub.

#3 ch_query() now distinguishes kubectl-exec failure from empty result.
   Captures stderr to a tempfile and surfaces the first error line on
   failure ("Bronze probe failed (last clickhouse error: …)") instead
   of the misleading "Bronze empty after 180s — check OBI logs". Also
   adds an upfront SELECT 1 reachability check + LIMIT 1 on probes to
   defend against schema drift returning multi-row results.

#4 Phase 2 prerequisite-state read now uses trap-based cleanup and
   separate HELM_ERR / PY_ERR tempfiles, so the warn message can say
   "helm error: …" or "values parse error: …" rather than head -1ing a
   file that mixed both sources. trap 'rm -f' EXIT plugs the leak path
   under set -e.

#5 Kent override comment generalized — drop the verbatim
   "containers[1].volumeMounts[0].name: Not found" quote (pinned to
   current kubectl wording AND a specific container index) for
   "volumeMounts[N].name: Not found validation errors".

#6 Add a "Two-step lookup" preamble above the find_kvisor_path comment
   and a docstring change so a reader of the recursive walk realises
   the kent fast-path short-circuits before it ever runs.

#7 Add debug() calls in the new code paths (OPERATOR_PRE_EXISTING
   snapshot, Phase 2 prereq read result, kent autoscaler-disable
   injection, probe stage transitions) so --verbose surfaces "why did
   the script choose X" for the kent-specific logic.

#8 Demote manual-verification steps 1–5 from ### to #### and add an
   intro paragraph under "### Manual Verification" so the heading
   isn't immediately followed by another heading at the same level.

Verified end-to-end against the kent test cluster:
- dry-run shows all expected info lines including the new debug output
- script syntax (bash -n) clean
- markdown heading hierarchy under '## Verification' is now well-formed

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
---
 .../scripts/enable-reliability-stack.sh       | 165 +++++++++++-------
 docs/reliability-stack-installation.md        |  12 +-
 2 files changed, 111 insertions(+), 66 deletions(-)

diff --git a/charts/kvisor/scripts/enable-reliability-stack.sh b/charts/kvisor/scripts/enable-reliability-stack.sh
index 8fbcdb8b..f1cf7d0e 100755
--- a/charts/kvisor/scripts/enable-reliability-stack.sh
+++ b/charts/kvisor/scripts/enable-reliability-stack.sh
@@ -679,20 +679,25 @@ print('TYPE=none')
       if [[ -z "$VALUES_PREFIX" ]]; then
         # Discover the values path to castai-kvisor by inspecting the release's current values.
         # The umbrella chart nests kvisor under an intermediate subchart (e.g. autoscaler.castai-kvisor).
-        # We walk the YAML tree to find the path containing a 'castai-kvisor' key with kvisor-like children.
+        # Two-step lookup: an explicit kent fast-path first (below), then a generic
+        # recursive walk for non-kent umbrellas.
         VALUES_PREFIX=$(helm $HELM_CTX get values -a "$DETECTED_RELEASE" -n "$NAMESPACE" -o json 2>/dev/null | python3 -c "
 import sys, json
 
+# Discriminator keys that identify a real castai-kvisor config (vs a stub).
+# 'agent' and 'controller' are the only top-level keys that uniquely belong
+# to kvisor — 'castai' (cluster identity) and 'enabled' are too generic and
+# could appear on cluster-proxy-only or other minimal stubs.
+KVISOR_DISCRIMINATOR = {'agent', 'controller'}
+
 def find_kvisor_path(obj, path=''):
-    \"\"\"Recursively find the path to a 'castai-kvisor' key that has dict children (agent, controller, etc).\"\"\"
+    \"\"\"Recursively find a 'castai-kvisor' subtree with kvisor-shaped children.\"\"\"
     if not isinstance(obj, dict):
         return None
     for key, val in obj.items():
         current = f'{path}.{key}' if path else key
         if key == 'castai-kvisor' and isinstance(val, dict):
-            # Verify it looks like real kvisor config (has agent/controller/castai keys)
-            kvisor_keys = set(val.keys())
-            if kvisor_keys & {'agent', 'controller', 'castai', 'enabled'}:
+            if set(val.keys()) & KVISOR_DISCRIMINATOR:
                 return current
         result = find_kvisor_path(val, current)
         if result:
@@ -711,7 +716,7 @@ kent = data.get('kent', {})
 kent_kvisor = kent.get('castai-kvisor', {}) if isinstance(kent, dict) else {}
 if isinstance(kent, dict) and kent.get('enabled') is True \
         and isinstance(kent_kvisor, dict) \
-        and (kent_kvisor.keys() & {'agent', 'controller', 'castai', 'enabled'}):
+        and (set(kent_kvisor.keys()) & KVISOR_DISCRIMINATOR):
     print('kent.castai-kvisor')
     sys.exit(0)
 
@@ -1040,11 +1045,13 @@ if kubectl $KUBECTL_CTX get crd clickhouseinstallations.clickhouse.altinity.com
     warn "ClickHouseInstallation CRD exists but operator is not running — will install operator"
   fi
 fi
-# Snapshot the pre-Phase-1 state. The second-chance detect_operator at line ~841
-# (needed for print-only mode) re-runs after Phase 1 may have installed the
-# operator, which would otherwise make Phase 2 misclassify our just-installed
-# operator as "external, pre-existing" and set operator.enabled=false on it.
+# Snapshot the pre-Phase-1 state. The second-chance detect_operator in the
+# CRD/Operator Detection block (needed for print-only mode) re-runs after
+# Phase 1 may have installed the operator, which would otherwise make Phase 2
+# misclassify our just-installed operator as "external, pre-existing" and set
+# operator.enabled=false on it.
 OPERATOR_PRE_EXISTING="$OPERATOR_RUNNING"
+debug "OPERATOR_PRE_EXISTING snapshot captured: '${OPERATOR_PRE_EXISTING:-<empty>}'"
 
 if [[ -n "$OPERATOR_RUNNING" ]]; then
   : # nothing to do
@@ -1374,8 +1381,13 @@ step "Phase 2: Enabling Full Reliability Stack"
 # regardless of profile. Failures reading prior values are surfaced as a warn
 # (not silently swallowed) so a wedged release isn't masked.
 if [[ -z "$INSTALL_MODE" ]]; then
-  HELM_VALUES_ERR=$(mktemp)
-  PRIOR_STATE=$(helm $HELM_CTX get values -a "$RELEASE" -n "$NAMESPACE" -o json 2>"$HELM_VALUES_ERR" | VALUES_PREFIX="$VALUES_PREFIX" python3 -c "
+  HELM_ERR=$(mktemp)
+  PY_ERR=$(mktemp)
+  # Cleanup on any exit path (set -e, normal return, signal). Single trap covers
+  # both files; the cleanup is idempotent so re-entering the block is fine.
+  trap 'rm -f "$HELM_ERR" "$PY_ERR"' EXIT
+  PRIOR_STATE=$(helm $HELM_CTX get values -a "$RELEASE" -n "$NAMESPACE" -o json 2>"$HELM_ERR" \
+    | VALUES_PREFIX="$VALUES_PREFIX" python3 -c "
 import sys, json, os
 data = json.load(sys.stdin)
 prefix = os.environ.get('VALUES_PREFIX', '')
@@ -1390,20 +1402,27 @@ def norm(v):
 agent = norm(node.get('agent', {}).get('enabled') if isinstance(node, dict) else None)
 ctrl  = norm(node.get('controller', {}).get('enabled') if isinstance(node, dict) else None)
 print(f'{agent}|{ctrl}')
-" 2>>"$HELM_VALUES_ERR") || PRIOR_STATE="error|error"
+" 2>"$PY_ERR") || PRIOR_STATE="error|error"
   PRIOR_AGENT="${PRIOR_STATE%|*}"
   PRIOR_CTRL="${PRIOR_STATE#*|}"
+  debug "Phase 2 prereq read: PRIOR_AGENT=$PRIOR_AGENT, PRIOR_CTRL=$PRIOR_CTRL"
   if [[ "$PRIOR_AGENT" == "error" ]]; then
-    warn "Could not read prior release values ($(head -1 "$HELM_VALUES_ERR" 2>/dev/null || echo "unknown error")) — proceeding with chart defaults"
+    # Distinguish helm vs python failure source so the operator can debug the right thing.
+    if [[ -s "$HELM_ERR" ]]; then
+      warn "Could not read prior release values (helm error: $(head -1 "$HELM_ERR")) — proceeding with chart defaults"
+    elif [[ -s "$PY_ERR" ]]; then
+      warn "Could not parse prior release values (values parse error: $(head -1 "$PY_ERR")) — proceeding with chart defaults"
+    else
+      warn "Could not read prior release values (unknown error) — proceeding with chart defaults"
+    fi
   else
     if [[ "$PRIOR_AGENT" == "disabled" ]]; then
-      info "kvisor agent is currently disabled (${VALUES_PREFIX:+${VALUES_PREFIX}.}agent.enabled=false) — enabling as a reliability-metrics prerequisite"
+      info "kvisor agent resolves to disabled in the active profile (${VALUES_PREFIX:+${VALUES_PREFIX}.}agent.enabled=false) — enabling as a reliability-metrics prerequisite"
     fi
     if [[ "$PRIOR_CTRL" == "disabled" ]]; then
-      info "kvisor controller is currently disabled (${VALUES_PREFIX:+${VALUES_PREFIX}.}controller.enabled=false) — enabling as a reliability-metrics prerequisite"
+      info "kvisor controller resolves to disabled in the active profile (${VALUES_PREFIX:+${VALUES_PREFIX}.}controller.enabled=false) — enabling as a reliability-metrics prerequisite"
     fi
   fi
-  rm -f "$HELM_VALUES_ERR"
 fi
 
 # Build the helm command with all reliability flags
@@ -1439,13 +1458,14 @@ HELM_CMD="$HELM_CMD \\
 
 # In kent mode the umbrella also ships an autoscaler.castai-kvisor copy that
 # renders same-named resources but with reliability metrics off. Disabling it
-# prevents the strategic-merge collisions that produced orphan volumeMounts
-# (Deployment.apps invalid: containers[1].volumeMounts[0].name: Not found).
+# prevents the strategic-merge collisions that orphan volumeMounts (seen as
+# `volumeMounts[N].name: Not found` validation errors from kubectl).
 # Safe for kent users — autoscaler's kvisor copy is unused there.
 if [[ "$VALUES_PREFIX" == "kent.castai-kvisor" ]]; then
   HELM_CMD="$HELM_CMD \\
   --set autoscaler.castai-kvisor.enabled=false"
   info "Disabling duplicate autoscaler.castai-kvisor subchart (kent ships its own copy)"
+  debug "Auto-injected --set autoscaler.castai-kvisor.enabled=false to suppress duplicate render"
 fi
 
 # Override the ch-exporter API key secret ref if we detected a non-default secret
@@ -1706,60 +1726,83 @@ if [[ -z "$DRY_RUN" ]]; then
   # don't gate on them at install time.
   if [[ -n "$CH_POD" && "$CH_READY" == "true" && -n "$MIGRATE_DONE" ]]; then
     info "Probing data pipeline (Bronze → Silver → mothership)..."
+    CH_QUERY_ERR=$(mktemp)
+    trap 'rm -f "$CH_QUERY_ERR"' EXIT
+    # Run a single-row clickhouse query and report:
+    #   exit 0 + stdout = numeric value (success)
+    #   exit 1 + stderr in $CH_QUERY_ERR (kubectl/exec/clickhouse failure — distinguish from empty result)
+    # `LIMIT 1` defends against schema drift returning multiple rows that would
+    # concatenate into a fake-large number after the whitespace strip.
     ch_query() {
+      local q="$1"
       kubectl $KUBECTL_CTX exec "$CH_POD" -n "$NAMESPACE" -c clickhouse -- \
-        clickhouse-client -d metrics -q "$1" 2>/dev/null | tr -d '[:space:]'
+        clickhouse-client -d metrics -q "$q LIMIT 1" 2>"$CH_QUERY_ERR" | tr -d '[:space:]'
     }
+    # If the very first probe call fails to reach ClickHouse, surface that
+    # immediately rather than waiting for the 180s timeout to mislead the user.
+    CH_REACHABLE="true"
+    if ! ch_query "SELECT 1" >/dev/null 2>&1 && [[ -s "$CH_QUERY_ERR" ]]; then
+      warn "Cannot reach ClickHouse via kubectl exec ($(head -1 "$CH_QUERY_ERR")) — skipping pipeline probe"
+      CH_REACHABLE=""
+      VERIFY_FAILURES=$((VERIFY_FAILURES + 1))
+    fi
     PROBE_TIMEOUT=180
     PROBE_ELAPSED=0
     BRONZE_ROWS=""
     SILVER_ROWS=""
     EXPORT_TS=""
-    while (( PROBE_ELAPSED < PROBE_TIMEOUT )); do
-      if [[ -z "$BRONZE_ROWS" ]]; then
-        v=$(ch_query "SELECT count() FROM otel_metrics_gauge WHERE TimeUnix > now() - INTERVAL 2 MINUTE")
-        [[ "$v" =~ ^[0-9]+$ && "$v" -gt 0 ]] && BRONZE_ROWS="$v"
-      fi
-      if [[ -n "$BRONZE_ROWS" && -z "$SILVER_ROWS" ]]; then
-        v=$(ch_query "SELECT count() FROM reliability_metrics_gauge WHERE timestamp > now() - INTERVAL 5 MINUTE")
-        [[ "$v" =~ ^[0-9]+$ && "$v" -gt 0 ]] && SILVER_ROWS="$v"
+    if [[ -n "$CH_REACHABLE" ]]; then
+      while (( PROBE_ELAPSED < PROBE_TIMEOUT )); do
+        if [[ -z "$BRONZE_ROWS" ]]; then
+          v=$(ch_query "SELECT count() FROM otel_metrics_gauge WHERE TimeUnix > now() - INTERVAL 2 MINUTE")
+          [[ "$v" =~ ^[0-9]+$ && "$v" -gt 0 ]] && BRONZE_ROWS="$v" && debug "Bronze probe: $BRONZE_ROWS rows"
+        fi
+        if [[ -n "$BRONZE_ROWS" && -z "$SILVER_ROWS" ]]; then
+          v=$(ch_query "SELECT count() FROM reliability_metrics_gauge WHERE timestamp > now() - INTERVAL 5 MINUTE")
+          [[ "$v" =~ ^[0-9]+$ && "$v" -gt 0 ]] && SILVER_ROWS="$v" && debug "Silver probe: $SILVER_ROWS rows"
+        fi
+        if [[ -n "$SILVER_ROWS" && -z "$EXPORT_TS" ]]; then
+          v=$(ch_query "SELECT toUnixTimestamp(max(last_exported_time)) FROM export_progress FINAL")
+          [[ "$v" =~ ^[0-9]+$ && "$v" -gt 0 ]] && EXPORT_TS="$v" && debug "Export cursor: $EXPORT_TS"
+        fi
+        [[ -n "$BRONZE_ROWS" && -n "$SILVER_ROWS" && -n "$EXPORT_TS" ]] && break
+
+        # Progress hint every ~30s.
+        if (( PROBE_ELAPSED > 0 && PROBE_ELAPSED % 30 == 0 )); then
+          BPART="bronze …"; [[ -n "$BRONZE_ROWS" ]] && BPART="bronze ✓"
+          SPART="silver …"; [[ -n "$SILVER_ROWS" ]] && SPART="silver ✓"
+          EPART="export …"; [[ -n "$EXPORT_TS"   ]] && EPART="export ✓"
+          info "  ${PROBE_ELAPSED}s elapsed ($BPART, $SPART, $EPART)"
+        fi
+        sleep 10
+        PROBE_ELAPSED=$((PROBE_ELAPSED + 10))
+      done
+
+      if [[ -n "$BRONZE_ROWS" ]]; then
+        ok "Bronze: $BRONZE_ROWS gauge rows in last 2 min"
+      else
+        # Differentiate "ClickHouse went away mid-probe" from "no data flowing".
+        if [[ -s "$CH_QUERY_ERR" ]]; then
+          warn "Bronze probe failed (last clickhouse error: $(head -1 "$CH_QUERY_ERR")) — check ClickHouse pod health"
+        else
+          warn "Bronze empty after ${PROBE_TIMEOUT}s — check agent OTel collector logs"
+        fi
+        VERIFY_FAILURES=$((VERIFY_FAILURES + 1))
       fi
-      if [[ -n "$SILVER_ROWS" && -z "$EXPORT_TS" ]]; then
-        v=$(ch_query "SELECT toUnixTimestamp(max(last_exported_time)) FROM export_progress FINAL")
-        [[ "$v" =~ ^[0-9]+$ && "$v" -gt 0 ]] && EXPORT_TS="$v"
+      if [[ -n "$SILVER_ROWS" ]]; then
+        ok "Silver: $SILVER_ROWS gauge rows aggregated (1-min windows)"
+      else
+        warn "Silver empty — Bronze→Silver Materialized View may have failed"
+        VERIFY_FAILURES=$((VERIFY_FAILURES + 1))
       fi
-      [[ -n "$BRONZE_ROWS" && -n "$SILVER_ROWS" && -n "$EXPORT_TS" ]] && break
-
-      # Progress hint every ~30s.
-      if (( PROBE_ELAPSED > 0 && PROBE_ELAPSED % 30 == 0 )); then
-        BPART="bronze …"; [[ -n "$BRONZE_ROWS" ]] && BPART="bronze ✓"
-        SPART="silver …"; [[ -n "$SILVER_ROWS" ]] && SPART="silver ✓"
-        EPART="export …"; [[ -n "$EXPORT_TS"   ]] && EPART="export ✓"
-        info "  ${PROBE_ELAPSED}s elapsed ($BPART, $SPART, $EPART)"
+      if [[ -n "$EXPORT_TS" ]]; then
+        ok "ch-exporter forwarding to mothership (cursor advanced past epoch)"
+      else
+        warn "Export cursor still at epoch — check ch-exporter logs for connectivity issues"
+        VERIFY_FAILURES=$((VERIFY_FAILURES + 1))
       fi
-      sleep 10
-      PROBE_ELAPSED=$((PROBE_ELAPSED + 10))
-    done
-
-    if [[ -n "$BRONZE_ROWS" ]]; then
-      ok "Bronze: $BRONZE_ROWS gauge rows in last 2 min"
-    else
-      warn "Bronze empty after ${PROBE_TIMEOUT}s — check agent OTel collector logs"
-      VERIFY_FAILURES=$((VERIFY_FAILURES + 1))
-    fi
-    if [[ -n "$SILVER_ROWS" ]]; then
-      ok "Silver: $SILVER_ROWS gauge rows aggregated (1-min windows)"
-    else
-      warn "Silver empty — Bronze→Silver Materialized View may have failed"
-      VERIFY_FAILURES=$((VERIFY_FAILURES + 1))
-    fi
-    if [[ -n "$EXPORT_TS" ]]; then
-      ok "ch-exporter forwarding to mothership (cursor advanced past epoch)"
-    else
-      warn "Export cursor still at epoch — check ch-exporter logs for connectivity issues"
-      VERIFY_FAILURES=$((VERIFY_FAILURES + 1))
+      info "(Histogram tables — http/grpc/db/messaging — populate when application traffic flows; not checked here.)"
     fi
-    info "(Histogram tables — http/grpc/db/messaging — populate when application traffic flows; not checked here.)"
   fi
 
   # ── Pod status summary ────────────────────────────────────────────────────
diff --git a/docs/reliability-stack-installation.md b/docs/reliability-stack-installation.md
index b0344fc7..0684411e 100644
--- a/docs/reliability-stack-installation.md
+++ b/docs/reliability-stack-installation.md
@@ -531,7 +531,9 @@ Histogram tables only populate when something instrumentable (HTTP/gRPC/DB/messa
 
 ### Manual Verification
 
-### 1. Check Pod Status
+The steps below mirror the automated checks above. Run them after a manual install, or to investigate what failed when Phase 3 reports a problem.
+
+#### 1. Check Pod Status
 
 ```bash
 kubectl get pods -n castai-agent -l app.kubernetes.io/instance=castai-kvisor
@@ -546,7 +548,7 @@ Expected pods:
 | `chi-castai-kvisor-clickhouse-otel-0-0-0` | 2/2 (clickhouse, ch-exporter) | ClickHouse + exporter |
 | `castai-kvisor-clickhouse-operator-*` | 2/2 | Altinity operator (if `reliabilityMetrics.operator.enabled=true`) |
 
-### 2. Verify OBI Instrumentation
+#### 2. Verify OBI Instrumentation
 
 ```bash
 POD=$(kubectl get pods -n castai-agent \
@@ -560,7 +562,7 @@ Expected output:
 level=INFO msg="instrumenting process" cmd=myapp pid=1234 type=go
 ```
 
-### 3. Verify OTel Collectors
+#### 3. Verify OTel Collectors
 
 ```bash
 # Agent collector
@@ -573,7 +575,7 @@ kubectl logs -l app.kubernetes.io/name=castai-kvisor-controller \
 
 Should show `Everything is ready. Begin running and processing data.`
 
-### 4. Verify ClickHouse Data
+#### 4. Verify ClickHouse Data
 
 ```bash
 CH_POD=$(kubectl get pods -n castai-agent \
@@ -596,7 +598,7 @@ kubectl exec $CH_POD -n castai-agent -c clickhouse -- \
     FORMAT PrettyCompact"
 ```
 
-### 5. Verify ch-exporter
+#### 5. Verify ch-exporter
 
 ```bash
 kubectl logs $CH_POD -n castai-agent -c ch-exporter --tail=20