diff --git a/charts/kvisor/scripts/enable-reliability-stack.sh b/charts/kvisor/scripts/enable-reliability-stack.sh index 1e40697b..f1cf7d0e 100755 --- a/charts/kvisor/scripts/enable-reliability-stack.sh +++ b/charts/kvisor/scripts/enable-reliability-stack.sh @@ -679,20 +679,25 @@ print('TYPE=none') if [[ -z "$VALUES_PREFIX" ]]; then # Discover the values path to castai-kvisor by inspecting the release's current values. # The umbrella chart nests kvisor under an intermediate subchart (e.g. autoscaler.castai-kvisor). - # We walk the YAML tree to find the path containing a 'castai-kvisor' key with kvisor-like children. + # Two-step lookup: an explicit kent fast-path first (below), then a generic + # recursive walk for non-kent umbrellas. VALUES_PREFIX=$(helm $HELM_CTX get values -a "$DETECTED_RELEASE" -n "$NAMESPACE" -o json 2>/dev/null | python3 -c " import sys, json +# Discriminator keys that identify a real castai-kvisor config (vs a stub). +# 'agent' and 'controller' are the only top-level keys that uniquely belong +# to kvisor — 'castai' (cluster identity) and 'enabled' are too generic and +# could appear on cluster-proxy-only or other minimal stubs. +KVISOR_DISCRIMINATOR = {'agent', 'controller'} + def find_kvisor_path(obj, path=''): - \"\"\"Recursively find the path to a 'castai-kvisor' key that has dict children (agent, controller, etc).\"\"\" + \"\"\"Recursively find a 'castai-kvisor' subtree with kvisor-shaped children.\"\"\" if not isinstance(obj, dict): return None for key, val in obj.items(): current = f'{path}.{key}' if path else key if key == 'castai-kvisor' and isinstance(val, dict): - # Verify it looks like real kvisor config (has agent/controller/castai keys) - kvisor_keys = set(val.keys()) - if kvisor_keys & {'agent', 'controller', 'castai', 'enabled'}: + if set(val.keys()) & KVISOR_DISCRIMINATOR: return current result = find_kvisor_path(val, current) if result: @@ -700,6 +705,21 @@ def find_kvisor_path(obj, path=''): return None data = json.load(sys.stdin) + +# Karpenter Enterprise (kent) umbrellas ship an additional kent-specific +# castai-kvisor copy alongside the generic one. Both render same-named +# resources; only the kent copy is active for kent customers. Prefer it +# explicitly — but validate kvisor-shaped children so a future stub or +# rename falls through to the generic search instead of routing flags +# into a black hole. +kent = data.get('kent', {}) +kent_kvisor = kent.get('castai-kvisor', {}) if isinstance(kent, dict) else {} +if isinstance(kent, dict) and kent.get('enabled') is True \ + and isinstance(kent_kvisor, dict) \ + and (set(kent_kvisor.keys()) & KVISOR_DISCRIMINATOR): + print('kent.castai-kvisor') + sys.exit(0) + path = find_kvisor_path(data) if path: print(path) @@ -709,6 +729,9 @@ else: " 2>/dev/null) || VALUES_PREFIX="castai-kvisor" fi ok "Detected umbrella chart (release: $RELEASE, chart: $DETECTED_CHART_NAME)" + if [[ "$VALUES_PREFIX" == "kent.castai-kvisor" ]]; then + info "Detected kent (Karpenter Enterprise) mode — using kent.castai-kvisor prefix" + fi info "Auto-configured: --release $RELEASE --chart $CHART --values-prefix $VALUES_PREFIX" ;; *) @@ -1022,6 +1045,13 @@ if kubectl $KUBECTL_CTX get crd clickhouseinstallations.clickhouse.altinity.com warn "ClickHouseInstallation CRD exists but operator is not running — will install operator" fi fi +# Snapshot the pre-Phase-1 state. The second-chance detect_operator in the +# CRD/Operator Detection block (needed for print-only mode) re-runs after +# Phase 1 may have installed the operator, which would otherwise make Phase 2 +# misclassify our just-installed operator as "external, pre-existing" and set +# operator.enabled=false on it. +OPERATOR_PRE_EXISTING="$OPERATOR_RUNNING" +debug "OPERATOR_PRE_EXISTING snapshot captured: '${OPERATOR_PRE_EXISTING:-}'" if [[ -n "$OPERATOR_RUNNING" ]]; then : # nothing to do @@ -1343,6 +1373,58 @@ fi # ── Phase 2: Enable Full Reliability Stack ─────────────────────────────────── step "Phase 2: Enabling Full Reliability Stack" +# Reliability metrics need both kvisor agent (OBI eBPF) and controller running. +# Some umbrella profiles ship with agent.enabled=false by default — silently +# leaving us without an agent. Detect that for upgrades and notify the user +# we're flipping it on as a prerequisite. We always pass --set ...agent.enabled +# / controller.enabled below so install mode picks up the right defaults +# regardless of profile. Failures reading prior values are surfaced as a warn +# (not silently swallowed) so a wedged release isn't masked. +if [[ -z "$INSTALL_MODE" ]]; then + HELM_ERR=$(mktemp) + PY_ERR=$(mktemp) + # Cleanup on any exit path (set -e, normal return, signal). Single trap covers + # both files; the cleanup is idempotent so re-entering the block is fine. + trap 'rm -f "$HELM_ERR" "$PY_ERR"' EXIT + PRIOR_STATE=$(helm $HELM_CTX get values -a "$RELEASE" -n "$NAMESPACE" -o json 2>"$HELM_ERR" \ + | VALUES_PREFIX="$VALUES_PREFIX" python3 -c " +import sys, json, os +data = json.load(sys.stdin) +prefix = os.environ.get('VALUES_PREFIX', '') +node = data +for part in (prefix.split('.') if prefix else []): + node = node.get(part, {}) if isinstance(node, dict) else {} +def norm(v): + if v is None: return 'unset' + if v is True or (isinstance(v, str) and v.lower() == 'true'): return 'enabled' + if v is False or (isinstance(v, str) and v.lower() == 'false'): return 'disabled' + return 'unknown' +agent = norm(node.get('agent', {}).get('enabled') if isinstance(node, dict) else None) +ctrl = norm(node.get('controller', {}).get('enabled') if isinstance(node, dict) else None) +print(f'{agent}|{ctrl}') +" 2>"$PY_ERR") || PRIOR_STATE="error|error" + PRIOR_AGENT="${PRIOR_STATE%|*}" + PRIOR_CTRL="${PRIOR_STATE#*|}" + debug "Phase 2 prereq read: PRIOR_AGENT=$PRIOR_AGENT, PRIOR_CTRL=$PRIOR_CTRL" + if [[ "$PRIOR_AGENT" == "error" ]]; then + # Distinguish helm vs python failure source so the operator can debug the right thing. + if [[ -s "$HELM_ERR" ]]; then + warn "Could not read prior release values (helm error: $(head -1 "$HELM_ERR")) — proceeding with chart defaults" + elif [[ -s "$PY_ERR" ]]; then + warn "Could not parse prior release values (values parse error: $(head -1 "$PY_ERR")) — proceeding with chart defaults" + else + warn "Could not read prior release values (unknown error) — proceeding with chart defaults" + fi + else + if [[ "$PRIOR_AGENT" == "disabled" ]]; then + info "kvisor agent resolves to disabled in the active profile (${VALUES_PREFIX:+${VALUES_PREFIX}.}agent.enabled=false) — enabling as a reliability-metrics prerequisite" + fi + if [[ "$PRIOR_CTRL" == "disabled" ]]; then + info "kvisor controller resolves to disabled in the active profile (${VALUES_PREFIX:+${VALUES_PREFIX}.}controller.enabled=false) — enabling as a reliability-metrics prerequisite" + fi + fi +fi + # Build the helm command with all reliability flags HELM_CMD="$(build_helm_base)" @@ -1364,14 +1446,28 @@ if [[ -n "$VALUES_PREFIX" ]]; then $(setkey enabled=true)" fi HELM_CMD="$HELM_CMD \\ + $(setkey agent.enabled=true) \\ $(setkey agent.reliabilityMetrics.enabled=true) \\ $(setkey agent.reliabilityMetrics.obi.sizingProfile=$OBI_PROFILE) \\ $(setkey agent.reliabilityMetrics.obi.dynamicSizing=$DYNAMIC_SIZING) \\ + $(setkey controller.enabled=true) \\ $(setkey controller.reliabilityMetrics.enabled=true) \\ $(setkey reliabilityMetrics.enabled=true) \\ $(setkey reliabilityMetrics.install.enabled=true) \\ $(setkey reliabilityMetrics.exporter.enabled=true)" +# In kent mode the umbrella also ships an autoscaler.castai-kvisor copy that +# renders same-named resources but with reliability metrics off. Disabling it +# prevents the strategic-merge collisions that orphan volumeMounts (seen as +# `volumeMounts[N].name: Not found` validation errors from kubectl). +# Safe for kent users — autoscaler's kvisor copy is unused there. +if [[ "$VALUES_PREFIX" == "kent.castai-kvisor" ]]; then + HELM_CMD="$HELM_CMD \\ + --set autoscaler.castai-kvisor.enabled=false" + info "Disabling duplicate autoscaler.castai-kvisor subchart (kent ships its own copy)" + debug "Auto-injected --set autoscaler.castai-kvisor.enabled=false to suppress duplicate render" +fi + # Override the ch-exporter API key secret ref if we detected a non-default secret if [[ -z "$INSTALL_MODE" && -n "$DETECTED_API_KEY_SECRET" && "$DETECTED_API_KEY_SECRET" != "castai-kvisor" ]]; then HELM_CMD="$HELM_CMD \\ @@ -1390,10 +1486,11 @@ if [[ "$RELEASE" != "castai-kvisor" ]]; then info "Overriding ClickHouse address → $CH_ADDR" fi -# Only skip installing our operator if an external one was already running before we started. -# If we installed it in Phase 1 (either CRD was missing, or CRD existed but operator wasn't running), -# we need operator.enabled=true so it stays managed by this release. -if [[ -n "$OPERATOR_RUNNING" ]]; then +# Only skip installing our operator if an external one was already running BEFORE +# this script started — checked via OPERATOR_PRE_EXISTING (snapshot before Phase 1). +# Using OPERATOR_RUNNING here would misclassify the operator that Phase 1 just +# installed as "external" and tear it down, orphaning the ClickHouseInstallation. +if [[ -n "$OPERATOR_PRE_EXISTING" ]]; then HELM_CMD="$HELM_CMD \\ $(setkey reliabilityMetrics.operator.enabled=false)" info "Using existing ClickHouse operator (not installing chart's operator)" @@ -1619,83 +1716,92 @@ if [[ -z "$DRY_RUN" ]]; then fi fi - # ── 3g. Data pipeline — wait for exporter to push metrics ────────────────── + # ── 3g. Data pipeline — Bronze → Silver → mothership ────────────────────── + # Helm reports "deployed" the moment its manifests apply; that doesn't prove + # data is flowing. Probe the pipeline end-to-end so a wedged collector / busted + # MV / unreachable mothership doesn't slip past us. Gauge metrics from the + # controller's k8s_cluster receiver populate within seconds of startup, so + # they're the cheapest signal that the whole path is alive. Histogram tables + # (http/grpc/db/messaging) stay empty until application traffic flows — we + # don't gate on them at install time. if [[ -n "$CH_POD" && "$CH_READY" == "true" && -n "$MIGRATE_DONE" ]]; then - info "Waiting for data to flow through the pipeline..." - info "This may take 2-4 minutes (OBI → Collector → ClickHouse → Exporter)" - - # Wait for Bronze data (OBI → Collector → ClickHouse) - PIPELINE_TRIES=0 - PIPELINE_MAX=60 - BRONZE_OK="" - SILVER_OK="" - EXPORT_OK="" - while [[ $PIPELINE_TRIES -lt $PIPELINE_MAX ]]; do - # Check Bronze: both histogram (HTTP/gRPC/DB) and gauge (k8s state) tables - if [[ -z "$BRONZE_OK" ]]; then - BRONZE_COUNT=$(kubectl $KUBECTL_CTX exec "$CH_POD" -n "$NAMESPACE" -c clickhouse -- \ - clickhouse-client -d metrics -q " - SELECT sum(c) FROM ( - SELECT count() as c FROM otel_metrics_histogram WHERE TimeUnix > now() - INTERVAL 5 MINUTE - UNION ALL SELECT count() FROM otel_metrics_gauge WHERE TimeUnix > now() - INTERVAL 5 MINUTE - )" 2>/dev/null) || BRONZE_COUNT="0" - - if [[ "${BRONZE_COUNT:-0}" -gt 0 ]]; then - BRONZE_OK="true" - ok "Bronze data flowing ($BRONZE_COUNT rows in last 5 min)" + info "Probing data pipeline (Bronze → Silver → mothership)..." + CH_QUERY_ERR=$(mktemp) + trap 'rm -f "$CH_QUERY_ERR"' EXIT + # Run a single-row clickhouse query and report: + # exit 0 + stdout = numeric value (success) + # exit 1 + stderr in $CH_QUERY_ERR (kubectl/exec/clickhouse failure — distinguish from empty result) + # `LIMIT 1` defends against schema drift returning multiple rows that would + # concatenate into a fake-large number after the whitespace strip. + ch_query() { + local q="$1" + kubectl $KUBECTL_CTX exec "$CH_POD" -n "$NAMESPACE" -c clickhouse -- \ + clickhouse-client -d metrics -q "$q LIMIT 1" 2>"$CH_QUERY_ERR" | tr -d '[:space:]' + } + # If the very first probe call fails to reach ClickHouse, surface that + # immediately rather than waiting for the 180s timeout to mislead the user. + CH_REACHABLE="true" + if ! ch_query "SELECT 1" >/dev/null 2>&1 && [[ -s "$CH_QUERY_ERR" ]]; then + warn "Cannot reach ClickHouse via kubectl exec ($(head -1 "$CH_QUERY_ERR")) — skipping pipeline probe" + CH_REACHABLE="" + VERIFY_FAILURES=$((VERIFY_FAILURES + 1)) + fi + PROBE_TIMEOUT=180 + PROBE_ELAPSED=0 + BRONZE_ROWS="" + SILVER_ROWS="" + EXPORT_TS="" + if [[ -n "$CH_REACHABLE" ]]; then + while (( PROBE_ELAPSED < PROBE_TIMEOUT )); do + if [[ -z "$BRONZE_ROWS" ]]; then + v=$(ch_query "SELECT count() FROM otel_metrics_gauge WHERE TimeUnix > now() - INTERVAL 2 MINUTE") + [[ "$v" =~ ^[0-9]+$ && "$v" -gt 0 ]] && BRONZE_ROWS="$v" && debug "Bronze probe: $BRONZE_ROWS rows" fi - fi - - # Check Silver tables (can appear quickly via MVs, check independently of Bronze) - if [[ -z "$SILVER_OK" ]]; then - SILVER_COUNT=$(kubectl $KUBECTL_CTX exec "$CH_POD" -n "$NAMESPACE" -c clickhouse -- \ - clickhouse-client -d metrics -q " - SELECT sum(c) FROM ( - SELECT count() as c FROM reliability_metrics_http - UNION ALL SELECT count() FROM reliability_metrics_grpc - UNION ALL SELECT count() FROM reliability_metrics_gauge - )" 2>/dev/null) || SILVER_COUNT="0" - - if [[ "${SILVER_COUNT:-0}" -gt 0 ]]; then - SILVER_OK="true" - ok "Silver data aggregated ($SILVER_COUNT rows)" + if [[ -n "$BRONZE_ROWS" && -z "$SILVER_ROWS" ]]; then + v=$(ch_query "SELECT count() FROM reliability_metrics_gauge WHERE timestamp > now() - INTERVAL 5 MINUTE") + [[ "$v" =~ ^[0-9]+$ && "$v" -gt 0 ]] && SILVER_ROWS="$v" && debug "Silver probe: $SILVER_ROWS rows" fi - fi + if [[ -n "$SILVER_ROWS" && -z "$EXPORT_TS" ]]; then + v=$(ch_query "SELECT toUnixTimestamp(max(last_exported_time)) FROM export_progress FINAL") + [[ "$v" =~ ^[0-9]+$ && "$v" -gt 0 ]] && EXPORT_TS="$v" && debug "Export cursor: $EXPORT_TS" + fi + [[ -n "$BRONZE_ROWS" && -n "$SILVER_ROWS" && -n "$EXPORT_TS" ]] && break + + # Progress hint every ~30s. + if (( PROBE_ELAPSED > 0 && PROBE_ELAPSED % 30 == 0 )); then + BPART="bronze …"; [[ -n "$BRONZE_ROWS" ]] && BPART="bronze ✓" + SPART="silver …"; [[ -n "$SILVER_ROWS" ]] && SPART="silver ✓" + EPART="export …"; [[ -n "$EXPORT_TS" ]] && EPART="export ✓" + info " ${PROBE_ELAPSED}s elapsed ($BPART, $SPART, $EPART)" + fi + sleep 10 + PROBE_ELAPSED=$((PROBE_ELAPSED + 10)) + done - # Check exporter logs (check every iteration — exporter may already be running) - if [[ -z "$EXPORT_OK" ]]; then - EXPORTER_LOG=$(kubectl $KUBECTL_CTX logs "$CH_POD" -n "$NAMESPACE" -c ch-exporter --tail=50 2>/dev/null) || EXPORTER_LOG="" - if echo "$EXPORTER_LOG" | grep -q "rows exported to CastAI\|data is synced to CastAI"; then - EXPORT_OK="true" - EXPORTED_TABLES=$(echo "$EXPORTER_LOG" | grep -o 'table=[^ ]*' | sort -u | sed 's/table=//' | tr '\n' ', ' | sed 's/,$//') - ok "Exporter pushing metrics to mothership (tables: $EXPORTED_TABLES)" + if [[ -n "$BRONZE_ROWS" ]]; then + ok "Bronze: $BRONZE_ROWS gauge rows in last 2 min" + else + # Differentiate "ClickHouse went away mid-probe" from "no data flowing". + if [[ -s "$CH_QUERY_ERR" ]]; then + warn "Bronze probe failed (last clickhouse error: $(head -1 "$CH_QUERY_ERR")) — check ClickHouse pod health" + else + warn "Bronze empty after ${PROBE_TIMEOUT}s — check agent OTel collector logs" fi + VERIFY_FAILURES=$((VERIFY_FAILURES + 1)) fi - - # All three confirmed — done - if [[ -n "$BRONZE_OK" && -n "$SILVER_OK" && -n "$EXPORT_OK" ]]; then - break + if [[ -n "$SILVER_ROWS" ]]; then + ok "Silver: $SILVER_ROWS gauge rows aggregated (1-min windows)" + else + warn "Silver empty — Bronze→Silver Materialized View may have failed" + VERIFY_FAILURES=$((VERIFY_FAILURES + 1)) fi - - PIPELINE_TRIES=$((PIPELINE_TRIES + 1)) - # Show progress every 15 seconds - if [[ $((PIPELINE_TRIES % 5)) -eq 0 ]]; then - ELAPSED=$((PIPELINE_TRIES * 3)) - STATUS_PARTS="" - [[ -n "$BRONZE_OK" ]] && STATUS_PARTS="bronze ✓" || STATUS_PARTS="bronze …" - [[ -n "$SILVER_OK" ]] && STATUS_PARTS="$STATUS_PARTS, silver ✓" || STATUS_PARTS="$STATUS_PARTS, silver …" - [[ -n "$EXPORT_OK" ]] && STATUS_PARTS="$STATUS_PARTS, export ✓" || STATUS_PARTS="$STATUS_PARTS, export …" - info " ${ELAPSED}s elapsed ($STATUS_PARTS)" + if [[ -n "$EXPORT_TS" ]]; then + ok "ch-exporter forwarding to mothership (cursor advanced past epoch)" + else + warn "Export cursor still at epoch — check ch-exporter logs for connectivity issues" + VERIFY_FAILURES=$((VERIFY_FAILURES + 1)) fi - sleep 3 - done - - if [[ -z "$BRONZE_OK" || -z "$SILVER_OK" || -z "$EXPORT_OK" ]]; then - echo "" - [[ -z "$BRONZE_OK" ]] && warn "No Bronze data after $((PIPELINE_MAX * 3))s — check OBI and collector logs" - [[ -n "$BRONZE_OK" && -z "$SILVER_OK" ]] && warn "Bronze data present but no Silver data yet — materialized views may need more time" - [[ -z "$EXPORT_OK" && -n "$SILVER_OK" ]] && warn "Exporter hasn't sent to mothership yet (120s export delay for data settlement — may resolve shortly)" - VERIFY_FAILURES=$((VERIFY_FAILURES + 1)) + info "(Histogram tables — http/grpc/db/messaging — populate when application traffic flows; not checked here.)" fi fi diff --git a/docs/reliability-stack-installation.md b/docs/reliability-stack-installation.md index 146debd3..0684411e 100644 --- a/docs/reliability-stack-installation.md +++ b/docs/reliability-stack-installation.md @@ -109,6 +109,12 @@ The `enable-reliability-stack.sh` script handles the ClickHouse Operator CRD boo The script **auto-detects** whether kvisor is installed standalone (`castai-kvisor` chart) or via the CAST AI umbrella chart (`castai` chart) and adjusts the release name, chart reference, and values prefix accordingly. No manual flags needed in most cases. +It also auto-handles a few umbrella-specific edge cases: + +- **Karpenter Enterprise (kent) mode**: when `kent.enabled=true` is detected, the script picks the `kent.castai-kvisor` values prefix and auto-injects `--set autoscaler.castai-kvisor.enabled=false` to suppress the umbrella's duplicate `castai-kvisor` copy that would otherwise strategic-merge over kent's render. +- **Disabled-by-default kvisor agent/controller**: some umbrella profiles ship with `agent.enabled=false` (e.g. kent's cluster-proxy-only flavor). The script detects this for upgrades, prints a notice, and always passes `--set ...agent.enabled=true / controller.enabled=true` so install mode picks up the right defaults regardless of profile. +- **Phase 3 data-pipeline verification**: after the helm upgrade succeeds, the script waits for the migrate Job + ClickHouse StatefulSet, then probes Bronze → Silver → mothership end-to-end (gauge metrics) to confirm the pipeline is actually flowing — not just that helm marked the release `deployed`. + ```bash # Basic usage — auto-detects standalone vs umbrella, auto-detects OBI profile ./charts/kvisor/scripts/enable-reliability-stack.sh @@ -178,12 +184,15 @@ helm install castai-kvisor castai-helm/castai-kvisor \ > **⚠️ Umbrella Chart** > -> When kvisor is installed via the `castai` umbrella chart (not standalone `castai-kvisor`), three things differ: +> When kvisor is installed via the `castai` umbrella chart (not standalone `castai-kvisor`), four things differ: > 1. The API key secret is `castai-credentials` (not `castai-kvisor`) -> 2. All `--set` keys must be prefixed to route into the kvisor subchart (e.g. `autoscaler.castai-kvisor.*` — the exact prefix depends on the umbrella chart structure) +> 2. All `--set` keys must be prefixed to route into the kvisor subchart. The exact prefix depends on the umbrella's active profile: +> - **Karpenter Enterprise (kent) mode** (`kent.enabled=true`): use `kent.castai-kvisor.*` AND set `autoscaler.castai-kvisor.enabled=false` to suppress the duplicate copy +> - **Other umbrella variants**: typically `autoscaler.castai-kvisor.*` > 3. The ClickHouse service name becomes `castai-clickhouse` (not `castai-kvisor-clickhouse`) +> 4. Some profiles disable kvisor's `agent`/`controller` by default — you must explicitly set `agent.enabled=true` and `controller.enabled=true` under the chosen prefix > -> **Recommended:** Use the `enable-reliability-stack.sh` script — it auto-detects umbrella vs standalone and discovers the correct values prefix automatically. +> **Recommended:** Use the `enable-reliability-stack.sh` script — it handles all four points automatically. > > Manual example for reference (verify the prefix for your umbrella chart version): > ```bash @@ -508,18 +517,23 @@ The `enable-reliability-stack.sh` script automatically verifies all components a 2. Controller Deployment rollout (k8s_cluster receiver) 3. ClickHouse pod readiness (waits for PVC provisioning + operator reconciliation) 4. Migration job completion (Bronze/Silver table creation) -5. End-to-end data pipeline: Bronze data → Silver aggregation → Exporter pushing to CAST AI +5. End-to-end data pipeline: Bronze gauge → Silver gauge → ch-exporter cursor advancing past epoch -The pipeline check takes 2-4 minutes to confirm data flows all the way through. You'll see progress updates like: +The pipeline probe takes ~30–90s on a healthy cluster — gauge metrics from the controller's `k8s_cluster` receiver populate within seconds of startup, the Materialized View aggregates them into Silver every minute, and the export cursor advances 5 seconds later. Output looks like: ``` -✓ Bronze data flowing (142 rows in last 5 min) -✓ Silver data aggregated (38 rows) -✓ Exporter pushing metrics to mothership (tables: reliability_metrics_http, reliability_metrics_gauge) +✓ Bronze: 2448 gauge rows in last 2 min +✓ Silver: 5768 gauge rows aggregated (1-min windows) +✓ ch-exporter forwarding to mothership (cursor advanced past epoch) +ℹ (Histogram tables — http/grpc/db/messaging — populate when application traffic flows; not checked here.) ``` +Histogram tables only populate when something instrumentable (HTTP/gRPC/DB/messaging) is running in the cluster, so they're not gated at install time. + ### Manual Verification -### 1. Check Pod Status +The steps below mirror the automated checks above. Run them after a manual install, or to investigate what failed when Phase 3 reports a problem. + +#### 1. Check Pod Status ```bash kubectl get pods -n castai-agent -l app.kubernetes.io/instance=castai-kvisor @@ -534,7 +548,7 @@ Expected pods: | `chi-castai-kvisor-clickhouse-otel-0-0-0` | 2/2 (clickhouse, ch-exporter) | ClickHouse + exporter | | `castai-kvisor-clickhouse-operator-*` | 2/2 | Altinity operator (if `reliabilityMetrics.operator.enabled=true`) | -### 2. Verify OBI Instrumentation +#### 2. Verify OBI Instrumentation ```bash POD=$(kubectl get pods -n castai-agent \ @@ -548,7 +562,7 @@ Expected output: level=INFO msg="instrumenting process" cmd=myapp pid=1234 type=go ``` -### 3. Verify OTel Collectors +#### 3. Verify OTel Collectors ```bash # Agent collector @@ -561,7 +575,7 @@ kubectl logs -l app.kubernetes.io/name=castai-kvisor-controller \ Should show `Everything is ready. Begin running and processing data.` -### 4. Verify ClickHouse Data +#### 4. Verify ClickHouse Data ```bash CH_POD=$(kubectl get pods -n castai-agent \ @@ -584,7 +598,7 @@ kubectl exec $CH_POD -n castai-agent -c clickhouse -- \ FORMAT PrettyCompact" ``` -### 5. Verify ch-exporter +#### 5. Verify ch-exporter ```bash kubectl logs $CH_POD -n castai-agent -c ch-exporter --tail=20