refactor: deprecate clickhouse.json feature gate in favor of per-exporter json config

Replace the deprecated --feature-gates=clickhouse.json CLI flag with the
per-exporter json: true config option, as recommended by the upstream
OpenTelemetry ClickHouse exporter (v0.149.0).

The old OTEL_AGENT_FEATURE_GATE_ARG env var is still supported for other
feature gates. When it contains clickhouse.json, the entrypoint strips
that specific gate, maps it to HYPERDX_CLICKHOUSE_JSON_ENABLED=true, and
prints a deprecation warning. Any remaining feature gates are preserved
and passed through to the collector binary as before.

Changes:
- Add HYPERDX_CLICKHOUSE_JSON_ENABLED env var as the new canonical config
- Add json field to ClickHouse exporter configs in opampController.ts
- Add json field to standalone config (config.standalone.yaml)
- Strip clickhouse.json from OTEL_AGENT_FEATURE_GATE_ARG with deprecation
  warning, preserving other feature gates
- Update docker-compose.dev.yml to use the new env var

Ref: HDX-3994

Author: wrn14897

docker-compose.dev.yml

@@ -1,27 +1,27 @@
 name: ${HDX_DEV_PROJECT:-hdx-oss-dev}
 x-hdx-labels: &hdx-labels
-  hdx.dev.slot: '${HDX_DEV_SLOT:-0}'
-  hdx.dev.branch: '${HDX_DEV_BRANCH:-unknown}'
-  hdx.dev.worktree: '${HDX_DEV_WORKTREE:-unknown}'
+  hdx.dev.slot: "${HDX_DEV_SLOT:-0}"
+  hdx.dev.branch: "${HDX_DEV_BRANCH:-unknown}"
+  hdx.dev.worktree: "${HDX_DEV_WORKTREE:-unknown}"
 services:
   db:
     labels:
       <<: *hdx-labels
       hdx.dev.service: mongodb
-      hdx.dev.port: '${HDX_DEV_MONGO_PORT:-27017}'
+      hdx.dev.port: "${HDX_DEV_MONGO_PORT:-27017}"
     image: mongo:5.0.32-focal
     volumes:
       - .volumes/db_dev_${HDX_DEV_SLOT:-0}:/data/db
     ports:
-      - '${HDX_DEV_MONGO_PORT:-27017}:27017'
+      - "${HDX_DEV_MONGO_PORT:-27017}:27017"
     networks:
       - internal
   otel-collector:
     labels:
       <<: *hdx-labels
       hdx.dev.service: otel-collector
-      hdx.dev.port: '${HDX_DEV_OTEL_HTTP_PORT:-4318}'
-      hdx.dev.url: 'http://localhost:${HDX_DEV_OTEL_HTTP_PORT:-4318}'
+      hdx.dev.port: "${HDX_DEV_OTEL_HTTP_PORT:-4318}"
+      hdx.dev.url: "http://localhost:${HDX_DEV_OTEL_HTTP_PORT:-4318}"
     build:
       context: .
       dockerfile: docker/otel-collector/Dockerfile
@@ -30,26 +30,26 @@ services:
         OTEL_COLLECTOR_VERSION: ${OTEL_COLLECTOR_VERSION}
         OTEL_COLLECTOR_CORE_VERSION: ${OTEL_COLLECTOR_CORE_VERSION}
     environment:
-      CLICKHOUSE_ENDPOINT: 'tcp://ch-server:9000?dial_timeout=10s'
-      CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT: 'ch-server:9363'
+      CLICKHOUSE_ENDPOINT: "tcp://ch-server:9000?dial_timeout=10s"
+      CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT: "ch-server:9363"
       HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: ${HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE}
       HYPERDX_API_KEY: ${HYPERDX_API_KEY}
       HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
-      OPAMP_SERVER_URL: 'http://host.docker.internal:${HYPERDX_OPAMP_PORT:-4320}'
-      CUSTOM_OTELCOL_CONFIG_FILE: '/etc/otelcol-contrib/custom.config.yaml'
+      OPAMP_SERVER_URL: "http://host.docker.internal:${HYPERDX_OPAMP_PORT:-4320}"
+      CUSTOM_OTELCOL_CONFIG_FILE: "/etc/otelcol-contrib/custom.config.yaml"
       # Uncomment to enable stdout logging for the OTel collector
-      OTEL_SUPERVISOR_LOGS: 'true'
-      HYPERDX_OTEL_EXPORTER_TABLES_TTL: '24h'
+      OTEL_SUPERVISOR_LOGS: "true"
+      HYPERDX_OTEL_EXPORTER_TABLES_TTL: "24h"
     volumes:
       - ./docker/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml
       - ./docker/otel-collector/supervisor_docker.yaml.tmpl:/etc/otel/supervisor.yaml.tmpl
       # Add a custom config file
       - ./docker/otel-collector/custom.config.yaml:/etc/otelcol-contrib/custom.config.yaml
     ports:
-      - '${HDX_DEV_OTEL_HEALTH_PORT:-13133}:13133' # health_check extension
-      - '${HDX_DEV_OTEL_GRPC_PORT:-4317}:4317' # OTLP gRPC receiver
-      - '${HDX_DEV_OTEL_HTTP_PORT:-4318}:4318' # OTLP http receiver
-      - '${HDX_DEV_OTEL_METRICS_PORT:-8888}:8888' # metrics extension
+      - "${HDX_DEV_OTEL_HEALTH_PORT:-13133}:13133" # health_check extension
+      - "${HDX_DEV_OTEL_GRPC_PORT:-4317}:4317" # OTLP gRPC receiver
+      - "${HDX_DEV_OTEL_HTTP_PORT:-4318}:4318" # OTLP http receiver
+      - "${HDX_DEV_OTEL_METRICS_PORT:-8888}:8888" # metrics extension
     restart: always
     networks:
       - internal
@@ -60,8 +60,8 @@ services:
     labels:
       <<: *hdx-labels
       hdx.dev.service: otel-collector-json
-      hdx.dev.port: '${HDX_DEV_OTEL_JSON_HTTP_PORT:-14318}'
-      hdx.dev.url: 'http://localhost:${HDX_DEV_OTEL_JSON_HTTP_PORT:-14318}'
+      hdx.dev.port: "${HDX_DEV_OTEL_JSON_HTTP_PORT:-14318}"
+      hdx.dev.url: "http://localhost:${HDX_DEV_OTEL_JSON_HTTP_PORT:-14318}"
     build:
       context: .
       dockerfile: docker/otel-collector/Dockerfile
@@ -70,24 +70,25 @@ services:
         OTEL_COLLECTOR_VERSION: ${OTEL_COLLECTOR_VERSION}
         OTEL_COLLECTOR_CORE_VERSION: ${OTEL_COLLECTOR_CORE_VERSION}
     environment:
-      CLICKHOUSE_ENDPOINT: 'tcp://ch-server:9000?dial_timeout=10s'
-      CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT: 'ch-server:9363'
-      HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: 'otel_json'
-      HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA: 'true'
+      CLICKHOUSE_ENDPOINT: "tcp://ch-server:9000?dial_timeout=10s"
+      CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT: "ch-server:9363"
+      HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: "otel_json"
+      HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA: "true"
       HYPERDX_API_KEY: ${HYPERDX_API_KEY}
       HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
-      OPAMP_SERVER_URL: 'http://host.docker.internal:${HYPERDX_OPAMP_PORT:-4320}'
-      CUSTOM_OTELCOL_CONFIG_FILE: '/etc/otelcol-contrib/custom.config.yaml'
+      OPAMP_SERVER_URL: "http://host.docker.internal:${HYPERDX_OPAMP_PORT:-4320}"
+      CUSTOM_OTELCOL_CONFIG_FILE: "/etc/otelcol-contrib/custom.config.yaml"
       # Uncomment to enable stdout logging for the OTel collector
-      OTEL_SUPERVISOR_LOGS: 'true'
-      # Uncomment to enable JSON schema in ClickHouse
+      OTEL_SUPERVISOR_LOGS: "true"
+      # Enable JSON schema in the ClickHouse exporter (per-exporter config)
       # Be sure to also set BETA_CH_OTEL_JSON_SCHEMA_ENABLED to 'true' in ch-server
-      OTEL_AGENT_FEATURE_GATE_ARG: '--feature-gates=clickhouse.json'
+      # HYPERDX_CLICKHOUSE_JSON_ENABLED: 'true'
+      OTEL_AGENT_FEATURE_GATE_ARG: "--feature-gates=clickhouse.json"
     volumes:
       - ./docker/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml
       - ./docker/otel-collector/supervisor_docker.yaml.tmpl:/etc/otel/supervisor.yaml.tmpl
     ports:
-      - '${HDX_DEV_OTEL_JSON_HTTP_PORT:-14318}:4318' # OTLP http receiver
+      - "${HDX_DEV_OTEL_JSON_HTTP_PORT:-14318}:4318" # OTLP http receiver
     restart: always
     networks:
       - internal
@@ -98,18 +99,18 @@ services:
     labels:
       <<: *hdx-labels
       hdx.dev.service: clickhouse
-      hdx.dev.port: '${HDX_DEV_CH_HTTP_PORT:-8123}'
-      hdx.dev.url: 'http://localhost:${HDX_DEV_CH_HTTP_PORT:-8123}'
+      hdx.dev.port: "${HDX_DEV_CH_HTTP_PORT:-8123}"
+      hdx.dev.url: "http://localhost:${HDX_DEV_CH_HTTP_PORT:-8123}"
     image: clickhouse/clickhouse-server:26.1-alpine
     ports:
-      - '${HDX_DEV_CH_HTTP_PORT:-8123}:8123' # http api
-      - '${HDX_DEV_CH_NATIVE_PORT:-9000}:9000' # native
+      - "${HDX_DEV_CH_HTTP_PORT:-8123}:8123" # http api
+      - "${HDX_DEV_CH_NATIVE_PORT:-9000}:9000" # native
     environment:
       # default settings
       CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
       HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: ${HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE}
       # Set to 'true' to allow for proper OTel JSON Schema creation
-      # Be sure to also set the OTEL_AGENT_FEATURE_GATE_ARG env in otel-collector
+      # Be sure to also set HYPERDX_CLICKHOUSE_JSON_ENABLED in otel-collector
       # BETA_CH_OTEL_JSON_SCHEMA_ENABLED: 'true'
     volumes:
       - ./docker/clickhouse/local/config.xml:/etc/clickhouse-server/config.xml

docker/otel-collector/config.standalone.yaml

@@ -33,6 +33,7 @@ exporters:
     logs_table_name: hyperdx_sessions
     timeout: 5s
     create_schema: ${env:HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA:-false}
+    json: ${env:HYPERDX_CLICKHOUSE_JSON_ENABLED:-false}
     retry_on_failure:
       enabled: true
       initial_interval: 5s
@@ -46,6 +47,7 @@ exporters:
     ttl: 720h
     timeout: 5s
     create_schema: ${env:HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA:-false}
+    json: ${env:HYPERDX_CLICKHOUSE_JSON_ENABLED:-false}
     retry_on_failure:
       enabled: true
       initial_interval: 5s

docker/otel-collector/entrypoint.sh

@@ -1,8 +1,26 @@
 #!/bin/sh
 set -e
 
-# Fall back to legacy schema when the ClickHouse JSON feature gate is enabled
+# DEPRECATED: The clickhouse.json feature gate has been removed upstream.
+# When OTEL_AGENT_FEATURE_GATE_ARG contains clickhouse.json, strip it and
+# map it to HYPERDX_CLICKHOUSE_JSON_ENABLED instead. Other feature gates
+# are preserved and passed through to the collector.
 if echo "$OTEL_AGENT_FEATURE_GATE_ARG" | grep -q "clickhouse.json"; then
+  echo "WARNING: '--feature-gates=clickhouse.json' is deprecated and no longer supported by the collector."
+  echo "WARNING: Use HYPERDX_CLICKHOUSE_JSON_ENABLED=true instead. This flag will be removed in a future release."
+  export HYPERDX_CLICKHOUSE_JSON_ENABLED=true
+
+  # Strip clickhouse.json from the feature gates, keeping any other gates
+  REMAINING_GATES=$(echo "$OTEL_AGENT_FEATURE_GATE_ARG" | sed 's/--feature-gates=//' | tr ',' '\n' | grep -v 'clickhouse.json' | tr '\n' ',' | sed 's/,$//')
+  if [ -n "$REMAINING_GATES" ]; then
+    export OTEL_AGENT_FEATURE_GATE_ARG="--feature-gates=$REMAINING_GATES"
+  else
+    unset OTEL_AGENT_FEATURE_GATE_ARG
+  fi
+fi
+
+# Fall back to legacy schema when ClickHouse JSON exporter mode is enabled
+if [ "$HYPERDX_CLICKHOUSE_JSON_ENABLED" = "true" ]; then
   export HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA=true
 fi
 
@@ -39,7 +57,7 @@ if [ -z "$OPAMP_SERVER_URL" ]; then
     COLLECTOR_ARGS="$COLLECTOR_ARGS --config $CUSTOM_OTELCOL_CONFIG_FILE"
   fi
 
-  # Pass feature gates to the collector in standalone mode
+  # Pass remaining feature gates to the collector in standalone mode
   if [ -n "$OTEL_AGENT_FEATURE_GATE_ARG" ]; then
     COLLECTOR_ARGS="$COLLECTOR_ARGS $OTEL_AGENT_FEATURE_GATE_ARG"
   fi

docker/otel-collector/supervisor_docker.yaml.tmpl

@@ -23,8 +23,8 @@ agent:
 {{- if getenv "CUSTOM_OTELCOL_CONFIG_FILE" }}
     - {{ getenv "CUSTOM_OTELCOL_CONFIG_FILE" }}
 {{- end }}
-  args:
 {{- if getenv "OTEL_AGENT_FEATURE_GATE_ARG" }}
+  args:
     - {{ getenv "OTEL_AGENT_FEATURE_GATE_ARG" }}
 {{- end }}
 

packages/api/src/opamp/controllers/opampController.ts

@@ -82,6 +82,7 @@ type CollectorConfig = {
       logs_table_name: string;
       timeout: string;
       create_schema: string;
+      json: string;
       retry_on_failure: {
         enabled: boolean;
         initial_interval: string;
@@ -97,6 +98,7 @@ type CollectorConfig = {
       ttl: string;
       timeout: string;
       create_schema: string;
+      json: string;
       retry_on_failure: {
         enabled: boolean;
         initial_interval: string;
@@ -205,6 +207,7 @@ export const buildOtelCollectorConfig = (
         timeout: '5s',
         create_schema:
           '${env:HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA:-false}',
+        json: '${env:HYPERDX_CLICKHOUSE_JSON_ENABLED:-false}',
         retry_on_failure: {
           enabled: true,
           initial_interval: '5s',
@@ -221,6 +224,7 @@ export const buildOtelCollectorConfig = (
         timeout: '5s',
         create_schema:
           '${env:HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA:-false}',
+        json: '${env:HYPERDX_CLICKHOUSE_JSON_ENABLED:-false}',
         retry_on_failure: {
           enabled: true,
           initial_interval: '5s',